geoblacklight_sidecar_images 0.8.1 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +77 -0
- data/.gitignore +1 -0
- data/.solr_wrapper.yml +2 -0
- data/.standard.yml +1 -0
- data/Gemfile +22 -13
- data/README.md +5 -7
- data/Rakefile +52 -16
- data/app/jobs/geoblacklight_sidecar_images/store_image_job.rb +2 -2
- data/app/mailers/geoblacklight_sidecar_images/application_mailer.rb +2 -2
- data/app/models/concerns/wms_rewrite_concern.rb +1 -1
- data/app/models/sidecar_image_state_machine.rb +6 -6
- data/app/models/sidecar_image_transition.rb +1 -1
- data/app/services/geoblacklight_sidecar_images/image_service/iiif.rb +1 -1
- data/app/services/geoblacklight_sidecar_images/image_service/wms.rb +4 -4
- data/app/services/geoblacklight_sidecar_images/image_service.rb +45 -45
- data/db/migrate/20180118203155_create_solr_document_sidecars.rb +1 -1
- data/db/migrate/20180118203519_create_sidecar_image_transitions.rb +6 -6
- data/geoblacklight_sidecar_images.gemspec +30 -31
- data/lib/generators/geoblacklight_sidecar_images/config_generator.rb +13 -4
- data/lib/generators/geoblacklight_sidecar_images/example_docs_generator.rb +3 -3
- data/lib/generators/geoblacklight_sidecar_images/install_generator.rb +14 -14
- data/lib/generators/geoblacklight_sidecar_images/jobs_generator.rb +3 -3
- data/lib/generators/geoblacklight_sidecar_images/models_generator.rb +7 -7
- data/lib/generators/geoblacklight_sidecar_images/templates/config/initializers/statesman.rb +1 -1
- data/lib/generators/geoblacklight_sidecar_images/views_generator.rb +4 -4
- data/lib/geoblacklight_sidecar_images/version.rb +1 -1
- data/lib/geoblacklight_sidecar_images.rb +1 -1
- data/lib/tasks/geoblacklight_sidecar_images_tasks.rake +61 -68
- data/solr/conf/schema.xml +4 -6
- data/spec/fixtures/files/esri-tiled_map_layer.json +7 -7
- data/spec/fixtures/files/umn_solr_thumb.json +1 -1
- data/spec/jobs/store_image_job_spec.rb +4 -4
- data/spec/lib/tasks/geoblacklight_sidecar_images_spec.rb +33 -0
- data/spec/models/solr_document_sidecar_spec.rb +4 -4
- data/spec/models/solr_document_spec.rb +5 -5
- data/spec/rails_helper.rb +10 -12
- data/spec/services/image_service_spec.rb +70 -21
- data/spec/spec_helper.rb +32 -0
- data/spec/test_app_templates/Gemfile.extra +1 -0
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +12 -12
- data/template.rb +9 -9
- metadata +57 -104
- data/.coveralls.yml +0 -1
- data/.rubocop.yml +0 -81
- data/.travis.yml +0 -29
@@ -1,40 +1,39 @@
|
|
1
|
-
$LOAD_PATH.push File.expand_path(
|
1
|
+
$LOAD_PATH.push File.expand_path("../lib", __FILE__)
|
2
2
|
|
3
3
|
# Maintain your gem's version:
|
4
|
-
require
|
4
|
+
require "geoblacklight_sidecar_images/version"
|
5
5
|
|
6
6
|
# Describe your gem and declare its dependencies:
|
7
7
|
Gem::Specification.new do |s|
|
8
|
-
s.name
|
9
|
-
s.version
|
10
|
-
s.authors
|
11
|
-
s.email
|
12
|
-
s.homepage
|
13
|
-
s.summary
|
14
|
-
s.license
|
8
|
+
s.name = "geoblacklight_sidecar_images"
|
9
|
+
s.version = GeoblacklightSidecarImages::VERSION
|
10
|
+
s.authors = ["Eric Larson", "Eliot Jordan"]
|
11
|
+
s.email = ["ewlarson@gmail.com"]
|
12
|
+
s.homepage = "https://github.com/geoblacklight/geoblacklight_sidecar_images"
|
13
|
+
s.summary = "Store local copies of remote imagery in GeoBlacklight"
|
14
|
+
s.license = "Apache 2.0"
|
15
15
|
|
16
|
-
s.files
|
17
|
-
s.
|
18
|
-
s.
|
19
|
-
s.require_paths = ['lib']
|
16
|
+
s.files = `git ls-files -z`.split(%(\x0))
|
17
|
+
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
20
19
|
|
21
|
-
s.add_dependency
|
22
|
-
s.add_dependency
|
23
|
-
s.add_dependency
|
24
|
-
s.add_dependency
|
25
|
-
s.add_dependency
|
26
|
-
s.add_dependency
|
20
|
+
s.add_dependency "faraday", ">= 1.0"
|
21
|
+
s.add_dependency "geoblacklight", "~> 3.3"
|
22
|
+
s.add_dependency "image_processing", "~> 1.6"
|
23
|
+
s.add_dependency "mimemagic", "~> 0.3"
|
24
|
+
s.add_dependency "mini_magick", "~> 4.9.4"
|
25
|
+
s.add_dependency "rails", ">= 5.2", "< 7.1"
|
26
|
+
s.add_dependency "statesman", ">= 3.4"
|
27
27
|
|
28
|
-
s.add_development_dependency
|
29
|
-
s.add_development_dependency
|
30
|
-
s.add_development_dependency
|
31
|
-
s.add_development_dependency
|
32
|
-
s.add_development_dependency
|
33
|
-
s.add_development_dependency
|
34
|
-
s.add_development_dependency
|
35
|
-
s.add_development_dependency
|
36
|
-
s.add_development_dependency
|
37
|
-
s.add_development_dependency
|
38
|
-
s.add_development_dependency
|
39
|
-
s.add_development_dependency 'sprockets', '< 4'
|
28
|
+
s.add_development_dependency "byebug"
|
29
|
+
s.add_development_dependency "capybara"
|
30
|
+
s.add_development_dependency "webdrivers"
|
31
|
+
s.add_development_dependency "database_cleaner", "~> 1.3"
|
32
|
+
s.add_development_dependency "engine_cart", "~> 2.2"
|
33
|
+
s.add_development_dependency "rspec-rails", "~> 3.0"
|
34
|
+
s.add_development_dependency "selenium-webdriver"
|
35
|
+
s.add_development_dependency "simplecov", "~> 0.22"
|
36
|
+
s.add_development_dependency "solr_wrapper", "~> 4.0"
|
37
|
+
s.add_development_dependency "sprockets", "< 4"
|
38
|
+
s.add_development_dependency "standard", "~> 1.24"
|
40
39
|
end
|
@@ -1,18 +1,27 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails/generators"
|
4
4
|
|
5
5
|
module GeoblacklightSidecarImages
|
6
6
|
class ConfigGenerator < Rails::Generators::Base
|
7
|
-
source_root File.expand_path(
|
7
|
+
source_root File.expand_path("templates", __dir__)
|
8
8
|
|
9
9
|
desc <<-DESCRIPTION
|
10
10
|
This generator makes the following changes to your application:
|
11
11
|
1. Copies config files to host config
|
12
12
|
DESCRIPTION
|
13
13
|
|
14
|
-
def
|
15
|
-
|
14
|
+
def set_active_storage_processor
|
15
|
+
app_config = <<-"APP"
|
16
|
+
|
17
|
+
config.active_storage.variant_processor = :mini_magick
|
18
|
+
APP
|
19
|
+
|
20
|
+
inject_into_file "config/application.rb", app_config, after: "config.generators.system_tests = nil"
|
21
|
+
end
|
22
|
+
|
23
|
+
def create_statesman_initializer
|
24
|
+
copy_file "config/initializers/statesman.rb", "config/initializers/statesman.rb"
|
16
25
|
end
|
17
26
|
end
|
18
27
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails/generators"
|
4
4
|
|
5
5
|
module GeoblacklightSidecarImages
|
6
6
|
class ExampleDocsGenerator < Rails::Generators::Base
|
7
|
-
source_root File.expand_path(
|
7
|
+
source_root File.expand_path("../../../spec/fixtures", __dir__)
|
8
8
|
|
9
9
|
desc <<-DESCRIPTION
|
10
10
|
This generator makes the following changes to your application:
|
@@ -12,7 +12,7 @@ module GeoblacklightSidecarImages
|
|
12
12
|
DESCRIPTION
|
13
13
|
|
14
14
|
def create_services
|
15
|
-
directory
|
15
|
+
directory "files", "solr/geoblacklight/example_docs"
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
@@ -1,14 +1,14 @@
|
|
1
|
-
require
|
1
|
+
require "rails/generators"
|
2
2
|
|
3
3
|
module GeoblacklightSidecarImages
|
4
4
|
class Install < Rails::Generators::Base
|
5
|
-
source_root File.expand_path(
|
5
|
+
source_root File.expand_path("templates", __dir__)
|
6
6
|
|
7
|
-
desc
|
7
|
+
desc "Install GeoblacklightSidecarImages"
|
8
8
|
|
9
9
|
def add_settings_vars
|
10
|
-
inject_into_file
|
11
|
-
"INSTITUTION_LOCAL_NAME: 'Princeton'
|
10
|
+
inject_into_file "config/settings.yml", after: "INSTITUTION: 'Stanford'\n" do
|
11
|
+
"INSTITUTION_LOCAL_NAME: 'Princeton'
|
12
12
|
INSTITUTION_GEOSERVER_URL: 'https://geoserver.princeton.edu'
|
13
13
|
PROXY_GEOSERVER_URL: 'http://localhost:3000'
|
14
14
|
PROXY_GEOSERVER_AUTH: 'Basic base64encodedusername:password'
|
@@ -17,36 +17,36 @@ GBLSI_THUMBNAIL_FIELD: 'thumbnail_path_ss'"
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def generate_gblsci_assets
|
20
|
-
copy_file
|
20
|
+
copy_file "gblsci.scss", "app/assets/stylesheets/gblsci.scss"
|
21
21
|
end
|
22
22
|
|
23
23
|
def generate_gblsci_example_docs
|
24
|
-
generate
|
24
|
+
generate "geoblacklight_sidecar_images:example_docs"
|
25
25
|
end
|
26
26
|
|
27
27
|
def generate_gblsci_jobs
|
28
|
-
generate
|
28
|
+
generate "geoblacklight_sidecar_images:jobs"
|
29
29
|
end
|
30
30
|
|
31
31
|
def generate_gblsci_models
|
32
|
-
generate
|
32
|
+
generate "geoblacklight_sidecar_images:models"
|
33
33
|
end
|
34
34
|
|
35
35
|
def generate_gblsci_views
|
36
|
-
generate
|
36
|
+
generate "geoblacklight_sidecar_images:views"
|
37
37
|
end
|
38
38
|
|
39
39
|
def generate_action_storage
|
40
|
-
rake
|
40
|
+
rake "active_storage:install"
|
41
41
|
end
|
42
42
|
|
43
43
|
def generate_gblsci_config
|
44
|
-
generate
|
44
|
+
generate "geoblacklight_sidecar_images:config"
|
45
45
|
end
|
46
46
|
|
47
47
|
def bundle_install
|
48
|
-
Bundler.
|
49
|
-
run
|
48
|
+
Bundler.with_unbundled_env do
|
49
|
+
run "bundle install"
|
50
50
|
end
|
51
51
|
end
|
52
52
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails/generators"
|
4
4
|
|
5
5
|
module GeoblacklightSidecarImages
|
6
6
|
class JobsGenerator < Rails::Generators::Base
|
7
|
-
source_root File.expand_path(
|
7
|
+
source_root File.expand_path("templates", __dir__)
|
8
8
|
|
9
9
|
desc <<-DESCRIPTION
|
10
10
|
This generator makes the following changes to your application:
|
@@ -16,7 +16,7 @@ module GeoblacklightSidecarImages
|
|
16
16
|
config.active_job.queue_adapter = :inline
|
17
17
|
JOBS
|
18
18
|
|
19
|
-
inject_into_file
|
19
|
+
inject_into_file "config/environments/development.rb", job_config, before: /^end/
|
20
20
|
end
|
21
21
|
end
|
22
22
|
end
|
@@ -1,13 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "rails/generators"
|
4
|
+
require "rails/generators/migration"
|
5
5
|
|
6
6
|
module GeoblacklightSidecarImages
|
7
7
|
class ModelsGenerator < Rails::Generators::Base
|
8
8
|
include Rails::Generators::Migration
|
9
9
|
|
10
|
-
source_root File.expand_path(
|
10
|
+
source_root File.expand_path("templates", __dir__)
|
11
11
|
|
12
12
|
desc <<-DESCRIPTION
|
13
13
|
This generator makes the following changes to your application:
|
@@ -18,7 +18,7 @@ module GeoblacklightSidecarImages
|
|
18
18
|
|
19
19
|
# Setup the database migrations
|
20
20
|
def copy_migrations
|
21
|
-
rake
|
21
|
+
rake "geoblacklight_sidecar_images:install:migrations"
|
22
22
|
end
|
23
23
|
|
24
24
|
def include_sidecar_solrdocument
|
@@ -39,13 +39,13 @@ module GeoblacklightSidecarImages
|
|
39
39
|
end
|
40
40
|
SIDECAR
|
41
41
|
|
42
|
-
inject_into_file
|
42
|
+
inject_into_file "app/models/solr_document.rb", sidecar, before: /^end/
|
43
43
|
end
|
44
44
|
|
45
45
|
def include_wms_rewrite_solrdocument
|
46
46
|
inject_into_file(
|
47
|
-
|
48
|
-
after:
|
47
|
+
"app/models/solr_document.rb",
|
48
|
+
after: "include Geoblacklight::SolrDocument"
|
49
49
|
) do
|
50
50
|
"\n include WmsRewriteConcern"
|
51
51
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails/generators"
|
4
4
|
|
5
5
|
module GeoblacklightSidecarImages
|
6
6
|
class ViewsGenerator < Rails::Generators::Base
|
7
|
-
source_root File.expand_path(
|
7
|
+
source_root File.expand_path("templates", __dir__)
|
8
8
|
|
9
9
|
desc <<-DESCRIPTION
|
10
10
|
This generator makes the following changes to your application:
|
@@ -13,8 +13,8 @@ module GeoblacklightSidecarImages
|
|
13
13
|
|
14
14
|
def create_views
|
15
15
|
copy_file(
|
16
|
-
|
17
|
-
|
16
|
+
"views/catalog/_index_split_default.html.erb",
|
17
|
+
"app/views/catalog/_index_split_default.html.erb"
|
18
18
|
)
|
19
19
|
end
|
20
20
|
end
|
@@ -1,10 +1,10 @@
|
|
1
|
-
require
|
1
|
+
require "csv"
|
2
2
|
|
3
3
|
namespace :gblsci do
|
4
4
|
namespace :sample_data do
|
5
|
-
desc
|
5
|
+
desc "Ingests a directory of geoblacklight.json files"
|
6
6
|
task seed: :environment do
|
7
|
-
Dir.glob(File.join(Rails.root,
|
7
|
+
Dir.glob(File.join(Rails.root, "solr", "geoblacklight", "example_docs", "**", "*.json")).each do |fn|
|
8
8
|
puts "Ingesting #{fn}"
|
9
9
|
begin
|
10
10
|
Blacklight.default_index.connection.add(JSON.parse(File.read(fn)))
|
@@ -12,40 +12,38 @@ namespace :gblsci do
|
|
12
12
|
puts "Failed to ingest #{fn}: #{e.inspect}"
|
13
13
|
end
|
14
14
|
end
|
15
|
-
puts
|
15
|
+
puts "Committing changes to Solr"
|
16
16
|
Blacklight.default_index.connection.commit
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
namespace :images do
|
21
|
-
desc
|
22
|
-
task
|
23
|
-
GeoblacklightSidecarImages::StoreImageJob.perform_later(
|
21
|
+
desc "Harvest image for specific document"
|
22
|
+
task harvest_doc_id: :environment do
|
23
|
+
GeoblacklightSidecarImages::StoreImageJob.perform_later(ENV["DOC_ID"])
|
24
24
|
end
|
25
25
|
|
26
|
-
desc
|
26
|
+
desc "Harvest all images"
|
27
27
|
task harvest_all: :environment do
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
next
|
43
|
-
end
|
28
|
+
query = "*:*"
|
29
|
+
index = Geoblacklight::SolrDocument.index
|
30
|
+
results = index.send_and_receive(index.blacklight_config.solr_path,
|
31
|
+
q: query,
|
32
|
+
fl: "*",
|
33
|
+
rows: 100_000_000)
|
34
|
+
# num_found = results.response[:numFound]
|
35
|
+
# doc_counter = 0
|
36
|
+
results.docs.each do |document|
|
37
|
+
sleep(1)
|
38
|
+
begin
|
39
|
+
GeoblacklightSidecarImages::StoreImageJob.perform_later(document["layer_slug_s"])
|
40
|
+
rescue Blacklight::Exceptions::RecordNotFound
|
41
|
+
next
|
44
42
|
end
|
45
43
|
end
|
46
44
|
end
|
47
45
|
|
48
|
-
desc
|
46
|
+
desc "Hash of SolrDocumentSidecar image state counts"
|
49
47
|
task harvest_states: :environment do
|
50
48
|
states = [
|
51
49
|
:initialized,
|
@@ -62,12 +60,12 @@ namespace :gblsci do
|
|
62
60
|
col_state[state] = sidecars.size
|
63
61
|
end
|
64
62
|
|
65
|
-
col_state.each do |col,state|
|
63
|
+
col_state.each do |col, state|
|
66
64
|
puts "#{col} - #{state}"
|
67
65
|
end
|
68
66
|
end
|
69
67
|
|
70
|
-
desc
|
68
|
+
desc "Re-queues incomplete states for harvesting"
|
71
69
|
task harvest_retry: :environment do
|
72
70
|
states = [
|
73
71
|
:initialized,
|
@@ -83,24 +81,22 @@ namespace :gblsci do
|
|
83
81
|
puts "#{state} - #{sidecars.size}"
|
84
82
|
|
85
83
|
sidecars.each do |sc|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
puts "orphaned / #{sc.document_id}"
|
91
|
-
end
|
84
|
+
document = Geoblacklight::SolrDocument.find(sc.document_id)
|
85
|
+
GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
|
86
|
+
rescue
|
87
|
+
puts "orphaned / #{sc.document_id}"
|
92
88
|
end
|
93
89
|
end
|
94
90
|
end
|
95
91
|
|
96
|
-
desc
|
92
|
+
desc "Write harvest state report (CSV)"
|
97
93
|
task harvest_report: :environment do
|
98
94
|
# Create a CSV Dump of Results
|
99
|
-
file = "#{Rails.root}/public/#{Time.now.strftime(
|
95
|
+
file = "#{Rails.root}/public/#{Time.now.strftime("%Y-%m-%d_%H-%M-%S")}.sidecar_report.csv"
|
100
96
|
|
101
97
|
sidecars = SolrDocumentSidecar.all
|
102
98
|
|
103
|
-
CSV.open(file,
|
99
|
+
CSV.open(file, "w") do |writer|
|
104
100
|
header = [
|
105
101
|
"Sidecar ID",
|
106
102
|
"Document ID",
|
@@ -117,31 +113,30 @@ namespace :gblsci do
|
|
117
113
|
writer << header
|
118
114
|
|
119
115
|
sidecars.each do |sc|
|
120
|
-
cat = CatalogController.new
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
end
|
116
|
+
# cat = CatalogController.new
|
117
|
+
|
118
|
+
document = Geoblacklight::SolrDocument.find(sc.document_id)
|
119
|
+
writer << [
|
120
|
+
sc.id,
|
121
|
+
sc.document_id,
|
122
|
+
sc.image_state.current_state,
|
123
|
+
document._source["layer_geom_type_s"],
|
124
|
+
document._source["dc_title_s"],
|
125
|
+
document._source["dct_provenance_s"],
|
126
|
+
sc.image_state.last_transition.metadata["exception"],
|
127
|
+
sc.image_state.last_transition.metadata["viewer_protocol"],
|
128
|
+
sc.image_state.last_transition.metadata["image_url"],
|
129
|
+
sc.image_state.last_transition.metadata["gblsi_thumbnail_uri"]
|
130
|
+
]
|
131
|
+
rescue => e
|
132
|
+
puts "Exception: #{e.inspect}"
|
133
|
+
puts "orphaned / #{sc.document_id}"
|
134
|
+
next
|
140
135
|
end
|
141
136
|
end
|
142
137
|
end
|
143
138
|
|
144
|
-
desc
|
139
|
+
desc "Destroy all harvested images and sidecar AR objects"
|
145
140
|
task harvest_purge_all: :environment do
|
146
141
|
# Remove all images
|
147
142
|
sidecars = SolrDocumentSidecar.all
|
@@ -154,43 +149,41 @@ namespace :gblsci do
|
|
154
149
|
SolrDocumentSidecar.destroy_all
|
155
150
|
end
|
156
151
|
|
157
|
-
desc
|
152
|
+
desc "Destroy orphaned images and sidecar AR objects"
|
158
153
|
# When a SolrDocumentSidecar AR object exists,
|
159
154
|
# but it's corresponding SolrDocument is no longer in the Solr index.
|
160
155
|
task harvest_purge_orphans: :environment do
|
161
156
|
# Remove all images
|
162
157
|
sidecars = SolrDocumentSidecar.all
|
163
158
|
sidecars.each do |sc|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
puts "orphaned / #{sc.document_id} / destroyed"
|
169
|
-
end
|
159
|
+
Geoblacklight::SolrDocument.find(sc.document_id)
|
160
|
+
rescue
|
161
|
+
sc.destroy
|
162
|
+
puts "orphaned / #{sc.document_id} / destroyed"
|
170
163
|
end
|
171
164
|
end
|
172
165
|
|
173
|
-
desc
|
166
|
+
desc "Destroy select sidecar AR objects by CSV file"
|
174
167
|
task harvest_destroy_batch: :environment do
|
175
168
|
# Expects a CSV file in Rails.root/tmp/destroy_batch.csv
|
176
169
|
#
|
177
170
|
# From your local machine, copy it up to production server like this:
|
178
171
|
# scp destroy_batch.csv swadm@geoprod:/swadm/var/www/geoblacklight/current/tmp/
|
179
172
|
CSV.foreach("#{Rails.root}/tmp/destroy_batch.csv", headers: true) do |row|
|
180
|
-
sc = SolrDocumentSidecar.find_by(:
|
173
|
+
sc = SolrDocumentSidecar.find_by(document_id: row[0])
|
181
174
|
sc.destroy
|
182
175
|
puts "document_id - #{row[0]} - destroyed"
|
183
176
|
end
|
184
177
|
end
|
185
178
|
|
186
|
-
desc
|
179
|
+
desc "Inspect failed state objects"
|
187
180
|
task harvest_failed_state_inspect: :environment do
|
188
181
|
states = [
|
189
182
|
:failed
|
190
183
|
]
|
191
184
|
|
192
185
|
states.each do |state|
|
193
|
-
|
186
|
+
SolrDocumentSidecar.in_state(state).each do |sc|
|
194
187
|
puts "#{state} - #{sc.document_id} - #{sc.image_state.last_transition.metadata.inspect}"
|
195
188
|
end
|
196
189
|
end
|
data/solr/conf/schema.xml
CHANGED
@@ -143,13 +143,11 @@
|
|
143
143
|
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_d"/>
|
144
144
|
|
145
145
|
<fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
|
146
|
-
geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers"/>
|
146
|
+
geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
|
147
147
|
<!-- Adding field type for bboxField that enables, among other things, overlap ratio calculations -->
|
148
148
|
<fieldType name="bbox" class="solr.BBoxField"
|
149
149
|
geo="true" distanceUnits="kilometers" numberType="pdouble" />
|
150
150
|
<fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
|
151
|
-
|
152
|
-
|
153
151
|
</types>
|
154
152
|
|
155
153
|
<!-- for scoring formula -->
|
@@ -169,8 +167,8 @@
|
|
169
167
|
<copyField source="layer_slug_s" dest="layer_slug_ti" maxChars="100"/>
|
170
168
|
|
171
169
|
<!-- core text search -->
|
172
|
-
<copyField source="*
|
173
|
-
<copyField source="*
|
170
|
+
<copyField source="*_s" dest="text" />
|
171
|
+
<copyField source="*_sm" dest="text" />
|
174
172
|
|
175
173
|
<!-- for sorting text fields -->
|
176
174
|
<copyField source="dct_provenance_s" dest="dct_provenance_sort"/>
|
@@ -195,4 +193,4 @@
|
|
195
193
|
|
196
194
|
<!-- for bbox value -->
|
197
195
|
<copyField source="solr_geom" dest="solr_bboxtype"/>
|
198
|
-
</schema>
|
196
|
+
</schema>
|
@@ -1,20 +1,20 @@
|
|
1
1
|
{
|
2
2
|
"geoblacklight_version": "1.0",
|
3
|
-
"dc_description_s": "This map
|
3
|
+
"dc_description_s": "This map presents a digital version of the Operational Navigation Charts (ONC) at 1:1,000,000-scale, produced by the US National Geospatial-Intelligence Agency (NGA). The map includes over 200 charts across the world, excluding parts of North America, Europe, Asia, and Oceania where charts are not publicly available. For more information on this map, including the terms of use, visit us online.",
|
4
4
|
"dc_format_s": "GeoTIFF",
|
5
|
-
"dc_identifier_s": "
|
5
|
+
"dc_identifier_s": "esri-tiled-map-layer",
|
6
6
|
"dc_language_s": "English",
|
7
7
|
"dc_publisher_s": "United States Department of Agriculture, Natural Resources Conservation Service",
|
8
8
|
"dc_rights_s": "Public",
|
9
|
-
"dc_title_s": "
|
9
|
+
"dc_title_s": "Specialty/World_Navigation_Charts (MapServer)",
|
10
10
|
"dc_type_s": "Dataset",
|
11
|
-
"dct_references_s": "{\"urn:x-esri:serviceType:ArcGIS#TiledMapLayer\": \"
|
11
|
+
"dct_references_s": "{\"urn:x-esri:serviceType:ArcGIS#TiledMapLayer\": \"https://services.arcgisonline.com/arcgis/rest/services/Specialty/World_Navigation_Charts/MapServer\"}",
|
12
12
|
"dct_temporal_sm": [
|
13
|
-
"
|
13
|
+
"2013"
|
14
14
|
],
|
15
15
|
"dct_provenance_s": "NYU",
|
16
|
-
"layer_slug_s": "
|
17
|
-
"layer_id_s": "
|
16
|
+
"layer_slug_s": "esri-tiled-map-layer",
|
17
|
+
"layer_id_s": "esri-tiled-map-layer",
|
18
18
|
"layer_geom_type_s": "Raster",
|
19
19
|
"layer_modified_dt": "2015-06-16T00:59:49Z",
|
20
20
|
"solr_geom": "ENVELOPE(-129.4956, -64.4393, 48.6336, 21.8079)",
|
@@ -14,7 +14,7 @@
|
|
14
14
|
"layer_slug_s":"aaf63e35-3d5f-4e62-b59d-377f008e9aad",
|
15
15
|
"dc_format_s":"TIFF",
|
16
16
|
"dc_creator_sm":["Minnesota. Department of Highways."],
|
17
|
-
"thumbnail_path_ss":"https://
|
17
|
+
"thumbnail_path_ss":"https://cdm16022.contentdm.oclc.org/utils/getthumbnail/collection/p16022coll206/id/133.jpg",
|
18
18
|
"dc_type_s":"Still image",
|
19
19
|
"dc_identifier_s":"aaf63e35-3d5f-4e62-b59d-377f008e9aad",
|
20
20
|
"dc_relation_sm":["http://sws.geonames.org/5037779/about/rdf"],
|
@@ -1,14 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails_helper"
|
4
4
|
|
5
5
|
describe GeoblacklightSidecarImages::StoreImageJob, type: :job do
|
6
6
|
let(:document) { SolrDocument.new(document_attributes) }
|
7
7
|
|
8
|
-
describe
|
9
|
-
let(:document_attributes) { json_data(
|
8
|
+
describe "#perform_later" do
|
9
|
+
let(:document_attributes) { json_data("umn_iiif_jpg") }
|
10
10
|
|
11
|
-
it
|
11
|
+
it "stores an image" do
|
12
12
|
ActiveJob::Base.queue_adapter = :test
|
13
13
|
expect { GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id) }.to have_enqueued_job(GeoblacklightSidecarImages::StoreImageJob)
|
14
14
|
end
|