geoblacklight_sidecar_images 0.8.1 → 0.9.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +77 -0
  3. data/.gitignore +1 -0
  4. data/.solr_wrapper.yml +2 -0
  5. data/.standard.yml +1 -0
  6. data/Gemfile +22 -13
  7. data/README.md +5 -7
  8. data/Rakefile +52 -16
  9. data/app/jobs/geoblacklight_sidecar_images/store_image_job.rb +2 -2
  10. data/app/mailers/geoblacklight_sidecar_images/application_mailer.rb +2 -2
  11. data/app/models/concerns/wms_rewrite_concern.rb +1 -1
  12. data/app/models/sidecar_image_state_machine.rb +6 -6
  13. data/app/models/sidecar_image_transition.rb +1 -1
  14. data/app/services/geoblacklight_sidecar_images/image_service/iiif.rb +1 -1
  15. data/app/services/geoblacklight_sidecar_images/image_service/wms.rb +4 -4
  16. data/app/services/geoblacklight_sidecar_images/image_service.rb +45 -45
  17. data/db/migrate/20180118203155_create_solr_document_sidecars.rb +1 -1
  18. data/db/migrate/20180118203519_create_sidecar_image_transitions.rb +6 -6
  19. data/geoblacklight_sidecar_images.gemspec +30 -31
  20. data/lib/generators/geoblacklight_sidecar_images/config_generator.rb +13 -4
  21. data/lib/generators/geoblacklight_sidecar_images/example_docs_generator.rb +3 -3
  22. data/lib/generators/geoblacklight_sidecar_images/install_generator.rb +14 -14
  23. data/lib/generators/geoblacklight_sidecar_images/jobs_generator.rb +3 -3
  24. data/lib/generators/geoblacklight_sidecar_images/models_generator.rb +7 -7
  25. data/lib/generators/geoblacklight_sidecar_images/templates/config/initializers/statesman.rb +1 -1
  26. data/lib/generators/geoblacklight_sidecar_images/views_generator.rb +4 -4
  27. data/lib/geoblacklight_sidecar_images/version.rb +1 -1
  28. data/lib/geoblacklight_sidecar_images.rb +1 -1
  29. data/lib/tasks/geoblacklight_sidecar_images_tasks.rake +61 -68
  30. data/solr/conf/schema.xml +4 -6
  31. data/spec/fixtures/files/esri-tiled_map_layer.json +7 -7
  32. data/spec/fixtures/files/umn_solr_thumb.json +1 -1
  33. data/spec/jobs/store_image_job_spec.rb +4 -4
  34. data/spec/lib/tasks/geoblacklight_sidecar_images_spec.rb +33 -0
  35. data/spec/models/solr_document_sidecar_spec.rb +4 -4
  36. data/spec/models/solr_document_spec.rb +5 -5
  37. data/spec/rails_helper.rb +10 -12
  38. data/spec/services/image_service_spec.rb +70 -21
  39. data/spec/spec_helper.rb +32 -0
  40. data/spec/test_app_templates/Gemfile.extra +1 -0
  41. data/spec/test_app_templates/lib/generators/test_app_generator.rb +12 -12
  42. data/template.rb +9 -9
  43. metadata +57 -104
  44. data/.coveralls.yml +0 -1
  45. data/.rubocop.yml +0 -81
  46. data/.travis.yml +0 -29
@@ -1,40 +1,39 @@
1
- $LOAD_PATH.push File.expand_path('../lib', __FILE__)
1
+ $LOAD_PATH.push File.expand_path("../lib", __FILE__)
2
2
 
3
3
  # Maintain your gem's version:
4
- require 'geoblacklight_sidecar_images/version'
4
+ require "geoblacklight_sidecar_images/version"
5
5
 
6
6
  # Describe your gem and declare its dependencies:
7
7
  Gem::Specification.new do |s|
8
- s.name = 'geoblacklight_sidecar_images'
9
- s.version = GeoblacklightSidecarImages::VERSION
10
- s.authors = ['Eric Larson', 'Eliot Jordan']
11
- s.email = ['ewlarson@umn.edu']
12
- s.homepage = 'https://github.com/geoblacklight/geoblacklight_sidecar_images'
13
- s.summary = 'Store local copies of remote imagery in GeoBlacklight'
14
- s.license = 'Apache 2.0'
8
+ s.name = "geoblacklight_sidecar_images"
9
+ s.version = GeoblacklightSidecarImages::VERSION
10
+ s.authors = ["Eric Larson", "Eliot Jordan"]
11
+ s.email = ["ewlarson@gmail.com"]
12
+ s.homepage = "https://github.com/geoblacklight/geoblacklight_sidecar_images"
13
+ s.summary = "Store local copies of remote imagery in GeoBlacklight"
14
+ s.license = "Apache 2.0"
15
15
 
16
- s.files = `git ls-files -z`.split(%Q{\x0})
17
- s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
- s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
19
- s.require_paths = ['lib']
16
+ s.files = `git ls-files -z`.split(%(\x0))
17
+ s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
20
19
 
21
- s.add_dependency 'geoblacklight', '>= 3.3'
22
- s.add_dependency 'mini_magick', '~> 4.9.4'
23
- s.add_dependency 'image_processing', '~> 1.6'
24
- s.add_dependency 'statesman', '>= 3.4'
25
- s.add_dependency 'mimemagic', '~> 0.3'
26
- s.add_dependency 'rails', '>= 5.2', '< 6.2'
20
+ s.add_dependency "faraday", ">= 1.0"
21
+ s.add_dependency "geoblacklight", "~> 3.3"
22
+ s.add_dependency "image_processing", "~> 1.6"
23
+ s.add_dependency "mimemagic", "~> 0.3"
24
+ s.add_dependency "mini_magick", "~> 4.9.4"
25
+ s.add_dependency "rails", ">= 5.2", "< 7.1"
26
+ s.add_dependency "statesman", ">= 3.4"
27
27
 
28
- s.add_development_dependency 'byebug'
29
- s.add_development_dependency 'capybara'
30
- s.add_development_dependency 'webdrivers'
31
- s.add_development_dependency 'database_cleaner', '~> 1.3'
32
- s.add_development_dependency 'engine_cart', '~> 2.2'
33
- s.add_development_dependency 'rspec-rails', '~> 3.0'
34
- s.add_development_dependency 'rubocop', '~> 1.9'
35
- s.add_development_dependency 'rubocop-rspec', '~> 1.30.0'
36
- s.add_development_dependency 'rubocop-rails', '~> 2.9'
37
- s.add_development_dependency 'selenium-webdriver'
38
- s.add_development_dependency 'solr_wrapper', '~> 3.1'
39
- s.add_development_dependency 'sprockets', '< 4'
28
+ s.add_development_dependency "byebug"
29
+ s.add_development_dependency "capybara"
30
+ s.add_development_dependency "webdrivers"
31
+ s.add_development_dependency "database_cleaner", "~> 1.3"
32
+ s.add_development_dependency "engine_cart", "~> 2.2"
33
+ s.add_development_dependency "rspec-rails", "~> 3.0"
34
+ s.add_development_dependency "selenium-webdriver"
35
+ s.add_development_dependency "simplecov", "~> 0.22"
36
+ s.add_development_dependency "solr_wrapper", "~> 4.0"
37
+ s.add_development_dependency "sprockets", "< 4"
38
+ s.add_development_dependency "standard", "~> 1.24"
40
39
  end
@@ -1,18 +1,27 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
3
+ require "rails/generators"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  class ConfigGenerator < Rails::Generators::Base
7
- source_root File.expand_path('templates', __dir__)
7
+ source_root File.expand_path("templates", __dir__)
8
8
 
9
9
  desc <<-DESCRIPTION
10
10
  This generator makes the following changes to your application:
11
11
  1. Copies config files to host config
12
12
  DESCRIPTION
13
13
 
14
- def create_store_image_jobs
15
- copy_file 'config/initializers/statesman.rb', 'config/initializers/statesman.rb'
14
+ def set_active_storage_processor
15
+ app_config = <<-"APP"
16
+
17
+ config.active_storage.variant_processor = :mini_magick
18
+ APP
19
+
20
+ inject_into_file "config/application.rb", app_config, after: "config.generators.system_tests = nil"
21
+ end
22
+
23
+ def create_statesman_initializer
24
+ copy_file "config/initializers/statesman.rb", "config/initializers/statesman.rb"
16
25
  end
17
26
  end
18
27
  end
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
3
+ require "rails/generators"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  class ExampleDocsGenerator < Rails::Generators::Base
7
- source_root File.expand_path('../../../spec/fixtures', __dir__)
7
+ source_root File.expand_path("../../../spec/fixtures", __dir__)
8
8
 
9
9
  desc <<-DESCRIPTION
10
10
  This generator makes the following changes to your application:
@@ -12,7 +12,7 @@ module GeoblacklightSidecarImages
12
12
  DESCRIPTION
13
13
 
14
14
  def create_services
15
- directory 'files', 'solr/geoblacklight/example_docs'
15
+ directory "files", "solr/geoblacklight/example_docs"
16
16
  end
17
17
  end
18
18
  end
@@ -1,14 +1,14 @@
1
- require 'rails/generators'
1
+ require "rails/generators"
2
2
 
3
3
  module GeoblacklightSidecarImages
4
4
  class Install < Rails::Generators::Base
5
- source_root File.expand_path('templates', __dir__)
5
+ source_root File.expand_path("templates", __dir__)
6
6
 
7
- desc 'Install GeoblacklightSidecarImages'
7
+ desc "Install GeoblacklightSidecarImages"
8
8
 
9
9
  def add_settings_vars
10
- inject_into_file 'config/settings.yml', after: "INSTITUTION: 'Stanford'\n" do
11
- "INSTITUTION_LOCAL_NAME: 'Princeton'
10
+ inject_into_file "config/settings.yml", after: "INSTITUTION: 'Stanford'\n" do
11
+ "INSTITUTION_LOCAL_NAME: 'Princeton'
12
12
  INSTITUTION_GEOSERVER_URL: 'https://geoserver.princeton.edu'
13
13
  PROXY_GEOSERVER_URL: 'http://localhost:3000'
14
14
  PROXY_GEOSERVER_AUTH: 'Basic base64encodedusername:password'
@@ -17,36 +17,36 @@ GBLSI_THUMBNAIL_FIELD: 'thumbnail_path_ss'"
17
17
  end
18
18
 
19
19
  def generate_gblsci_assets
20
- copy_file 'gblsci.scss', 'app/assets/stylesheets/gblsci.scss'
20
+ copy_file "gblsci.scss", "app/assets/stylesheets/gblsci.scss"
21
21
  end
22
22
 
23
23
  def generate_gblsci_example_docs
24
- generate 'geoblacklight_sidecar_images:example_docs'
24
+ generate "geoblacklight_sidecar_images:example_docs"
25
25
  end
26
26
 
27
27
  def generate_gblsci_jobs
28
- generate 'geoblacklight_sidecar_images:jobs'
28
+ generate "geoblacklight_sidecar_images:jobs"
29
29
  end
30
30
 
31
31
  def generate_gblsci_models
32
- generate 'geoblacklight_sidecar_images:models'
32
+ generate "geoblacklight_sidecar_images:models"
33
33
  end
34
34
 
35
35
  def generate_gblsci_views
36
- generate 'geoblacklight_sidecar_images:views'
36
+ generate "geoblacklight_sidecar_images:views"
37
37
  end
38
38
 
39
39
  def generate_action_storage
40
- rake 'active_storage:install'
40
+ rake "active_storage:install"
41
41
  end
42
42
 
43
43
  def generate_gblsci_config
44
- generate 'geoblacklight_sidecar_images:config'
44
+ generate "geoblacklight_sidecar_images:config"
45
45
  end
46
46
 
47
47
  def bundle_install
48
- Bundler.with_clean_env do
49
- run 'bundle install'
48
+ Bundler.with_unbundled_env do
49
+ run "bundle install"
50
50
  end
51
51
  end
52
52
  end
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
3
+ require "rails/generators"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  class JobsGenerator < Rails::Generators::Base
7
- source_root File.expand_path('templates', __dir__)
7
+ source_root File.expand_path("templates", __dir__)
8
8
 
9
9
  desc <<-DESCRIPTION
10
10
  This generator makes the following changes to your application:
@@ -16,7 +16,7 @@ module GeoblacklightSidecarImages
16
16
  config.active_job.queue_adapter = :inline
17
17
  JOBS
18
18
 
19
- inject_into_file 'config/environments/development.rb', job_config, before: /^end/
19
+ inject_into_file "config/environments/development.rb", job_config, before: /^end/
20
20
  end
21
21
  end
22
22
  end
@@ -1,13 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
4
- require 'rails/generators/migration'
3
+ require "rails/generators"
4
+ require "rails/generators/migration"
5
5
 
6
6
  module GeoblacklightSidecarImages
7
7
  class ModelsGenerator < Rails::Generators::Base
8
8
  include Rails::Generators::Migration
9
9
 
10
- source_root File.expand_path('templates', __dir__)
10
+ source_root File.expand_path("templates", __dir__)
11
11
 
12
12
  desc <<-DESCRIPTION
13
13
  This generator makes the following changes to your application:
@@ -18,7 +18,7 @@ module GeoblacklightSidecarImages
18
18
 
19
19
  # Setup the database migrations
20
20
  def copy_migrations
21
- rake 'geoblacklight_sidecar_images:install:migrations'
21
+ rake "geoblacklight_sidecar_images:install:migrations"
22
22
  end
23
23
 
24
24
  def include_sidecar_solrdocument
@@ -39,13 +39,13 @@ module GeoblacklightSidecarImages
39
39
  end
40
40
  SIDECAR
41
41
 
42
- inject_into_file 'app/models/solr_document.rb', sidecar, before: /^end/
42
+ inject_into_file "app/models/solr_document.rb", sidecar, before: /^end/
43
43
  end
44
44
 
45
45
  def include_wms_rewrite_solrdocument
46
46
  inject_into_file(
47
- 'app/models/solr_document.rb',
48
- after: 'include Geoblacklight::SolrDocument'
47
+ "app/models/solr_document.rb",
48
+ after: "include Geoblacklight::SolrDocument"
49
49
  ) do
50
50
  "\n include WmsRewriteConcern"
51
51
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'statesman'
3
+ require "statesman"
4
4
 
5
5
  Statesman.configure do
6
6
  storage_adapter(Statesman::Adapters::ActiveRecord)
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
3
+ require "rails/generators"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  class ViewsGenerator < Rails::Generators::Base
7
- source_root File.expand_path('templates', __dir__)
7
+ source_root File.expand_path("templates", __dir__)
8
8
 
9
9
  desc <<-DESCRIPTION
10
10
  This generator makes the following changes to your application:
@@ -13,8 +13,8 @@ module GeoblacklightSidecarImages
13
13
 
14
14
  def create_views
15
15
  copy_file(
16
- 'views/catalog/_index_split_default.html.erb',
17
- 'app/views/catalog/_index_split_default.html.erb'
16
+ "views/catalog/_index_split_default.html.erb",
17
+ "app/views/catalog/_index_split_default.html.erb"
18
18
  )
19
19
  end
20
20
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module GeoblacklightSidecarImages
4
- VERSION = '0.8.1'
4
+ VERSION = "0.9.1"
5
5
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'geoblacklight_sidecar_images/engine'
3
+ require "geoblacklight_sidecar_images/engine"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  # Your code goes here...
@@ -1,10 +1,10 @@
1
- require 'csv'
1
+ require "csv"
2
2
 
3
3
  namespace :gblsci do
4
4
  namespace :sample_data do
5
- desc 'Ingests a directory of geoblacklight.json files'
5
+ desc "Ingests a directory of geoblacklight.json files"
6
6
  task seed: :environment do
7
- Dir.glob(File.join(Rails.root, 'solr', 'geoblacklight', 'example_docs', '**', '*.json')).each do |fn|
7
+ Dir.glob(File.join(Rails.root, "solr", "geoblacklight", "example_docs", "**", "*.json")).each do |fn|
8
8
  puts "Ingesting #{fn}"
9
9
  begin
10
10
  Blacklight.default_index.connection.add(JSON.parse(File.read(fn)))
@@ -12,40 +12,38 @@ namespace :gblsci do
12
12
  puts "Failed to ingest #{fn}: #{e.inspect}"
13
13
  end
14
14
  end
15
- puts 'Committing changes to Solr'
15
+ puts "Committing changes to Solr"
16
16
  Blacklight.default_index.connection.commit
17
17
  end
18
18
  end
19
19
 
20
20
  namespace :images do
21
- desc 'Harvest image for specific document'
22
- task :harvest_doc_id, [:doc_id] => [:environment] do |_t, args|
23
- GeoblacklightSidecarImages::StoreImageJob.perform_later(args[:doc_id])
21
+ desc "Harvest image for specific document"
22
+ task harvest_doc_id: :environment do
23
+ GeoblacklightSidecarImages::StoreImageJob.perform_later(ENV["DOC_ID"])
24
24
  end
25
25
 
26
- desc 'Harvest all images'
26
+ desc "Harvest all images"
27
27
  task harvest_all: :environment do
28
- begin
29
- query = '*:*'
30
- index = Geoblacklight::SolrDocument.index
31
- results = index.send_and_receive(index.blacklight_config.solr_path,
32
- q: query,
33
- fl: "*",
34
- rows: 100_000_000)
35
- num_found = results.response[:numFound]
36
- doc_counter = 0
37
- results.docs.each do |document|
38
- sleep(1)
39
- begin
40
- GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
41
- rescue Blacklight::Exceptions::RecordNotFound
42
- next
43
- end
28
+ query = "*:*"
29
+ index = Geoblacklight::SolrDocument.index
30
+ results = index.send_and_receive(index.blacklight_config.solr_path,
31
+ q: query,
32
+ fl: "*",
33
+ rows: 100_000_000)
34
+ # num_found = results.response[:numFound]
35
+ # doc_counter = 0
36
+ results.docs.each do |document|
37
+ sleep(1)
38
+ begin
39
+ GeoblacklightSidecarImages::StoreImageJob.perform_later(document["layer_slug_s"])
40
+ rescue Blacklight::Exceptions::RecordNotFound
41
+ next
44
42
  end
45
43
  end
46
44
  end
47
45
 
48
- desc 'Hash of SolrDocumentSidecar image state counts'
46
+ desc "Hash of SolrDocumentSidecar image state counts"
49
47
  task harvest_states: :environment do
50
48
  states = [
51
49
  :initialized,
@@ -62,12 +60,12 @@ namespace :gblsci do
62
60
  col_state[state] = sidecars.size
63
61
  end
64
62
 
65
- col_state.each do |col,state|
63
+ col_state.each do |col, state|
66
64
  puts "#{col} - #{state}"
67
65
  end
68
66
  end
69
67
 
70
- desc 'Re-queues incomplete states for harvesting'
68
+ desc "Re-queues incomplete states for harvesting"
71
69
  task harvest_retry: :environment do
72
70
  states = [
73
71
  :initialized,
@@ -83,24 +81,22 @@ namespace :gblsci do
83
81
  puts "#{state} - #{sidecars.size}"
84
82
 
85
83
  sidecars.each do |sc|
86
- begin
87
- document = Geoblacklight::SolrDocument.find(sc.document_id)
88
- GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
89
- rescue
90
- puts "orphaned / #{sc.document_id}"
91
- end
84
+ document = Geoblacklight::SolrDocument.find(sc.document_id)
85
+ GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
86
+ rescue
87
+ puts "orphaned / #{sc.document_id}"
92
88
  end
93
89
  end
94
90
  end
95
91
 
96
- desc 'Write harvest state report (CSV)'
92
+ desc "Write harvest state report (CSV)"
97
93
  task harvest_report: :environment do
98
94
  # Create a CSV Dump of Results
99
- file = "#{Rails.root}/public/#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}.sidecar_report.csv"
95
+ file = "#{Rails.root}/public/#{Time.now.strftime("%Y-%m-%d_%H-%M-%S")}.sidecar_report.csv"
100
96
 
101
97
  sidecars = SolrDocumentSidecar.all
102
98
 
103
- CSV.open(file, 'w') do |writer|
99
+ CSV.open(file, "w") do |writer|
104
100
  header = [
105
101
  "Sidecar ID",
106
102
  "Document ID",
@@ -117,31 +113,30 @@ namespace :gblsci do
117
113
  writer << header
118
114
 
119
115
  sidecars.each do |sc|
120
- cat = CatalogController.new
121
- begin
122
- document = Geoblacklight::SolrDocument.find(sc.document_id)
123
- writer << [
124
- sc.id,
125
- sc.document_id,
126
- sc.image_state.current_state,
127
- document._source['layer_geom_type_s'],
128
- document._source['dc_title_s'],
129
- document._source['dct_provenance_s'],
130
- sc.image_state.last_transition.metadata['exception'],
131
- sc.image_state.last_transition.metadata['viewer_protocol'],
132
- sc.image_state.last_transition.metadata['image_url'],
133
- sc.image_state.last_transition.metadata['gblsi_thumbnail_uri']
134
- ]
135
- rescue Exception => e
136
- puts "Exception: #{e.inspect}"
137
- puts "orphaned / #{sc.document_id}"
138
- next
139
- end
116
+ # cat = CatalogController.new
117
+
118
+ document = Geoblacklight::SolrDocument.find(sc.document_id)
119
+ writer << [
120
+ sc.id,
121
+ sc.document_id,
122
+ sc.image_state.current_state,
123
+ document._source["layer_geom_type_s"],
124
+ document._source["dc_title_s"],
125
+ document._source["dct_provenance_s"],
126
+ sc.image_state.last_transition.metadata["exception"],
127
+ sc.image_state.last_transition.metadata["viewer_protocol"],
128
+ sc.image_state.last_transition.metadata["image_url"],
129
+ sc.image_state.last_transition.metadata["gblsi_thumbnail_uri"]
130
+ ]
131
+ rescue => e
132
+ puts "Exception: #{e.inspect}"
133
+ puts "orphaned / #{sc.document_id}"
134
+ next
140
135
  end
141
136
  end
142
137
  end
143
138
 
144
- desc 'Destroy all harvested images and sidecar AR objects'
139
+ desc "Destroy all harvested images and sidecar AR objects"
145
140
  task harvest_purge_all: :environment do
146
141
  # Remove all images
147
142
  sidecars = SolrDocumentSidecar.all
@@ -154,43 +149,41 @@ namespace :gblsci do
154
149
  SolrDocumentSidecar.destroy_all
155
150
  end
156
151
 
157
- desc 'Destroy orphaned images and sidecar AR objects'
152
+ desc "Destroy orphaned images and sidecar AR objects"
158
153
  # When a SolrDocumentSidecar AR object exists,
159
154
  # but it's corresponding SolrDocument is no longer in the Solr index.
160
155
  task harvest_purge_orphans: :environment do
161
156
  # Remove all images
162
157
  sidecars = SolrDocumentSidecar.all
163
158
  sidecars.each do |sc|
164
- begin
165
- document = Geoblacklight::SolrDocument.find(sc.document_id)
166
- rescue
167
- sc.destroy
168
- puts "orphaned / #{sc.document_id} / destroyed"
169
- end
159
+ Geoblacklight::SolrDocument.find(sc.document_id)
160
+ rescue
161
+ sc.destroy
162
+ puts "orphaned / #{sc.document_id} / destroyed"
170
163
  end
171
164
  end
172
165
 
173
- desc 'Destroy select sidecar AR objects by CSV file'
166
+ desc "Destroy select sidecar AR objects by CSV file"
174
167
  task harvest_destroy_batch: :environment do
175
168
  # Expects a CSV file in Rails.root/tmp/destroy_batch.csv
176
169
  #
177
170
  # From your local machine, copy it up to production server like this:
178
171
  # scp destroy_batch.csv swadm@geoprod:/swadm/var/www/geoblacklight/current/tmp/
179
172
  CSV.foreach("#{Rails.root}/tmp/destroy_batch.csv", headers: true) do |row|
180
- sc = SolrDocumentSidecar.find_by(:document_id => row[0])
173
+ sc = SolrDocumentSidecar.find_by(document_id: row[0])
181
174
  sc.destroy
182
175
  puts "document_id - #{row[0]} - destroyed"
183
176
  end
184
177
  end
185
178
 
186
- desc 'Inspect failed state objects'
179
+ desc "Inspect failed state objects"
187
180
  task harvest_failed_state_inspect: :environment do
188
181
  states = [
189
182
  :failed
190
183
  ]
191
184
 
192
185
  states.each do |state|
193
- sidecars = SolrDocumentSidecar.in_state(state).each do |sc|
186
+ SolrDocumentSidecar.in_state(state).each do |sc|
194
187
  puts "#{state} - #{sc.document_id} - #{sc.image_state.last_transition.metadata.inspect}"
195
188
  end
196
189
  end
data/solr/conf/schema.xml CHANGED
@@ -143,13 +143,11 @@
143
143
  <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_d"/>
144
144
 
145
145
  <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
146
- geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers"/>
146
+ geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
147
147
  <!-- Adding field type for bboxField that enables, among other things, overlap ratio calculations -->
148
148
  <fieldType name="bbox" class="solr.BBoxField"
149
149
  geo="true" distanceUnits="kilometers" numberType="pdouble" />
150
150
  <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
151
-
152
-
153
151
  </types>
154
152
 
155
153
  <!-- for scoring formula -->
@@ -169,8 +167,8 @@
169
167
  <copyField source="layer_slug_s" dest="layer_slug_ti" maxChars="100"/>
170
168
 
171
169
  <!-- core text search -->
172
- <copyField source="*_ti" dest="text" />
173
- <copyField source="*_tmi" dest="text" />
170
+ <copyField source="*_s" dest="text" />
171
+ <copyField source="*_sm" dest="text" />
174
172
 
175
173
  <!-- for sorting text fields -->
176
174
  <copyField source="dct_provenance_s" dest="dct_provenance_sort"/>
@@ -195,4 +193,4 @@
195
193
 
196
194
  <!-- for bbox value -->
197
195
  <copyField source="solr_geom" dest="solr_bboxtype"/>
198
- </schema>
196
+ </schema>
@@ -1,20 +1,20 @@
1
1
  {
2
2
  "geoblacklight_version": "1.0",
3
- "dc_description_s": "This map shows the Soil Survey Geographic (SSURGO) by the United States Department of Agriculture's Natural Resources Conservation Service. It also shows data that was developed by the National Cooperative Soil Survey and supersedes the State Soil Geographic (STATSGO) dataset published in 1994. SSURGO digitizing duplicates the original soil survey maps. This level of mapping is designed for use by landowners, townships, and county natural resource planning and management. The user should be knowledgeable of soils data and their characteristics. The smallest scale map shows the Global Soil Regions map by the United States Department of Agriculture’s Natural Resources Conservation Service.",
3
+ "dc_description_s": "This map presents a digital version of the Operational Navigation Charts (ONC) at 1:1,000,000-scale, produced by the US National Geospatial-Intelligence Agency (NGA). The map includes over 200 charts across the world, excluding parts of North America, Europe, Asia, and Oceania where charts are not publicly available. For more information on this map, including the terms of use, visit us online.",
4
4
  "dc_format_s": "GeoTIFF",
5
- "dc_identifier_s": "test-soil-survey-map",
5
+ "dc_identifier_s": "esri-tiled-map-layer",
6
6
  "dc_language_s": "English",
7
7
  "dc_publisher_s": "United States Department of Agriculture, Natural Resources Conservation Service",
8
8
  "dc_rights_s": "Public",
9
- "dc_title_s": "Soil Survey Geographic (SSURGO)",
9
+ "dc_title_s": "Specialty/World_Navigation_Charts (MapServer)",
10
10
  "dc_type_s": "Dataset",
11
- "dct_references_s": "{\"urn:x-esri:serviceType:ArcGIS#TiledMapLayer\": \"http://services.arcgisonline.com/arcgis/rest/services/Specialty/Soil_Survey_Map/MapServer\"}",
11
+ "dct_references_s": "{\"urn:x-esri:serviceType:ArcGIS#TiledMapLayer\": \"https://services.arcgisonline.com/arcgis/rest/services/Specialty/World_Navigation_Charts/MapServer\"}",
12
12
  "dct_temporal_sm": [
13
- "2010"
13
+ "2013"
14
14
  ],
15
15
  "dct_provenance_s": "NYU",
16
- "layer_slug_s": "nyu-test-soil-survey-map",
17
- "layer_id_s": "test-soil-survey_map",
16
+ "layer_slug_s": "esri-tiled-map-layer",
17
+ "layer_id_s": "esri-tiled-map-layer",
18
18
  "layer_geom_type_s": "Raster",
19
19
  "layer_modified_dt": "2015-06-16T00:59:49Z",
20
20
  "solr_geom": "ENVELOPE(-129.4956, -64.4393, 48.6336, 21.8079)",
@@ -14,7 +14,7 @@
14
14
  "layer_slug_s":"aaf63e35-3d5f-4e62-b59d-377f008e9aad",
15
15
  "dc_format_s":"TIFF",
16
16
  "dc_creator_sm":["Minnesota. Department of Highways."],
17
- "thumbnail_path_ss":"https://umedia.lib.umn.edu/sites/default/files/imagecache/square300/reference/562/image/jpeg/1089695.jpg",
17
+ "thumbnail_path_ss":"https://cdm16022.contentdm.oclc.org/utils/getthumbnail/collection/p16022coll206/id/133.jpg",
18
18
  "dc_type_s":"Still image",
19
19
  "dc_identifier_s":"aaf63e35-3d5f-4e62-b59d-377f008e9aad",
20
20
  "dc_relation_sm":["http://sws.geonames.org/5037779/about/rdf"],
@@ -1,14 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails_helper'
3
+ require "rails_helper"
4
4
 
5
5
  describe GeoblacklightSidecarImages::StoreImageJob, type: :job do
6
6
  let(:document) { SolrDocument.new(document_attributes) }
7
7
 
8
- describe '#perform_later' do
9
- let(:document_attributes) { json_data('umn_iiif_jpg') }
8
+ describe "#perform_later" do
9
+ let(:document_attributes) { json_data("umn_iiif_jpg") }
10
10
 
11
- it 'stores an image' do
11
+ it "stores an image" do
12
12
  ActiveJob::Base.queue_adapter = :test
13
13
  expect { GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id) }.to have_enqueued_job(GeoblacklightSidecarImages::StoreImageJob)
14
14
  end