geoblacklight_sidecar_images 0.8.1 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +69 -0
  3. data/.gitignore +1 -0
  4. data/.solr_wrapper.yml +2 -0
  5. data/.standard.yml +1 -0
  6. data/Gemfile +22 -13
  7. data/README.md +5 -7
  8. data/Rakefile +52 -16
  9. data/app/jobs/geoblacklight_sidecar_images/store_image_job.rb +2 -2
  10. data/app/mailers/geoblacklight_sidecar_images/application_mailer.rb +2 -2
  11. data/app/models/concerns/wms_rewrite_concern.rb +1 -1
  12. data/app/models/sidecar_image_state_machine.rb +6 -6
  13. data/app/models/sidecar_image_transition.rb +1 -1
  14. data/app/services/geoblacklight_sidecar_images/image_service/iiif.rb +1 -1
  15. data/app/services/geoblacklight_sidecar_images/image_service/wms.rb +4 -4
  16. data/app/services/geoblacklight_sidecar_images/image_service.rb +46 -46
  17. data/db/migrate/20180118203155_create_solr_document_sidecars.rb +1 -1
  18. data/db/migrate/20180118203519_create_sidecar_image_transitions.rb +6 -6
  19. data/geoblacklight_sidecar_images.gemspec +30 -31
  20. data/lib/generators/geoblacklight_sidecar_images/config_generator.rb +13 -4
  21. data/lib/generators/geoblacklight_sidecar_images/example_docs_generator.rb +3 -3
  22. data/lib/generators/geoblacklight_sidecar_images/install_generator.rb +14 -14
  23. data/lib/generators/geoblacklight_sidecar_images/jobs_generator.rb +3 -3
  24. data/lib/generators/geoblacklight_sidecar_images/models_generator.rb +7 -7
  25. data/lib/generators/geoblacklight_sidecar_images/templates/config/initializers/statesman.rb +1 -1
  26. data/lib/generators/geoblacklight_sidecar_images/views_generator.rb +4 -4
  27. data/lib/geoblacklight_sidecar_images/version.rb +1 -1
  28. data/lib/geoblacklight_sidecar_images.rb +1 -1
  29. data/lib/tasks/geoblacklight_sidecar_images_tasks.rake +61 -68
  30. data/solr/conf/schema.xml +4 -6
  31. data/spec/fixtures/files/umn_solr_thumb.json +1 -1
  32. data/spec/jobs/store_image_job_spec.rb +4 -4
  33. data/spec/lib/tasks/geoblacklight_sidecar_images_spec.rb +33 -0
  34. data/spec/models/solr_document_sidecar_spec.rb +4 -4
  35. data/spec/models/solr_document_spec.rb +5 -5
  36. data/spec/rails_helper.rb +10 -12
  37. data/spec/services/image_service_spec.rb +24 -18
  38. data/spec/spec_helper.rb +32 -0
  39. data/spec/test_app_templates/Gemfile.extra +1 -0
  40. data/spec/test_app_templates/lib/generators/test_app_generator.rb +12 -12
  41. data/template.rb +9 -9
  42. metadata +59 -106
  43. data/.coveralls.yml +0 -1
  44. data/.rubocop.yml +0 -81
  45. data/.travis.yml +0 -29
@@ -1,40 +1,39 @@
1
- $LOAD_PATH.push File.expand_path('../lib', __FILE__)
1
+ $LOAD_PATH.push File.expand_path("../lib", __FILE__)
2
2
 
3
3
  # Maintain your gem's version:
4
- require 'geoblacklight_sidecar_images/version'
4
+ require "geoblacklight_sidecar_images/version"
5
5
 
6
6
  # Describe your gem and declare its dependencies:
7
7
  Gem::Specification.new do |s|
8
- s.name = 'geoblacklight_sidecar_images'
9
- s.version = GeoblacklightSidecarImages::VERSION
10
- s.authors = ['Eric Larson', 'Eliot Jordan']
11
- s.email = ['ewlarson@umn.edu']
12
- s.homepage = 'https://github.com/geoblacklight/geoblacklight_sidecar_images'
13
- s.summary = 'Store local copies of remote imagery in GeoBlacklight'
14
- s.license = 'Apache 2.0'
8
+ s.name = "geoblacklight_sidecar_images"
9
+ s.version = GeoblacklightSidecarImages::VERSION
10
+ s.authors = ["Eric Larson", "Eliot Jordan"]
11
+ s.email = ["ewlarson@gmail.com"]
12
+ s.homepage = "https://github.com/geoblacklight/geoblacklight_sidecar_images"
13
+ s.summary = "Store local copies of remote imagery in GeoBlacklight"
14
+ s.license = "Apache 2.0"
15
15
 
16
- s.files = `git ls-files -z`.split(%Q{\x0})
17
- s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
- s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
19
- s.require_paths = ['lib']
16
+ s.files = `git ls-files -z`.split(%(\x0))
17
+ s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
20
19
 
21
- s.add_dependency 'geoblacklight', '>= 3.3'
22
- s.add_dependency 'mini_magick', '~> 4.9.4'
23
- s.add_dependency 'image_processing', '~> 1.6'
24
- s.add_dependency 'statesman', '>= 3.4'
25
- s.add_dependency 'mimemagic', '~> 0.3'
26
- s.add_dependency 'rails', '>= 5.2', '< 6.2'
20
+ s.add_dependency "faraday", ">= 1.0"
21
+ s.add_dependency "geoblacklight", ">= 3.3"
22
+ s.add_dependency "image_processing", "~> 1.6"
23
+ s.add_dependency "mimemagic", "~> 0.3"
24
+ s.add_dependency "mini_magick", "~> 4.9.4"
25
+ s.add_dependency "rails", ">= 5.2", "< 7.1"
26
+ s.add_dependency "statesman", ">= 3.4"
27
27
 
28
- s.add_development_dependency 'byebug'
29
- s.add_development_dependency 'capybara'
30
- s.add_development_dependency 'webdrivers'
31
- s.add_development_dependency 'database_cleaner', '~> 1.3'
32
- s.add_development_dependency 'engine_cart', '~> 2.2'
33
- s.add_development_dependency 'rspec-rails', '~> 3.0'
34
- s.add_development_dependency 'rubocop', '~> 1.9'
35
- s.add_development_dependency 'rubocop-rspec', '~> 1.30.0'
36
- s.add_development_dependency 'rubocop-rails', '~> 2.9'
37
- s.add_development_dependency 'selenium-webdriver'
38
- s.add_development_dependency 'solr_wrapper', '~> 3.1'
39
- s.add_development_dependency 'sprockets', '< 4'
28
+ s.add_development_dependency "byebug"
29
+ s.add_development_dependency "capybara"
30
+ s.add_development_dependency "webdrivers"
31
+ s.add_development_dependency "database_cleaner", "~> 1.3"
32
+ s.add_development_dependency "engine_cart", "~> 2.2"
33
+ s.add_development_dependency "rspec-rails", "~> 3.0"
34
+ s.add_development_dependency "selenium-webdriver"
35
+ s.add_development_dependency "simplecov", "~> 0.22"
36
+ s.add_development_dependency "solr_wrapper", "~> 4.0"
37
+ s.add_development_dependency "sprockets", "< 4"
38
+ s.add_development_dependency "standard", "~> 1.24"
40
39
  end
@@ -1,18 +1,27 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
3
+ require "rails/generators"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  class ConfigGenerator < Rails::Generators::Base
7
- source_root File.expand_path('templates', __dir__)
7
+ source_root File.expand_path("templates", __dir__)
8
8
 
9
9
  desc <<-DESCRIPTION
10
10
  This generator makes the following changes to your application:
11
11
  1. Copies config files to host config
12
12
  DESCRIPTION
13
13
 
14
- def create_store_image_jobs
15
- copy_file 'config/initializers/statesman.rb', 'config/initializers/statesman.rb'
14
+ def set_active_storage_processor
15
+ app_config = <<-"APP"
16
+
17
+ config.active_storage.variant_processor = :mini_magick
18
+ APP
19
+
20
+ inject_into_file "config/application.rb", app_config, after: "config.generators.system_tests = nil"
21
+ end
22
+
23
+ def create_statesman_initializer
24
+ copy_file "config/initializers/statesman.rb", "config/initializers/statesman.rb"
16
25
  end
17
26
  end
18
27
  end
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
3
+ require "rails/generators"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  class ExampleDocsGenerator < Rails::Generators::Base
7
- source_root File.expand_path('../../../spec/fixtures', __dir__)
7
+ source_root File.expand_path("../../../spec/fixtures", __dir__)
8
8
 
9
9
  desc <<-DESCRIPTION
10
10
  This generator makes the following changes to your application:
@@ -12,7 +12,7 @@ module GeoblacklightSidecarImages
12
12
  DESCRIPTION
13
13
 
14
14
  def create_services
15
- directory 'files', 'solr/geoblacklight/example_docs'
15
+ directory "files", "solr/geoblacklight/example_docs"
16
16
  end
17
17
  end
18
18
  end
@@ -1,14 +1,14 @@
1
- require 'rails/generators'
1
+ require "rails/generators"
2
2
 
3
3
  module GeoblacklightSidecarImages
4
4
  class Install < Rails::Generators::Base
5
- source_root File.expand_path('templates', __dir__)
5
+ source_root File.expand_path("templates", __dir__)
6
6
 
7
- desc 'Install GeoblacklightSidecarImages'
7
+ desc "Install GeoblacklightSidecarImages"
8
8
 
9
9
  def add_settings_vars
10
- inject_into_file 'config/settings.yml', after: "INSTITUTION: 'Stanford'\n" do
11
- "INSTITUTION_LOCAL_NAME: 'Princeton'
10
+ inject_into_file "config/settings.yml", after: "INSTITUTION: 'Stanford'\n" do
11
+ "INSTITUTION_LOCAL_NAME: 'Princeton'
12
12
  INSTITUTION_GEOSERVER_URL: 'https://geoserver.princeton.edu'
13
13
  PROXY_GEOSERVER_URL: 'http://localhost:3000'
14
14
  PROXY_GEOSERVER_AUTH: 'Basic base64encodedusername:password'
@@ -17,36 +17,36 @@ GBLSI_THUMBNAIL_FIELD: 'thumbnail_path_ss'"
17
17
  end
18
18
 
19
19
  def generate_gblsci_assets
20
- copy_file 'gblsci.scss', 'app/assets/stylesheets/gblsci.scss'
20
+ copy_file "gblsci.scss", "app/assets/stylesheets/gblsci.scss"
21
21
  end
22
22
 
23
23
  def generate_gblsci_example_docs
24
- generate 'geoblacklight_sidecar_images:example_docs'
24
+ generate "geoblacklight_sidecar_images:example_docs"
25
25
  end
26
26
 
27
27
  def generate_gblsci_jobs
28
- generate 'geoblacklight_sidecar_images:jobs'
28
+ generate "geoblacklight_sidecar_images:jobs"
29
29
  end
30
30
 
31
31
  def generate_gblsci_models
32
- generate 'geoblacklight_sidecar_images:models'
32
+ generate "geoblacklight_sidecar_images:models"
33
33
  end
34
34
 
35
35
  def generate_gblsci_views
36
- generate 'geoblacklight_sidecar_images:views'
36
+ generate "geoblacklight_sidecar_images:views"
37
37
  end
38
38
 
39
39
  def generate_action_storage
40
- rake 'active_storage:install'
40
+ rake "active_storage:install"
41
41
  end
42
42
 
43
43
  def generate_gblsci_config
44
- generate 'geoblacklight_sidecar_images:config'
44
+ generate "geoblacklight_sidecar_images:config"
45
45
  end
46
46
 
47
47
  def bundle_install
48
- Bundler.with_clean_env do
49
- run 'bundle install'
48
+ Bundler.with_unbundled_env do
49
+ run "bundle install"
50
50
  end
51
51
  end
52
52
  end
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
3
+ require "rails/generators"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  class JobsGenerator < Rails::Generators::Base
7
- source_root File.expand_path('templates', __dir__)
7
+ source_root File.expand_path("templates", __dir__)
8
8
 
9
9
  desc <<-DESCRIPTION
10
10
  This generator makes the following changes to your application:
@@ -16,7 +16,7 @@ module GeoblacklightSidecarImages
16
16
  config.active_job.queue_adapter = :inline
17
17
  JOBS
18
18
 
19
- inject_into_file 'config/environments/development.rb', job_config, before: /^end/
19
+ inject_into_file "config/environments/development.rb", job_config, before: /^end/
20
20
  end
21
21
  end
22
22
  end
@@ -1,13 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
4
- require 'rails/generators/migration'
3
+ require "rails/generators"
4
+ require "rails/generators/migration"
5
5
 
6
6
  module GeoblacklightSidecarImages
7
7
  class ModelsGenerator < Rails::Generators::Base
8
8
  include Rails::Generators::Migration
9
9
 
10
- source_root File.expand_path('templates', __dir__)
10
+ source_root File.expand_path("templates", __dir__)
11
11
 
12
12
  desc <<-DESCRIPTION
13
13
  This generator makes the following changes to your application:
@@ -18,7 +18,7 @@ module GeoblacklightSidecarImages
18
18
 
19
19
  # Setup the database migrations
20
20
  def copy_migrations
21
- rake 'geoblacklight_sidecar_images:install:migrations'
21
+ rake "geoblacklight_sidecar_images:install:migrations"
22
22
  end
23
23
 
24
24
  def include_sidecar_solrdocument
@@ -39,13 +39,13 @@ module GeoblacklightSidecarImages
39
39
  end
40
40
  SIDECAR
41
41
 
42
- inject_into_file 'app/models/solr_document.rb', sidecar, before: /^end/
42
+ inject_into_file "app/models/solr_document.rb", sidecar, before: /^end/
43
43
  end
44
44
 
45
45
  def include_wms_rewrite_solrdocument
46
46
  inject_into_file(
47
- 'app/models/solr_document.rb',
48
- after: 'include Geoblacklight::SolrDocument'
47
+ "app/models/solr_document.rb",
48
+ after: "include Geoblacklight::SolrDocument"
49
49
  ) do
50
50
  "\n include WmsRewriteConcern"
51
51
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'statesman'
3
+ require "statesman"
4
4
 
5
5
  Statesman.configure do
6
6
  storage_adapter(Statesman::Adapters::ActiveRecord)
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails/generators'
3
+ require "rails/generators"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  class ViewsGenerator < Rails::Generators::Base
7
- source_root File.expand_path('templates', __dir__)
7
+ source_root File.expand_path("templates", __dir__)
8
8
 
9
9
  desc <<-DESCRIPTION
10
10
  This generator makes the following changes to your application:
@@ -13,8 +13,8 @@ module GeoblacklightSidecarImages
13
13
 
14
14
  def create_views
15
15
  copy_file(
16
- 'views/catalog/_index_split_default.html.erb',
17
- 'app/views/catalog/_index_split_default.html.erb'
16
+ "views/catalog/_index_split_default.html.erb",
17
+ "app/views/catalog/_index_split_default.html.erb"
18
18
  )
19
19
  end
20
20
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module GeoblacklightSidecarImages
4
- VERSION = '0.8.1'
4
+ VERSION = "0.9.0"
5
5
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'geoblacklight_sidecar_images/engine'
3
+ require "geoblacklight_sidecar_images/engine"
4
4
 
5
5
  module GeoblacklightSidecarImages
6
6
  # Your code goes here...
@@ -1,10 +1,10 @@
1
- require 'csv'
1
+ require "csv"
2
2
 
3
3
  namespace :gblsci do
4
4
  namespace :sample_data do
5
- desc 'Ingests a directory of geoblacklight.json files'
5
+ desc "Ingests a directory of geoblacklight.json files"
6
6
  task seed: :environment do
7
- Dir.glob(File.join(Rails.root, 'solr', 'geoblacklight', 'example_docs', '**', '*.json')).each do |fn|
7
+ Dir.glob(File.join(Rails.root, "solr", "geoblacklight", "example_docs", "**", "*.json")).each do |fn|
8
8
  puts "Ingesting #{fn}"
9
9
  begin
10
10
  Blacklight.default_index.connection.add(JSON.parse(File.read(fn)))
@@ -12,40 +12,38 @@ namespace :gblsci do
12
12
  puts "Failed to ingest #{fn}: #{e.inspect}"
13
13
  end
14
14
  end
15
- puts 'Committing changes to Solr'
15
+ puts "Committing changes to Solr"
16
16
  Blacklight.default_index.connection.commit
17
17
  end
18
18
  end
19
19
 
20
20
  namespace :images do
21
- desc 'Harvest image for specific document'
22
- task :harvest_doc_id, [:doc_id] => [:environment] do |_t, args|
23
- GeoblacklightSidecarImages::StoreImageJob.perform_later(args[:doc_id])
21
+ desc "Harvest image for specific document"
22
+ task harvest_doc_id: :environment do
23
+ GeoblacklightSidecarImages::StoreImageJob.perform_later(ENV["DOC_ID"])
24
24
  end
25
25
 
26
- desc 'Harvest all images'
26
+ desc "Harvest all images"
27
27
  task harvest_all: :environment do
28
- begin
29
- query = '*:*'
30
- index = Geoblacklight::SolrDocument.index
31
- results = index.send_and_receive(index.blacklight_config.solr_path,
32
- q: query,
33
- fl: "*",
34
- rows: 100_000_000)
35
- num_found = results.response[:numFound]
36
- doc_counter = 0
37
- results.docs.each do |document|
38
- sleep(1)
39
- begin
40
- GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
41
- rescue Blacklight::Exceptions::RecordNotFound
42
- next
43
- end
28
+ query = "*:*"
29
+ index = Geoblacklight::SolrDocument.index
30
+ results = index.send_and_receive(index.blacklight_config.solr_path,
31
+ q: query,
32
+ fl: "*",
33
+ rows: 100_000_000)
34
+ # num_found = results.response[:numFound]
35
+ # doc_counter = 0
36
+ results.docs.each do |document|
37
+ sleep(1)
38
+ begin
39
+ GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
40
+ rescue Blacklight::Exceptions::RecordNotFound
41
+ next
44
42
  end
45
43
  end
46
44
  end
47
45
 
48
- desc 'Hash of SolrDocumentSidecar image state counts'
46
+ desc "Hash of SolrDocumentSidecar image state counts"
49
47
  task harvest_states: :environment do
50
48
  states = [
51
49
  :initialized,
@@ -62,12 +60,12 @@ namespace :gblsci do
62
60
  col_state[state] = sidecars.size
63
61
  end
64
62
 
65
- col_state.each do |col,state|
63
+ col_state.each do |col, state|
66
64
  puts "#{col} - #{state}"
67
65
  end
68
66
  end
69
67
 
70
- desc 'Re-queues incomplete states for harvesting'
68
+ desc "Re-queues incomplete states for harvesting"
71
69
  task harvest_retry: :environment do
72
70
  states = [
73
71
  :initialized,
@@ -83,24 +81,22 @@ namespace :gblsci do
83
81
  puts "#{state} - #{sidecars.size}"
84
82
 
85
83
  sidecars.each do |sc|
86
- begin
87
- document = Geoblacklight::SolrDocument.find(sc.document_id)
88
- GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
89
- rescue
90
- puts "orphaned / #{sc.document_id}"
91
- end
84
+ document = Geoblacklight::SolrDocument.find(sc.document_id)
85
+ GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
86
+ rescue
87
+ puts "orphaned / #{sc.document_id}"
92
88
  end
93
89
  end
94
90
  end
95
91
 
96
- desc 'Write harvest state report (CSV)'
92
+ desc "Write harvest state report (CSV)"
97
93
  task harvest_report: :environment do
98
94
  # Create a CSV Dump of Results
99
- file = "#{Rails.root}/public/#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}.sidecar_report.csv"
95
+ file = "#{Rails.root}/public/#{Time.now.strftime("%Y-%m-%d_%H-%M-%S")}.sidecar_report.csv"
100
96
 
101
97
  sidecars = SolrDocumentSidecar.all
102
98
 
103
- CSV.open(file, 'w') do |writer|
99
+ CSV.open(file, "w") do |writer|
104
100
  header = [
105
101
  "Sidecar ID",
106
102
  "Document ID",
@@ -117,31 +113,30 @@ namespace :gblsci do
117
113
  writer << header
118
114
 
119
115
  sidecars.each do |sc|
120
- cat = CatalogController.new
121
- begin
122
- document = Geoblacklight::SolrDocument.find(sc.document_id)
123
- writer << [
124
- sc.id,
125
- sc.document_id,
126
- sc.image_state.current_state,
127
- document._source['layer_geom_type_s'],
128
- document._source['dc_title_s'],
129
- document._source['dct_provenance_s'],
130
- sc.image_state.last_transition.metadata['exception'],
131
- sc.image_state.last_transition.metadata['viewer_protocol'],
132
- sc.image_state.last_transition.metadata['image_url'],
133
- sc.image_state.last_transition.metadata['gblsi_thumbnail_uri']
134
- ]
135
- rescue Exception => e
136
- puts "Exception: #{e.inspect}"
137
- puts "orphaned / #{sc.document_id}"
138
- next
139
- end
116
+ # cat = CatalogController.new
117
+
118
+ document = Geoblacklight::SolrDocument.find(sc.document_id)
119
+ writer << [
120
+ sc.id,
121
+ sc.document_id,
122
+ sc.image_state.current_state,
123
+ document._source["layer_geom_type_s"],
124
+ document._source["dc_title_s"],
125
+ document._source["dct_provenance_s"],
126
+ sc.image_state.last_transition.metadata["exception"],
127
+ sc.image_state.last_transition.metadata["viewer_protocol"],
128
+ sc.image_state.last_transition.metadata["image_url"],
129
+ sc.image_state.last_transition.metadata["gblsi_thumbnail_uri"]
130
+ ]
131
+ rescue => e
132
+ puts "Exception: #{e.inspect}"
133
+ puts "orphaned / #{sc.document_id}"
134
+ next
140
135
  end
141
136
  end
142
137
  end
143
138
 
144
- desc 'Destroy all harvested images and sidecar AR objects'
139
+ desc "Destroy all harvested images and sidecar AR objects"
145
140
  task harvest_purge_all: :environment do
146
141
  # Remove all images
147
142
  sidecars = SolrDocumentSidecar.all
@@ -154,43 +149,41 @@ namespace :gblsci do
154
149
  SolrDocumentSidecar.destroy_all
155
150
  end
156
151
 
157
- desc 'Destroy orphaned images and sidecar AR objects'
152
+ desc "Destroy orphaned images and sidecar AR objects"
158
153
  # When a SolrDocumentSidecar AR object exists,
159
154
  # but it's corresponding SolrDocument is no longer in the Solr index.
160
155
  task harvest_purge_orphans: :environment do
161
156
  # Remove all images
162
157
  sidecars = SolrDocumentSidecar.all
163
158
  sidecars.each do |sc|
164
- begin
165
- document = Geoblacklight::SolrDocument.find(sc.document_id)
166
- rescue
167
- sc.destroy
168
- puts "orphaned / #{sc.document_id} / destroyed"
169
- end
159
+ Geoblacklight::SolrDocument.find(sc.document_id)
160
+ rescue
161
+ sc.destroy
162
+ puts "orphaned / #{sc.document_id} / destroyed"
170
163
  end
171
164
  end
172
165
 
173
- desc 'Destroy select sidecar AR objects by CSV file'
166
+ desc "Destroy select sidecar AR objects by CSV file"
174
167
  task harvest_destroy_batch: :environment do
175
168
  # Expects a CSV file in Rails.root/tmp/destroy_batch.csv
176
169
  #
177
170
  # From your local machine, copy it up to production server like this:
178
171
  # scp destroy_batch.csv swadm@geoprod:/swadm/var/www/geoblacklight/current/tmp/
179
172
  CSV.foreach("#{Rails.root}/tmp/destroy_batch.csv", headers: true) do |row|
180
- sc = SolrDocumentSidecar.find_by(:document_id => row[0])
173
+ sc = SolrDocumentSidecar.find_by(document_id: row[0])
181
174
  sc.destroy
182
175
  puts "document_id - #{row[0]} - destroyed"
183
176
  end
184
177
  end
185
178
 
186
- desc 'Inspect failed state objects'
179
+ desc "Inspect failed state objects"
187
180
  task harvest_failed_state_inspect: :environment do
188
181
  states = [
189
182
  :failed
190
183
  ]
191
184
 
192
185
  states.each do |state|
193
- sidecars = SolrDocumentSidecar.in_state(state).each do |sc|
186
+ SolrDocumentSidecar.in_state(state).each do |sc|
194
187
  puts "#{state} - #{sc.document_id} - #{sc.image_state.last_transition.metadata.inspect}"
195
188
  end
196
189
  end
data/solr/conf/schema.xml CHANGED
@@ -143,13 +143,11 @@
143
143
  <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_d"/>
144
144
 
145
145
  <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
146
- geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers"/>
146
+ geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
147
147
  <!-- Adding field type for bboxField that enables, among other things, overlap ratio calculations -->
148
148
  <fieldType name="bbox" class="solr.BBoxField"
149
149
  geo="true" distanceUnits="kilometers" numberType="pdouble" />
150
150
  <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
151
-
152
-
153
151
  </types>
154
152
 
155
153
  <!-- for scoring formula -->
@@ -169,8 +167,8 @@
169
167
  <copyField source="layer_slug_s" dest="layer_slug_ti" maxChars="100"/>
170
168
 
171
169
  <!-- core text search -->
172
- <copyField source="*_ti" dest="text" />
173
- <copyField source="*_tmi" dest="text" />
170
+ <copyField source="*_s" dest="text" />
171
+ <copyField source="*_sm" dest="text" />
174
172
 
175
173
  <!-- for sorting text fields -->
176
174
  <copyField source="dct_provenance_s" dest="dct_provenance_sort"/>
@@ -195,4 +193,4 @@
195
193
 
196
194
  <!-- for bbox value -->
197
195
  <copyField source="solr_geom" dest="solr_bboxtype"/>
198
- </schema>
196
+ </schema>
@@ -14,7 +14,7 @@
14
14
  "layer_slug_s":"aaf63e35-3d5f-4e62-b59d-377f008e9aad",
15
15
  "dc_format_s":"TIFF",
16
16
  "dc_creator_sm":["Minnesota. Department of Highways."],
17
- "thumbnail_path_ss":"https://umedia.lib.umn.edu/sites/default/files/imagecache/square300/reference/562/image/jpeg/1089695.jpg",
17
+ "thumbnail_path_ss":"https://cdm16022.contentdm.oclc.org/utils/getthumbnail/collection/p16022coll206/id/133.jpg",
18
18
  "dc_type_s":"Still image",
19
19
  "dc_identifier_s":"aaf63e35-3d5f-4e62-b59d-377f008e9aad",
20
20
  "dc_relation_sm":["http://sws.geonames.org/5037779/about/rdf"],
@@ -1,14 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rails_helper'
3
+ require "rails_helper"
4
4
 
5
5
  describe GeoblacklightSidecarImages::StoreImageJob, type: :job do
6
6
  let(:document) { SolrDocument.new(document_attributes) }
7
7
 
8
- describe '#perform_later' do
9
- let(:document_attributes) { json_data('umn_iiif_jpg') }
8
+ describe "#perform_later" do
9
+ let(:document_attributes) { json_data("umn_iiif_jpg") }
10
10
 
11
- it 'stores an image' do
11
+ it "stores an image" do
12
12
  ActiveJob::Base.queue_adapter = :test
13
13
  expect { GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id) }.to have_enqueued_job(GeoblacklightSidecarImages::StoreImageJob)
14
14
  end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails_helper"
4
+ require "spec_helper"
5
+ require "rake"
6
+
7
+ describe "geoblacklight_sidecar_images_tasks.rake" do
8
+ include ActiveJob::TestHelper
9
+
10
+ # @TODO
11
+ # gblsci:sample_data:seed
12
+ # gblsci:images:harvest_all
13
+ # gblsci:images:harvest_states
14
+ # gblsci:images:harvest_retry
15
+ # gblsci:images:harvest_report
16
+ # gblsci:images:harvest_purge_all
17
+ # gblsci:images:harvest_purge_orphans
18
+ # gblsci:images:harvest_destroy_batch
19
+ # gblsci:images:harvest_failed_state_inspect
20
+
21
+ # DOC_ID='stanford-cz128vq0535' bundle exec rake gblsci:images:harvest_doc_id
22
+ describe "gblsci:images:harvest_doc_id" do
23
+ before do
24
+ Rails.application.load_tasks
25
+ end
26
+
27
+ it "enqueues background job to harvest image" do
28
+ ENV["DOC_ID"] = "princeton-02870w62c"
29
+ Rake::Task["gblsci:images:harvest_doc_id"].invoke
30
+ expect(enqueued_jobs.size).to eq(1)
31
+ end
32
+ end
33
+ end