geoblacklight_sidecar_images 0.8.1 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +69 -0
- data/.gitignore +1 -0
- data/.solr_wrapper.yml +2 -0
- data/.standard.yml +1 -0
- data/Gemfile +22 -13
- data/README.md +5 -7
- data/Rakefile +52 -16
- data/app/jobs/geoblacklight_sidecar_images/store_image_job.rb +2 -2
- data/app/mailers/geoblacklight_sidecar_images/application_mailer.rb +2 -2
- data/app/models/concerns/wms_rewrite_concern.rb +1 -1
- data/app/models/sidecar_image_state_machine.rb +6 -6
- data/app/models/sidecar_image_transition.rb +1 -1
- data/app/services/geoblacklight_sidecar_images/image_service/iiif.rb +1 -1
- data/app/services/geoblacklight_sidecar_images/image_service/wms.rb +4 -4
- data/app/services/geoblacklight_sidecar_images/image_service.rb +46 -46
- data/db/migrate/20180118203155_create_solr_document_sidecars.rb +1 -1
- data/db/migrate/20180118203519_create_sidecar_image_transitions.rb +6 -6
- data/geoblacklight_sidecar_images.gemspec +30 -31
- data/lib/generators/geoblacklight_sidecar_images/config_generator.rb +13 -4
- data/lib/generators/geoblacklight_sidecar_images/example_docs_generator.rb +3 -3
- data/lib/generators/geoblacklight_sidecar_images/install_generator.rb +14 -14
- data/lib/generators/geoblacklight_sidecar_images/jobs_generator.rb +3 -3
- data/lib/generators/geoblacklight_sidecar_images/models_generator.rb +7 -7
- data/lib/generators/geoblacklight_sidecar_images/templates/config/initializers/statesman.rb +1 -1
- data/lib/generators/geoblacklight_sidecar_images/views_generator.rb +4 -4
- data/lib/geoblacklight_sidecar_images/version.rb +1 -1
- data/lib/geoblacklight_sidecar_images.rb +1 -1
- data/lib/tasks/geoblacklight_sidecar_images_tasks.rake +61 -68
- data/solr/conf/schema.xml +4 -6
- data/spec/fixtures/files/umn_solr_thumb.json +1 -1
- data/spec/jobs/store_image_job_spec.rb +4 -4
- data/spec/lib/tasks/geoblacklight_sidecar_images_spec.rb +33 -0
- data/spec/models/solr_document_sidecar_spec.rb +4 -4
- data/spec/models/solr_document_spec.rb +5 -5
- data/spec/rails_helper.rb +10 -12
- data/spec/services/image_service_spec.rb +24 -18
- data/spec/spec_helper.rb +32 -0
- data/spec/test_app_templates/Gemfile.extra +1 -0
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +12 -12
- data/template.rb +9 -9
- metadata +59 -106
- data/.coveralls.yml +0 -1
- data/.rubocop.yml +0 -81
- data/.travis.yml +0 -29
@@ -1,40 +1,39 @@
|
|
1
|
-
$LOAD_PATH.push File.expand_path(
|
1
|
+
$LOAD_PATH.push File.expand_path("../lib", __FILE__)
|
2
2
|
|
3
3
|
# Maintain your gem's version:
|
4
|
-
require
|
4
|
+
require "geoblacklight_sidecar_images/version"
|
5
5
|
|
6
6
|
# Describe your gem and declare its dependencies:
|
7
7
|
Gem::Specification.new do |s|
|
8
|
-
s.name
|
9
|
-
s.version
|
10
|
-
s.authors
|
11
|
-
s.email
|
12
|
-
s.homepage
|
13
|
-
s.summary
|
14
|
-
s.license
|
8
|
+
s.name = "geoblacklight_sidecar_images"
|
9
|
+
s.version = GeoblacklightSidecarImages::VERSION
|
10
|
+
s.authors = ["Eric Larson", "Eliot Jordan"]
|
11
|
+
s.email = ["ewlarson@gmail.com"]
|
12
|
+
s.homepage = "https://github.com/geoblacklight/geoblacklight_sidecar_images"
|
13
|
+
s.summary = "Store local copies of remote imagery in GeoBlacklight"
|
14
|
+
s.license = "Apache 2.0"
|
15
15
|
|
16
|
-
s.files
|
17
|
-
s.
|
18
|
-
s.
|
19
|
-
s.require_paths = ['lib']
|
16
|
+
s.files = `git ls-files -z`.split(%(\x0))
|
17
|
+
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
20
19
|
|
21
|
-
s.add_dependency
|
22
|
-
s.add_dependency
|
23
|
-
s.add_dependency
|
24
|
-
s.add_dependency
|
25
|
-
s.add_dependency
|
26
|
-
s.add_dependency
|
20
|
+
s.add_dependency "faraday", ">= 1.0"
|
21
|
+
s.add_dependency "geoblacklight", ">= 3.3"
|
22
|
+
s.add_dependency "image_processing", "~> 1.6"
|
23
|
+
s.add_dependency "mimemagic", "~> 0.3"
|
24
|
+
s.add_dependency "mini_magick", "~> 4.9.4"
|
25
|
+
s.add_dependency "rails", ">= 5.2", "< 7.1"
|
26
|
+
s.add_dependency "statesman", ">= 3.4"
|
27
27
|
|
28
|
-
s.add_development_dependency
|
29
|
-
s.add_development_dependency
|
30
|
-
s.add_development_dependency
|
31
|
-
s.add_development_dependency
|
32
|
-
s.add_development_dependency
|
33
|
-
s.add_development_dependency
|
34
|
-
s.add_development_dependency
|
35
|
-
s.add_development_dependency
|
36
|
-
s.add_development_dependency
|
37
|
-
s.add_development_dependency
|
38
|
-
s.add_development_dependency
|
39
|
-
s.add_development_dependency 'sprockets', '< 4'
|
28
|
+
s.add_development_dependency "byebug"
|
29
|
+
s.add_development_dependency "capybara"
|
30
|
+
s.add_development_dependency "webdrivers"
|
31
|
+
s.add_development_dependency "database_cleaner", "~> 1.3"
|
32
|
+
s.add_development_dependency "engine_cart", "~> 2.2"
|
33
|
+
s.add_development_dependency "rspec-rails", "~> 3.0"
|
34
|
+
s.add_development_dependency "selenium-webdriver"
|
35
|
+
s.add_development_dependency "simplecov", "~> 0.22"
|
36
|
+
s.add_development_dependency "solr_wrapper", "~> 4.0"
|
37
|
+
s.add_development_dependency "sprockets", "< 4"
|
38
|
+
s.add_development_dependency "standard", "~> 1.24"
|
40
39
|
end
|
@@ -1,18 +1,27 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails/generators"
|
4
4
|
|
5
5
|
module GeoblacklightSidecarImages
|
6
6
|
class ConfigGenerator < Rails::Generators::Base
|
7
|
-
source_root File.expand_path(
|
7
|
+
source_root File.expand_path("templates", __dir__)
|
8
8
|
|
9
9
|
desc <<-DESCRIPTION
|
10
10
|
This generator makes the following changes to your application:
|
11
11
|
1. Copies config files to host config
|
12
12
|
DESCRIPTION
|
13
13
|
|
14
|
-
def
|
15
|
-
|
14
|
+
def set_active_storage_processor
|
15
|
+
app_config = <<-"APP"
|
16
|
+
|
17
|
+
config.active_storage.variant_processor = :mini_magick
|
18
|
+
APP
|
19
|
+
|
20
|
+
inject_into_file "config/application.rb", app_config, after: "config.generators.system_tests = nil"
|
21
|
+
end
|
22
|
+
|
23
|
+
def create_statesman_initializer
|
24
|
+
copy_file "config/initializers/statesman.rb", "config/initializers/statesman.rb"
|
16
25
|
end
|
17
26
|
end
|
18
27
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails/generators"
|
4
4
|
|
5
5
|
module GeoblacklightSidecarImages
|
6
6
|
class ExampleDocsGenerator < Rails::Generators::Base
|
7
|
-
source_root File.expand_path(
|
7
|
+
source_root File.expand_path("../../../spec/fixtures", __dir__)
|
8
8
|
|
9
9
|
desc <<-DESCRIPTION
|
10
10
|
This generator makes the following changes to your application:
|
@@ -12,7 +12,7 @@ module GeoblacklightSidecarImages
|
|
12
12
|
DESCRIPTION
|
13
13
|
|
14
14
|
def create_services
|
15
|
-
directory
|
15
|
+
directory "files", "solr/geoblacklight/example_docs"
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
@@ -1,14 +1,14 @@
|
|
1
|
-
require
|
1
|
+
require "rails/generators"
|
2
2
|
|
3
3
|
module GeoblacklightSidecarImages
|
4
4
|
class Install < Rails::Generators::Base
|
5
|
-
source_root File.expand_path(
|
5
|
+
source_root File.expand_path("templates", __dir__)
|
6
6
|
|
7
|
-
desc
|
7
|
+
desc "Install GeoblacklightSidecarImages"
|
8
8
|
|
9
9
|
def add_settings_vars
|
10
|
-
inject_into_file
|
11
|
-
"INSTITUTION_LOCAL_NAME: 'Princeton'
|
10
|
+
inject_into_file "config/settings.yml", after: "INSTITUTION: 'Stanford'\n" do
|
11
|
+
"INSTITUTION_LOCAL_NAME: 'Princeton'
|
12
12
|
INSTITUTION_GEOSERVER_URL: 'https://geoserver.princeton.edu'
|
13
13
|
PROXY_GEOSERVER_URL: 'http://localhost:3000'
|
14
14
|
PROXY_GEOSERVER_AUTH: 'Basic base64encodedusername:password'
|
@@ -17,36 +17,36 @@ GBLSI_THUMBNAIL_FIELD: 'thumbnail_path_ss'"
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def generate_gblsci_assets
|
20
|
-
copy_file
|
20
|
+
copy_file "gblsci.scss", "app/assets/stylesheets/gblsci.scss"
|
21
21
|
end
|
22
22
|
|
23
23
|
def generate_gblsci_example_docs
|
24
|
-
generate
|
24
|
+
generate "geoblacklight_sidecar_images:example_docs"
|
25
25
|
end
|
26
26
|
|
27
27
|
def generate_gblsci_jobs
|
28
|
-
generate
|
28
|
+
generate "geoblacklight_sidecar_images:jobs"
|
29
29
|
end
|
30
30
|
|
31
31
|
def generate_gblsci_models
|
32
|
-
generate
|
32
|
+
generate "geoblacklight_sidecar_images:models"
|
33
33
|
end
|
34
34
|
|
35
35
|
def generate_gblsci_views
|
36
|
-
generate
|
36
|
+
generate "geoblacklight_sidecar_images:views"
|
37
37
|
end
|
38
38
|
|
39
39
|
def generate_action_storage
|
40
|
-
rake
|
40
|
+
rake "active_storage:install"
|
41
41
|
end
|
42
42
|
|
43
43
|
def generate_gblsci_config
|
44
|
-
generate
|
44
|
+
generate "geoblacklight_sidecar_images:config"
|
45
45
|
end
|
46
46
|
|
47
47
|
def bundle_install
|
48
|
-
Bundler.
|
49
|
-
run
|
48
|
+
Bundler.with_unbundled_env do
|
49
|
+
run "bundle install"
|
50
50
|
end
|
51
51
|
end
|
52
52
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails/generators"
|
4
4
|
|
5
5
|
module GeoblacklightSidecarImages
|
6
6
|
class JobsGenerator < Rails::Generators::Base
|
7
|
-
source_root File.expand_path(
|
7
|
+
source_root File.expand_path("templates", __dir__)
|
8
8
|
|
9
9
|
desc <<-DESCRIPTION
|
10
10
|
This generator makes the following changes to your application:
|
@@ -16,7 +16,7 @@ module GeoblacklightSidecarImages
|
|
16
16
|
config.active_job.queue_adapter = :inline
|
17
17
|
JOBS
|
18
18
|
|
19
|
-
inject_into_file
|
19
|
+
inject_into_file "config/environments/development.rb", job_config, before: /^end/
|
20
20
|
end
|
21
21
|
end
|
22
22
|
end
|
@@ -1,13 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "rails/generators"
|
4
|
+
require "rails/generators/migration"
|
5
5
|
|
6
6
|
module GeoblacklightSidecarImages
|
7
7
|
class ModelsGenerator < Rails::Generators::Base
|
8
8
|
include Rails::Generators::Migration
|
9
9
|
|
10
|
-
source_root File.expand_path(
|
10
|
+
source_root File.expand_path("templates", __dir__)
|
11
11
|
|
12
12
|
desc <<-DESCRIPTION
|
13
13
|
This generator makes the following changes to your application:
|
@@ -18,7 +18,7 @@ module GeoblacklightSidecarImages
|
|
18
18
|
|
19
19
|
# Setup the database migrations
|
20
20
|
def copy_migrations
|
21
|
-
rake
|
21
|
+
rake "geoblacklight_sidecar_images:install:migrations"
|
22
22
|
end
|
23
23
|
|
24
24
|
def include_sidecar_solrdocument
|
@@ -39,13 +39,13 @@ module GeoblacklightSidecarImages
|
|
39
39
|
end
|
40
40
|
SIDECAR
|
41
41
|
|
42
|
-
inject_into_file
|
42
|
+
inject_into_file "app/models/solr_document.rb", sidecar, before: /^end/
|
43
43
|
end
|
44
44
|
|
45
45
|
def include_wms_rewrite_solrdocument
|
46
46
|
inject_into_file(
|
47
|
-
|
48
|
-
after:
|
47
|
+
"app/models/solr_document.rb",
|
48
|
+
after: "include Geoblacklight::SolrDocument"
|
49
49
|
) do
|
50
50
|
"\n include WmsRewriteConcern"
|
51
51
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails/generators"
|
4
4
|
|
5
5
|
module GeoblacklightSidecarImages
|
6
6
|
class ViewsGenerator < Rails::Generators::Base
|
7
|
-
source_root File.expand_path(
|
7
|
+
source_root File.expand_path("templates", __dir__)
|
8
8
|
|
9
9
|
desc <<-DESCRIPTION
|
10
10
|
This generator makes the following changes to your application:
|
@@ -13,8 +13,8 @@ module GeoblacklightSidecarImages
|
|
13
13
|
|
14
14
|
def create_views
|
15
15
|
copy_file(
|
16
|
-
|
17
|
-
|
16
|
+
"views/catalog/_index_split_default.html.erb",
|
17
|
+
"app/views/catalog/_index_split_default.html.erb"
|
18
18
|
)
|
19
19
|
end
|
20
20
|
end
|
@@ -1,10 +1,10 @@
|
|
1
|
-
require
|
1
|
+
require "csv"
|
2
2
|
|
3
3
|
namespace :gblsci do
|
4
4
|
namespace :sample_data do
|
5
|
-
desc
|
5
|
+
desc "Ingests a directory of geoblacklight.json files"
|
6
6
|
task seed: :environment do
|
7
|
-
Dir.glob(File.join(Rails.root,
|
7
|
+
Dir.glob(File.join(Rails.root, "solr", "geoblacklight", "example_docs", "**", "*.json")).each do |fn|
|
8
8
|
puts "Ingesting #{fn}"
|
9
9
|
begin
|
10
10
|
Blacklight.default_index.connection.add(JSON.parse(File.read(fn)))
|
@@ -12,40 +12,38 @@ namespace :gblsci do
|
|
12
12
|
puts "Failed to ingest #{fn}: #{e.inspect}"
|
13
13
|
end
|
14
14
|
end
|
15
|
-
puts
|
15
|
+
puts "Committing changes to Solr"
|
16
16
|
Blacklight.default_index.connection.commit
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
20
|
namespace :images do
|
21
|
-
desc
|
22
|
-
task
|
23
|
-
GeoblacklightSidecarImages::StoreImageJob.perform_later(
|
21
|
+
desc "Harvest image for specific document"
|
22
|
+
task harvest_doc_id: :environment do
|
23
|
+
GeoblacklightSidecarImages::StoreImageJob.perform_later(ENV["DOC_ID"])
|
24
24
|
end
|
25
25
|
|
26
|
-
desc
|
26
|
+
desc "Harvest all images"
|
27
27
|
task harvest_all: :environment do
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
next
|
43
|
-
end
|
28
|
+
query = "*:*"
|
29
|
+
index = Geoblacklight::SolrDocument.index
|
30
|
+
results = index.send_and_receive(index.blacklight_config.solr_path,
|
31
|
+
q: query,
|
32
|
+
fl: "*",
|
33
|
+
rows: 100_000_000)
|
34
|
+
# num_found = results.response[:numFound]
|
35
|
+
# doc_counter = 0
|
36
|
+
results.docs.each do |document|
|
37
|
+
sleep(1)
|
38
|
+
begin
|
39
|
+
GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
|
40
|
+
rescue Blacklight::Exceptions::RecordNotFound
|
41
|
+
next
|
44
42
|
end
|
45
43
|
end
|
46
44
|
end
|
47
45
|
|
48
|
-
desc
|
46
|
+
desc "Hash of SolrDocumentSidecar image state counts"
|
49
47
|
task harvest_states: :environment do
|
50
48
|
states = [
|
51
49
|
:initialized,
|
@@ -62,12 +60,12 @@ namespace :gblsci do
|
|
62
60
|
col_state[state] = sidecars.size
|
63
61
|
end
|
64
62
|
|
65
|
-
col_state.each do |col,state|
|
63
|
+
col_state.each do |col, state|
|
66
64
|
puts "#{col} - #{state}"
|
67
65
|
end
|
68
66
|
end
|
69
67
|
|
70
|
-
desc
|
68
|
+
desc "Re-queues incomplete states for harvesting"
|
71
69
|
task harvest_retry: :environment do
|
72
70
|
states = [
|
73
71
|
:initialized,
|
@@ -83,24 +81,22 @@ namespace :gblsci do
|
|
83
81
|
puts "#{state} - #{sidecars.size}"
|
84
82
|
|
85
83
|
sidecars.each do |sc|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
puts "orphaned / #{sc.document_id}"
|
91
|
-
end
|
84
|
+
document = Geoblacklight::SolrDocument.find(sc.document_id)
|
85
|
+
GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id)
|
86
|
+
rescue
|
87
|
+
puts "orphaned / #{sc.document_id}"
|
92
88
|
end
|
93
89
|
end
|
94
90
|
end
|
95
91
|
|
96
|
-
desc
|
92
|
+
desc "Write harvest state report (CSV)"
|
97
93
|
task harvest_report: :environment do
|
98
94
|
# Create a CSV Dump of Results
|
99
|
-
file = "#{Rails.root}/public/#{Time.now.strftime(
|
95
|
+
file = "#{Rails.root}/public/#{Time.now.strftime("%Y-%m-%d_%H-%M-%S")}.sidecar_report.csv"
|
100
96
|
|
101
97
|
sidecars = SolrDocumentSidecar.all
|
102
98
|
|
103
|
-
CSV.open(file,
|
99
|
+
CSV.open(file, "w") do |writer|
|
104
100
|
header = [
|
105
101
|
"Sidecar ID",
|
106
102
|
"Document ID",
|
@@ -117,31 +113,30 @@ namespace :gblsci do
|
|
117
113
|
writer << header
|
118
114
|
|
119
115
|
sidecars.each do |sc|
|
120
|
-
cat = CatalogController.new
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
end
|
116
|
+
# cat = CatalogController.new
|
117
|
+
|
118
|
+
document = Geoblacklight::SolrDocument.find(sc.document_id)
|
119
|
+
writer << [
|
120
|
+
sc.id,
|
121
|
+
sc.document_id,
|
122
|
+
sc.image_state.current_state,
|
123
|
+
document._source["layer_geom_type_s"],
|
124
|
+
document._source["dc_title_s"],
|
125
|
+
document._source["dct_provenance_s"],
|
126
|
+
sc.image_state.last_transition.metadata["exception"],
|
127
|
+
sc.image_state.last_transition.metadata["viewer_protocol"],
|
128
|
+
sc.image_state.last_transition.metadata["image_url"],
|
129
|
+
sc.image_state.last_transition.metadata["gblsi_thumbnail_uri"]
|
130
|
+
]
|
131
|
+
rescue => e
|
132
|
+
puts "Exception: #{e.inspect}"
|
133
|
+
puts "orphaned / #{sc.document_id}"
|
134
|
+
next
|
140
135
|
end
|
141
136
|
end
|
142
137
|
end
|
143
138
|
|
144
|
-
desc
|
139
|
+
desc "Destroy all harvested images and sidecar AR objects"
|
145
140
|
task harvest_purge_all: :environment do
|
146
141
|
# Remove all images
|
147
142
|
sidecars = SolrDocumentSidecar.all
|
@@ -154,43 +149,41 @@ namespace :gblsci do
|
|
154
149
|
SolrDocumentSidecar.destroy_all
|
155
150
|
end
|
156
151
|
|
157
|
-
desc
|
152
|
+
desc "Destroy orphaned images and sidecar AR objects"
|
158
153
|
# When a SolrDocumentSidecar AR object exists,
|
159
154
|
# but it's corresponding SolrDocument is no longer in the Solr index.
|
160
155
|
task harvest_purge_orphans: :environment do
|
161
156
|
# Remove all images
|
162
157
|
sidecars = SolrDocumentSidecar.all
|
163
158
|
sidecars.each do |sc|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
puts "orphaned / #{sc.document_id} / destroyed"
|
169
|
-
end
|
159
|
+
Geoblacklight::SolrDocument.find(sc.document_id)
|
160
|
+
rescue
|
161
|
+
sc.destroy
|
162
|
+
puts "orphaned / #{sc.document_id} / destroyed"
|
170
163
|
end
|
171
164
|
end
|
172
165
|
|
173
|
-
desc
|
166
|
+
desc "Destroy select sidecar AR objects by CSV file"
|
174
167
|
task harvest_destroy_batch: :environment do
|
175
168
|
# Expects a CSV file in Rails.root/tmp/destroy_batch.csv
|
176
169
|
#
|
177
170
|
# From your local machine, copy it up to production server like this:
|
178
171
|
# scp destroy_batch.csv swadm@geoprod:/swadm/var/www/geoblacklight/current/tmp/
|
179
172
|
CSV.foreach("#{Rails.root}/tmp/destroy_batch.csv", headers: true) do |row|
|
180
|
-
sc = SolrDocumentSidecar.find_by(:
|
173
|
+
sc = SolrDocumentSidecar.find_by(document_id: row[0])
|
181
174
|
sc.destroy
|
182
175
|
puts "document_id - #{row[0]} - destroyed"
|
183
176
|
end
|
184
177
|
end
|
185
178
|
|
186
|
-
desc
|
179
|
+
desc "Inspect failed state objects"
|
187
180
|
task harvest_failed_state_inspect: :environment do
|
188
181
|
states = [
|
189
182
|
:failed
|
190
183
|
]
|
191
184
|
|
192
185
|
states.each do |state|
|
193
|
-
|
186
|
+
SolrDocumentSidecar.in_state(state).each do |sc|
|
194
187
|
puts "#{state} - #{sc.document_id} - #{sc.image_state.last_transition.metadata.inspect}"
|
195
188
|
end
|
196
189
|
end
|
data/solr/conf/schema.xml
CHANGED
@@ -143,13 +143,11 @@
|
|
143
143
|
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_d"/>
|
144
144
|
|
145
145
|
<fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
|
146
|
-
geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers"/>
|
146
|
+
geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
|
147
147
|
<!-- Adding field type for bboxField that enables, among other things, overlap ratio calculations -->
|
148
148
|
<fieldType name="bbox" class="solr.BBoxField"
|
149
149
|
geo="true" distanceUnits="kilometers" numberType="pdouble" />
|
150
150
|
<fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
|
151
|
-
|
152
|
-
|
153
151
|
</types>
|
154
152
|
|
155
153
|
<!-- for scoring formula -->
|
@@ -169,8 +167,8 @@
|
|
169
167
|
<copyField source="layer_slug_s" dest="layer_slug_ti" maxChars="100"/>
|
170
168
|
|
171
169
|
<!-- core text search -->
|
172
|
-
<copyField source="*
|
173
|
-
<copyField source="*
|
170
|
+
<copyField source="*_s" dest="text" />
|
171
|
+
<copyField source="*_sm" dest="text" />
|
174
172
|
|
175
173
|
<!-- for sorting text fields -->
|
176
174
|
<copyField source="dct_provenance_s" dest="dct_provenance_sort"/>
|
@@ -195,4 +193,4 @@
|
|
195
193
|
|
196
194
|
<!-- for bbox value -->
|
197
195
|
<copyField source="solr_geom" dest="solr_bboxtype"/>
|
198
|
-
</schema>
|
196
|
+
</schema>
|
@@ -14,7 +14,7 @@
|
|
14
14
|
"layer_slug_s":"aaf63e35-3d5f-4e62-b59d-377f008e9aad",
|
15
15
|
"dc_format_s":"TIFF",
|
16
16
|
"dc_creator_sm":["Minnesota. Department of Highways."],
|
17
|
-
"thumbnail_path_ss":"https://
|
17
|
+
"thumbnail_path_ss":"https://cdm16022.contentdm.oclc.org/utils/getthumbnail/collection/p16022coll206/id/133.jpg",
|
18
18
|
"dc_type_s":"Still image",
|
19
19
|
"dc_identifier_s":"aaf63e35-3d5f-4e62-b59d-377f008e9aad",
|
20
20
|
"dc_relation_sm":["http://sws.geonames.org/5037779/about/rdf"],
|
@@ -1,14 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rails_helper"
|
4
4
|
|
5
5
|
describe GeoblacklightSidecarImages::StoreImageJob, type: :job do
|
6
6
|
let(:document) { SolrDocument.new(document_attributes) }
|
7
7
|
|
8
|
-
describe
|
9
|
-
let(:document_attributes) { json_data(
|
8
|
+
describe "#perform_later" do
|
9
|
+
let(:document_attributes) { json_data("umn_iiif_jpg") }
|
10
10
|
|
11
|
-
it
|
11
|
+
it "stores an image" do
|
12
12
|
ActiveJob::Base.queue_adapter = :test
|
13
13
|
expect { GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id) }.to have_enqueued_job(GeoblacklightSidecarImages::StoreImageJob)
|
14
14
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails_helper"
|
4
|
+
require "spec_helper"
|
5
|
+
require "rake"
|
6
|
+
|
7
|
+
describe "geoblacklight_sidecar_images_tasks.rake" do
|
8
|
+
include ActiveJob::TestHelper
|
9
|
+
|
10
|
+
# @TODO
|
11
|
+
# gblsci:sample_data:seed
|
12
|
+
# gblsci:images:harvest_all
|
13
|
+
# gblsci:images:harvest_states
|
14
|
+
# gblsci:images:harvest_retry
|
15
|
+
# gblsci:images:harvest_report
|
16
|
+
# gblsci:images:harvest_purge_all
|
17
|
+
# gblsci:images:harvest_purge_orphans
|
18
|
+
# gblsci:images:harvest_destroy_batch
|
19
|
+
# gblsci:images:harvest_failed_state_inspect
|
20
|
+
|
21
|
+
# DOC_ID='stanford-cz128vq0535' bundle exec rake gblsci:images:harvest_doc_id
|
22
|
+
describe "gblsci:images:harvest_doc_id" do
|
23
|
+
before do
|
24
|
+
Rails.application.load_tasks
|
25
|
+
end
|
26
|
+
|
27
|
+
it "enqueues background job to harvest image" do
|
28
|
+
ENV["DOC_ID"] = "princeton-02870w62c"
|
29
|
+
Rake::Task["gblsci:images:harvest_doc_id"].invoke
|
30
|
+
expect(enqueued_jobs.size).to eq(1)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|