geoblacklight_sidecar_images 0.1.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3233e1aaff539c85ba41a37cd3e87b7a8ff083d7
-  data.tar.gz: 330ef9966073b81e5fdc539287e25ef2333b98f8
+  metadata.gz: 520f1299deb657555a88fde5230b77fa5d37f79a
+  data.tar.gz: f9416c70247fc5a4c8354be1116f7570610c4959
 SHA512:
-  metadata.gz: 5a8af424a1f10bec71d607c4997810b054aab5948b662a83cdf2325ed3005cf90d6dc0bd0b5a496a2a0303d454941b38b968e9c81c3fd8b3c56a97fc36ebfcf7
-  data.tar.gz: d3fa62b61a58cbb07edd93ad94b1e93c2e652d9a06947a2011c9a40ffa30afd8949e33e3d68c3d7072f8c085bb3d82f794592c4f2984f6552e9b5711a2272542
+  metadata.gz: 888fef80d916c4469cef6412b64f083824bb4062f8dc53ec500424fac9b7ce90e8ff8e5db771bedf65d2c2ae8075d79b1b993ee254399d7954184e03caa06dd1
+  data.tar.gz: 49688593216ab742e2f97a851a5b57a5297d9123f38fde6d00f3d7ed96b081091e6526b5b78aec0fa5018cdac1df693df185b17ee96e29ec0792dd3abd1b3a19

data/.gitignore

@@ -16,3 +16,4 @@ spec/test_app/yarn-error.log
 spec/test_app/storage/
 spec/test_app/tmp/
 .internal_test_app
+.ruby-version

data/Gemfile

@@ -37,11 +37,11 @@ if File.exist?(file)
   end
 else
   Bundler.ui.warn "[EngineCart] Unable to find test application dependencies in #{file}, using placeholder dependencies"
-
-  gem 'aasm'
-  gem 'carrierwave'
-  gem 'geoblacklight', '~> 1.7'
+  gem 'geoblacklight', '~> 1.9'
   gem 'mini_magick'
-  gem 'rails', '>= 4.2', '< 6'
+  gem 'image_processing', '~> 1.6'
+  gem 'mimemagic', '~> 0.3'
+  gem 'statesman', '~> 3.4'
+  gem 'rails', '~> 5.2.0', '< 6'
 end
 # END ENGINE_CART BLOCK

data/README.md

@@ -8,16 +8,23 @@
 Store local copies of remote imagery in GeoBlacklight.
 
 ## Description
-This GeoBlacklight plugin helps capture remote images from geo web services and save them locally. It borrows the concept of a [SolrDocumentSidecar](https://github.com/projectblacklight/spotlight/blob/master/app/models/spotlight/solr_document_sidecar.rb) from [Spotlight](https://github.com/projectblacklight/spotlight), to have an ActiveRecord-based "sidecar" to match each non-AR SolrDocument. This allows us to use [carrierwave](https://github.com/carrierwaveuploader/carrierwave) to attach images to our documents.
+This GeoBlacklight plugin captures remote images from geographic web services and saves them locally. It borrows the concept of a [SolrDocumentSidecar](https://github.com/projectblacklight/spotlight/blob/master/app/models/spotlight/solr_document_sidecar.rb) from [Spotlight](https://github.com/projectblacklight/spotlight), to have an ActiveRecord-based "sidecar" to match each non-AR SolrDocument. This allows us to use [ActiveStorage](https://github.com/rails/rails/tree/master/activestorage) to attach images to our solr documents.
 
 ### Example Screenshot
 ![Screenshot](screenshot.png)
 
 ## Requirements
 
+* [Ruby on Rails 5.2](https://weblog.rubyonrails.org/releases/)
 * [GeoBlacklight](https://github.com/geoblacklight/geoblacklight)
 * [ImageMagick](https://github.com/ImageMagick/ImageMagick)
 
+## Suggested
+
+* Background Job Processor
+
+[Sidekiq](https://github.com/mperham/sidekiq) is an excellent choice if you need an opinion.
+
 ## Installation
 
 ### Existing GeoBlacklight Instance
@@ -52,21 +59,105 @@ $ rails new app-name -m https://raw.githubusercontent.com/ewlarson/geoblacklight
 ### Ingest Test Documents
 
 ```bash
-rake geoblacklight_sidecar_images:sample_data:ingest['<FULL_PATH_TO>/geoblacklight_sidecar_images/spec/fixtures/files']
+  # Run your GBL instance
+  rake geoblacklight:server
 ```
 
-### Cache images
+```bash
+rake gblsci:sample_data:ingest['<FULL_LOCAL_PATH_TO>/geoblacklight_sidecar_images/spec/fixtures/files']
+```
 
-#### All Thumbnails
+## Rake tasks
+
+### Harvest images
+
+#### Harvest all images
+
+Spawns background jobs to harvest images for all documents in your Solr index.
 
 ```bash
-rake geoblacklight_sidecar_images:images:precache_all
+rake gblsci:images:harvest_all
 ```
 
-#### Individual Thumbnail
+#### Harvest an individual image
+
+Allows you to add images one document id at a time.
 
 ```bash
-rake geoblacklight_sidecar_images:images:precache_id['minnesota-iiif-jpg-83f4648a-125c-4000-a12f-aba2b432e7cd']
+rake gblsci:images:harvest_doc_id['stanford-cz128vq0535']
+```
+
+#### Harvest all incomplete states
+
+Reattempt image harvesting for all non-successful state objects.
+
+```bash
+rake gblsci:images:harvest_retry
+```
+
+### Check image states
+
+```bash
+rake gblsci:images:harvest_states
+```
+
+We use a state machine library to track success/failure of our harvest tasks. The states we track are:
+
+* initialized - SolrDocumentSidecar created, no harvest attempt run
+* queued - Harvest attempt queued as background job
+* processing - Harvest attempt at work
+* succeeded - Harvest was successful, image attached
+* failed - Harvest failed, no image attached, error logged
+* placeheld - Harvest was not successful, placeholder imagery will be used
+
+```ruby
+SolrDocumentSidecar.image.attached? => false
+SolrDocumentSidecar.image_state.current_state => "placeheld"
+SolrDocumentSidecar.image_state.last_transition => #<SidecarImageTransition id: 207, to_state: "placeheld", metadata: {"solr_doc_id"=>"stanford-cg357zz0321", "solr_version"=>1616509329754554368, "placeheld"=>true, "viewer_protocol"=>"wms", "image_url"=>"http://geowebservices-restricted.stanford.edu/geoserver/wms/reflect?&FORMAT=image%2Fpng&TRANSPARENT=TRUE&LAYERS=druid:cg357zz0321&WIDTH=300&HEIGHT=300", "service_url"=>"http://geowebservices-restricted.stanford.edu/geoserver/wms/reflect?&FORMAT=image%2Fpng&TRANSPARENT=TRUE&LAYERS=druid:cg357zz0321&WIDTH=300&HEIGHT=300", "gblsi_thumbnail_uri"=>false, "error"=>"Faraday::Error::ConnectionFailed"},...>
+```
+
+### Destroy images
+
+#### Remove everything
+
+Remove all sidecar objects and attached images
+
+```bash
+rake gblsci:images:harvest_purge_all
+```
+
+#### Remove orphaned AR objects
+
+Remove all sidecar objects and attached images for AR objects without a corresponding Solr document
+
+```bash
+rake gblsci:images:harvest_purge_orphans
+```
+
+#### Remove a batch
+
+Remove sidecar objects and attached images via a CSV file of document ids
+
+```bash
+rake gblsci:images:harvest_purge_orphans
+```
+
+### Troubleshooting
+
+#### Harvest report
+
+Generate a CSV file of sidecar objects and associated image state. Useful for debugging problem items.
+
+```bash
+rake gblsci:images:harvest_report
+```
+
+#### Failed state inspect
+
+Prints details for failed state harvest objects to stdout
+
+```bash
+rake gblsci:images:harvest_failed_state_inspect
 ```
 
 ## Prioritize Solr Thumbnail Field URIs
@@ -88,20 +179,54 @@ If you add a thumbnail uri to your geoblacklight solr documents...
 
 Then you can edit your GeoBlacklight settings.yml file to point at that solr field (Settings.GBLSI_THUMBNAIL_FIELD). Any docs in your index that have a value for that field will harvest the image at that URI instead of trying to retrieve an image via IIIF or the other web services.
 
+## View customization
+
+This GBL plugin includes a custom catalog/_index_split_default.html.erb file. Look there for examples on calling the image method.
+
+```ruby
+# Is there an image?
+document.sidecar.image.attached?
+
+# Can the image size be manipulated?
+document.sidecar.image.variable?
+
+# Example image_tag with resize
+<%= image_tag document.sidecar.image.variant(resize: "100x100"), {class: 'media-object'} %>
+
+```
+
+You'll definitely want to update this file to fit your own design's needs.
+
 ## Development
 
 ```bash
+
+# Run test suite
 bundle exec rake ci
+
+# Launch test app server
 cd .internal_test_app/
 rake geoblacklight:server
+
+# Load test fixtures
+bundle exec rake gblsci:sample_data:ingest['<FULL_LOCAL_PATH_TO>/geoblacklight_sidecar_images/spec/fixtures/files']
+
+# Run harvest
+bundle exec rake gblsci:images:harvest_all
+
+# Tail log
+tail -f log/image_service_development.log
+
 ```
 
-Now you'll have an instance of GBLSI running. Follow the rake tasks above to ingest some data and harvest thumbnails.
+[See Localhost Results](http://localhost:3000/?per_page=50&q=&search_field=all_fields)
 
 ## TODOs
 
 * ~~0.0.1 - Initial gem~~
 * ~~0.1.0 - Prioritize local thumbnail solr field~~
-* 0.2.0 - Forgo attaching placeholder imagery
-* 0.3.0 - Add Statesman (state machine library)
-* 1.0.0 - Rails 5.2 branch / Switch to ActionStorage
+* ~~0.2.0 - Forgo attaching placeholder imagery~~
+* ~~0.3.0 - Add Statesman (state machine library)~~
+* ~~0.4.0 - Rails 5.2 branch / Switch to ActionStorage~~
+* 0.5.0 to 0.9.0 - Feedback; Improve test coverage; Collect additional real-world issues
+* 1.0.0 - Final 5.2 release

data/db/migrate/20180118203155_create_solr_document_sidecars.rb

@@ -1,10 +1,12 @@
-class CreateSolrDocumentSidecars < ActiveRecord::Migration[5.1]
+class CreateSolrDocumentSidecars < ActiveRecord::Migration[5.2]
   def change
     create_table :solr_document_sidecars do |t|
       t.string "document_id"
       t.string "document_type"
       t.string "image"
-      t.index ["document_type", "document_id"], name: "solr_document_sidecars_solr_document"
+      t.integer "version", :limit => 8
+
+      t.index ["document_type", "document_id"], name: "sidecars_solr_document"
 
       t.timestamps
     end

data/db/migrate/20180118203519_create_sidecar_image_transitions.rb

@@ -0,0 +1,29 @@
+class CreateSidecarImageTransitions < ActiveRecord::Migration[5.2]
+  def change
+    create_table :sidecar_image_transitions do |t|
+      t.string :to_state, null: false
+      t.text :metadata
+      t.integer :sort_key, null: false
+      t.bigint :solr_document_sidecar_id, null: false
+      t.boolean :most_recent
+
+      # If you decide not to include an updated timestamp column in your transition
+      # table, you'll need to configure the `updated_timestamp_column` setting in your
+      # migration class.
+      t.timestamps null: false
+    end
+
+    # Foreign keys are optional, but highly recommended
+    add_foreign_key :sidecar_image_transitions, :solr_document_sidecars
+
+    add_index(:sidecar_image_transitions,
+              [:solr_document_sidecar_id, :sort_key],
+              unique: true,
+              name: "index_sidecar_image_transitions_parent_sort")
+    add_index(:sidecar_image_transitions,
+              [:solr_document_sidecar_id, :most_recent],
+              unique: true,
+
+              name: "index_sidecar_image_transitions_parent_most_recent")
+  end
+end

data/geoblacklight_sidecar_images.gemspec

@@ -18,10 +18,12 @@ Gem::Specification.new do |s|
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.require_paths = ['lib']
 
-  s.add_dependency 'carrierwave', '~> 1.2'
-  s.add_dependency 'geoblacklight', '~> 1.7'
+  s.add_dependency 'geoblacklight', '~> 1.9'
   s.add_dependency 'mini_magick', '~> 4.8'
-  s.add_dependency 'rails', '>= 4.2', '< 6'
+  s.add_dependency 'image_processing', '~> 1.6'
+  s.add_dependency 'statesman', '~> 3.4'
+  s.add_dependency 'mimemagic', '~> 0.3'
+  s.add_dependency 'rails', '>= 5.2', '< 6'
 
   s.add_development_dependency 'byebug'
   s.add_development_dependency 'capybara'

data/lib/generators/geoblacklight_sidecar_images/{uploaders_generator.rb → config_generator.rb}

@@ -3,17 +3,16 @@
 require 'rails/generators'
 
 module GeoblacklightSidecarImages
-  class UploadersGenerator < Rails::Generators::Base
+  class ConfigGenerator < Rails::Generators::Base
     source_root File.expand_path('templates', __dir__)
 
     desc <<-DESCRIPTION
       This generator makes the following changes to your application:
-       1. Creates an app/uploaders directory
-       2. Creates uploader models within the app/uploaders directory
+       1. Copies config files to host config
     DESCRIPTION
 
-    def create_image_uploader
-      directory 'uploaders', 'app/uploaders'
+    def create_store_image_jobs
+      copy_file 'config/initializers/statesman.rb', 'config/initializers/statesman.rb'
     end
   end
 end

data/lib/generators/geoblacklight_sidecar_images/install_generator.rb

@@ -16,16 +16,6 @@ GBLSI_THUMBNAIL_FIELD: 'thumbnail_path_ss'"
       end
     end
 
-    def add_carrierwave_require
-      inject_into_file 'config/application.rb', after: "require 'rails/all'" do
-        "\n  require 'carrierwave'"
-      end
-    end
-
-    def generate_geoblacklight_assets
-      generate 'geoblacklight_sidecar_images:assets'
-    end
-
     def generate_geoblacklight_example_docs
       generate 'geoblacklight_sidecar_images:example_docs'
     end
@@ -42,14 +32,18 @@ GBLSI_THUMBNAIL_FIELD: 'thumbnail_path_ss'"
       generate 'geoblacklight_sidecar_images:services'
     end
 
-    def generate_geoblacklight_uploaders
-      generate 'geoblacklight_sidecar_images:uploaders'
-    end
-
     def generate_geoblacklight_views
       generate 'geoblacklight_sidecar_images:views'
     end
 
+    def generate_action_storage
+      rake 'active_storage:install'
+    end
+
+    def generate_geoblacklight_config
+      generate 'geoblacklight_sidecar_images:config'
+    end
+
     def bundle_install
       Bundler.with_clean_env do
         run 'bundle install'

data/lib/generators/geoblacklight_sidecar_images/models_generator.rb

@@ -24,10 +24,18 @@ module GeoblacklightSidecarImages
     def include_sidecar_solrdocument
       sidecar = <<-"SIDECAR"
         def sidecar
-          SolrDocumentSidecar.find_or_create_by!(
+          # Find or create, and set version
+          sidecar = SolrDocumentSidecar.where(
             document_id: id,
             document_type: self.class.to_s
-          )
+          ).first_or_create do |sc|
+            sc.version = self._source["_version_"]
+          end
+
+          sidecar.version = self._source["_version_"]
+          sidecar.save
+
+          sidecar
         end
       SIDECAR
 
@@ -41,6 +49,20 @@ module GeoblacklightSidecarImages
       )
     end
 
+    def create_sidecar_image_transition
+      copy_file(
+        'models/sidecar_image_transition.rb',
+        'app/models/sidecar_image_transition.rb'
+      )
+    end
+
+    def create_sidecar_image_state_machine
+      copy_file(
+        'models/sidecar_image_state_machine.rb',
+        'app/models/sidecar_image_state_machine.rb'
+      )
+    end
+
     def create_wms_rewrite_concern
       copy_file(
         'models/concerns/wms_rewrite_concern.rb',

data/lib/generators/geoblacklight_sidecar_images/templates/config/initializers/statesman.rb

@@ -0,0 +1,5 @@
+require 'statesman'
+
+Statesman.configure do
+  storage_adapter(Statesman::Adapters::ActiveRecord)
+end

data/lib/generators/geoblacklight_sidecar_images/templates/jobs/store_image_job.rb

@@ -3,8 +3,15 @@
 class StoreImageJob < ApplicationJob
   queue_as :default
 
-  def perform(document_hash)
-    doc = SolrDocument.new(document_hash)
-    ImageService.new(doc).store
+  def perform(solr_document_id)
+    cat = CatalogController.new
+    response, document = cat.fetch(solr_document_id)
+
+    metadata = Hash.new
+    metadata['solr_doc_id'] = document.id
+    metadata['solr_version'] = document.sidecar.version
+
+    document.sidecar.image_state.transition_to!(:queued, metadata)
+    ImageService.new(document).store
   end
 end

data/lib/generators/geoblacklight_sidecar_images/templates/models/sidecar_image_state_machine.rb

@@ -0,0 +1,19 @@
+class SidecarImageStateMachine
+  include Statesman::Machine
+
+  state :initialized, initial: true
+  state :queued
+  state :processing
+  state :succeeded
+  state :failed
+  state :placeheld
+
+  # Queued => Background Job Init
+  # Processing => Failed, Placeheld, Succeeded
+  transition from: :initialized,  to: [:queued, :processing]
+  transition from: :queued,       to: [:queued, :processing]
+  transition from: :processing,   to: [:queued, :processing, :placeheld, :succeeded, :failed]
+  transition from: :placeheld,    to: [:queued, :processing]
+  transition from: :failed,       to: [:queued, :processing]
+  transition from: :succeeded,    to: [:queued, :processing]
+end