geoblacklight_sidecar_images 0.1.0 → 0.5.0

data/lib/generators/geoblacklight_sidecar_images/templates/models/sidecar_image_transition.rb

@@ -0,0 +1,7 @@
+class SidecarImageTransition < ActiveRecord::Base
+  include Statesman::Adapters::ActiveRecordTransition
+
+  validates :to_state, inclusion: { in: SidecarImageStateMachine.states }
+
+  belongs_to :solr_document_sidecar, inverse_of: :sidecar_image_transitions
+end

data/lib/generators/geoblacklight_sidecar_images/templates/models/solr_document_sidecar.rb

@@ -3,11 +3,15 @@
 ##
 # Metadata for indexed documents
 class SolrDocumentSidecar < ApplicationRecord
-  mount_uploader :image, ImageUploader
+  include Statesman::Adapters::ActiveRecordQueries
 
   belongs_to :document, required: true, polymorphic: true
+  has_many :sidecar_image_transitions, autosave: false
+  has_one_attached :image
+
+  # If the sidecar solr document is updated, re-fetch thumbnail image
+  after_update :reimage, if: :saved_change_to_version?
 
-  # Roll our own polymorphism because our documents are not AREL-able
   def document
     document_type.new document_type.unique_key => document_id
   end
@@ -15,4 +19,32 @@ class SolrDocumentSidecar < ApplicationRecord
   def document_type
     (super.constantize if defined?(super)) || default_document_type
   end
+
+  def image_state
+    @state_machine ||= SidecarImageStateMachine.new(
+      self,
+      transition_class: SidecarImageTransition
+    )
+  end
+
+  def self.transition_class
+    SidecarImageTransition
+  end
+
+  def self.initial_state
+    :initialized
+  end
+
+  def self.image_url
+    Rails.application.routes.url_helpers.rails_blob_path(self.image, only_path: true)
+  end
+
+  private_class_method :initial_state
+
+  private
+
+  def reimage
+    self.image.purge if self.image.attached?
+    StoreImageJob.perform_later(self.document.id)
+  end
 end

data/lib/generators/geoblacklight_sidecar_images/templates/services/image_service.rb

@@ -1,10 +1,21 @@
 # frozen_string_literal: true
-
-require 'rack/mime'
+require "addressable/uri"
+require "mimemagic"
 
 class ImageService
+  attr_reader :document
+  attr_writer :metadata, :logger
+
   def initialize(document)
     @document = document
+
+    @metadata = Hash.new
+    @metadata['solr_doc_id'] = document.id
+    @metadata['solr_version'] = @document.sidecar.version
+    @metadata['placeheld'] = false
+
+    @document.sidecar.image_state.transition_to!(:processing, @metadata)
+
     @logger ||= ActiveSupport::TaggedLogging.new(
       Logger.new(
         File.join(
@@ -14,46 +25,66 @@ class ImageService
     )
   end
 
-  # Stores the document's image in SolrDocumentSidecar
-  # using Carrierwave
+  # Stores the document's image in ActiveStorage
   # @return [Boolean]
   #
-  # @TODO: EWL
   def store
-    sidecar = @document.sidecar
-    sidecar.image = image_tempfile(@document.id)
-    sidecar.save!
-    @logger.tagged(@document.id, 'STATUS') { @logger.info 'SUCCESS' }
-    @logger.tagged(@document.id, 'SIDECAR_IMAGE_URL') { @logger.info @document.sidecar.image_url }
-  rescue ActiveRecord::RecordInvalid, FloatDomainError => invalid
-    @logger.tagged(@document.id, 'STATUS') { @logger.info 'FAILURE' }
-    @logger.tagged(@document.id, 'EXCEPTION') { @logger.info invalid.inspect }
-  end
+    # Gentle hands
+    sleep(1)
 
-  # Returns hash containing placeholder thumbnail for the document.
-  # @return [Hash]
-  #   * :type [String] image mime type
-  #   * :data [String] image file data
-  def placeholder
-    placeholder_data
-  end
+    io_file = image_tempfile(@document.id)
 
-  private
+    if io_file.nil? || @metadata['placeheld'] == true
+      @document.sidecar.image_state.transition_to!(:placeheld, @metadata)
+      log_output
+    else
+      # Remote content-type headers are untrustworthy
+      # Pull the mimetype and file extension via MimeMagic
+      mm = MimeMagic.by_magic(File.open(io_file))
+
+      @metadata['MimeMagic_type'] = mm.type
+      @metadata['MimeMagic_mediatype'] = mm.mediatype
+      @metadata['MimeMagic_subtype'] = mm.subtype
+
+      if mm.mediatype == "image"
+        @document.sidecar.image.attach(
+          io: io_file,
+          filename: "#{@document.id}.#{mm.subtype}",
+          content_type: mm.type
+        )
+        @document.sidecar.image_state.transition_to!(:succeeded, @metadata)
+      else
+        @document.sidecar.image_state.transition_to!(:placeheld, @metadata)
+      end
 
-  def image_tempfile(document_id)
-    @logger.tagged(@document.id, 'remote_content_type') { @logger.info remote_content_type }
-    @logger.tagged(@document.id, 'viewer_protocol') { @logger.info @document.viewer_protocol }
-    @logger.tagged(@document.id, 'service_url') { @logger.info service_url }
-    @logger.tagged(@document.id, 'image_extension') { @logger.info image_extension }
+      log_output
+    end
+  rescue Exception => invalid
+    @metadata['exception'] = invalid.inspect
+    @document.sidecar.image_state.transition_to!(:failed, @metadata)
 
-    file = Tempfile.new([document_id, image_extension])
-    file.binmode
-    file.write(image_data[:data])
-    file.close
+    log_output
+  end
 
-    @logger.tagged(@document.id, 'IMAGE_TEMPFILE') { @logger.info file.inspect }
+  private
 
-    file
+  def image_tempfile(document_id)
+    @metadata['viewer_protocol']      = @document.viewer_protocol
+    @metadata['image_url']            = image_url
+    @metadata['service_url']          = service_url
+    @metadata['gblsi_thumbnail_uri']  = gblsi_thumbnail_uri
+
+    if image_data && @metadata['placeheld'] == false
+      temp_file = Tempfile.new([document_id, ".tmp"])
+      temp_file.binmode
+      temp_file.write(image_data)
+      temp_file.rewind
+
+      @metadata['image_tempfile'] = temp_file.inspect
+      temp_file
+    else
+      return nil
+    end
   end
 
   # Returns geoserver auth credentials if the document is a restriced Local WMS layer.
@@ -80,71 +111,38 @@ class ImageService
     end
   end
 
-  def placeholder_base_path
-    Rails.root.join('app', 'assets', 'images')
-  end
-
-  # Generates hash containing placeholder mime_type and image.
-  def placeholder_data
-    { type: 'image/png', data: placeholder_image }
-  end
-
-  # Gets placeholder image from disk.
-  def placeholder_image
-    File.read(placeholder_image_path)
-  end
-
-  # Path to placeholder image based on the layer geometry.
-  def placeholder_image_path
-    geom_type = @document.fetch('layer_geom_type_s', '').tr(' ', '-').downcase
-    thumb_path = "#{placeholder_base_path}/thumbnail-#{geom_type}.png"
-    return "#{placeholder_base_path}/thumbnail-paper-map.png" unless File.exist?(thumb_path)
-    thumb_path
-  end
-
   # Generates hash containing thumbnail mime_type and image.
   def image_data
-    return placeholder_data unless image_url
-    { type: remote_content_type, data: remote_image }
+    return nil unless image_url
+    remote_image
   end
 
-  # Gets thumbnail image from URL. On error, returns document's placeholder image.
-  def remote_content_type
+  # Gets thumbnail image from URL. On error, placehold image.
+  def remote_image
     auth = geoserver_credentials
 
-    conn = Faraday.new(url: image_url) do |b|
-      b.use FaradayMiddleware::FollowRedirects
-      b.adapter :net_http
-    end
-
-    conn.options.timeout = timeout
-    conn.options.timeout = timeout
-    conn.authorization :Basic, auth if auth
-
-    conn.head.headers['content-type']
-  rescue Faraday::Error::ConnectionFailed
-    placeholder_data[:type]
-  rescue Faraday::Error::TimeoutError
-    placeholder_data[:type]
-
-  # Rescuing Exception intentionally
-  rescue Exception
-    placeholder_data[:type]
-  end
+    uri = Addressable::URI.parse(image_url)
 
-  # Gets thumbnail image from URL. On error, returns document's placeholder image.
-  def remote_image
-    auth = geoserver_credentials
-    conn = Faraday.new(url: image_url)
-    conn.options.timeout = timeout
-    conn.options.timeout = timeout
-    conn.authorization :Basic, auth if auth
+    if uri.scheme.include?("http")
+      conn = Faraday.new(url: uri.normalize.to_s) do |b|
+        b.use FaradayMiddleware::FollowRedirects
+        b.adapter :net_http
+      end
 
-    conn.get.body
+      conn.options.timeout = timeout
+      conn.authorization :Basic, auth if auth
+      conn.get.body
+    else
+      return nil
+    end
   rescue Faraday::Error::ConnectionFailed
-    placeholder_image
+    @metadata['error'] = "Faraday::Error::ConnectionFailed"
+    @metadata['placeheld'] = true
+    return nil
   rescue Faraday::Error::TimeoutError
-    placeholder_image
+    @metadata['error'] = "Faraday::Error::TimeoutError"
+    @metadata['placeheld'] = true
+    return nil
   end
 
   # Returns the thumbnail url.
@@ -165,11 +163,6 @@ class ImageService
     end
   end
 
-  # Determines the image file extension
-  def image_extension
-    @image_extension ||= Rack::Mime::MIME_TYPES.rassoc(remote_content_type).try(:first) || '.png'
-  end
-
   # Checks if the document is Local restriced access and is a scanned map.
   def restricted_scanned_map?
     @document.local_restricted? && @document['layer_geom_type_s'] == 'Image'
@@ -189,9 +182,15 @@ class ImageService
     @service_url ||= begin
       return unless @document.available?
       protocol = @document.viewer_protocol
-      return if protocol == 'map' || protocol.nil?
+      if protocol == 'map' || protocol.nil?
+        @metadata['error'] = "Unsupported viewer protocol"
+        @metadata['placeheld'] = true
+        return nil
+      end
       "ImageService::#{protocol.camelcase}".constantize.image_url(@document, image_size)
     rescue NameError
+      @metadata['error'] = "service_url NameError"
+      @metadata['placeheld'] = true
       return nil
     end
   end
@@ -202,13 +201,21 @@ class ImageService
     JSON.parse(@document[@document.references.reference_field])['http://schema.org/thumbnailUrl']
   end
 
-  # Default thumbnail size.
+  # Default image size.
   def image_size
-    300
+    1500
   end
 
   # Faraday timeout value.
   def timeout
     30
   end
+
+  # Capture metadata within image harvest log
+  def log_output
+    @metadata["state"] = @document.sidecar.image_state.current_state
+    @metadata.each do |key,value|
+      @logger.tagged(@document.id, key.to_s) { @logger.info value }
+    end
+  end
 end

data/lib/generators/geoblacklight_sidecar_images/templates/views/catalog/_index_split_default.html.erb

@@ -11,7 +11,17 @@
     <div class='col-xs-12'>
       <div class='media'>
         <div class='media-left'>
-          <img class='media-object' src="<%=document.sidecar.image_url(:square)%>"/>
+          <% if document.sidecar.image.attached? %>
+            <% if document.sidecar.image.variable? %>
+              <%= image_tag document.sidecar.image.variant(resize: "100x100"), {class: 'media-object'} %>
+            <% else %>
+              <%= image_tag document.sidecar.image, {class: 'media-object'} %>
+            <% end %>
+          <% else %>
+            <span title="<%=document[Settings.FIELDS.GEOM_TYPE]%>">
+              <%= geoblacklight_icon(document[Settings.FIELDS.GEOM_TYPE]) %>
+            </span>
+          <% end %>
         </div>
         <div class='media-body'>
           <small>

data/lib/geoblacklight_sidecar_images/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GeoblacklightSidecarImages
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.5.0'.freeze
 end

data/lib/tasks/geoblacklight_sidecar_images_tasks.rake

@@ -1,4 +1,6 @@
-namespace :geoblacklight_sidecar_images do
+require 'csv'
+
+namespace :gblsci do
   namespace :sample_data do
     desc 'Ingests a directory of geoblacklight.json files'
     task :ingest, [:directory] => :environment do |_t, args|
@@ -17,60 +19,184 @@ namespace :geoblacklight_sidecar_images do
   end
 
   namespace :images do
-    desc 'Pre-cache specific image'
-    task :precache_id, [:doc_id] => [:environment] do |_t, args|
-      query = "dc_identifier_s:#{args[:doc_id]}"
-      layers = %w[
-        layer_slug_s
-        layer_id_s
-        dc_rights_s
-        dct_provenance_s
-        layer_geom_type_s
-        dct_references_s
-      ]
-      index = Geoblacklight::SolrDocument.index
-      results = index.send_and_receive(index.blacklight_config.solr_path,
-                                       q: query,
-                                       fl: layers.join(','),
-                                       rows: 100_000_000)
-      num_found = results.response[:numFound]
-      doc_counter = 0
-      results.docs.each do |document|
-        begin
-          StoreImageJob.perform_later(document.to_h)
-        rescue Blacklight::Exceptions::RecordNotFound
-          next
-        end
-      end
+    desc 'Harvest image for specific document'
+    task :harvest_doc_id, [:doc_id] => [:environment] do |_t, args|
+      StoreImageJob.perform_later(args[:doc_id])
     end
 
-    desc 'Pre-cache all images'
-    task :precache_all, [:override_existing] => [:environment] do |_t, args|
+    desc 'Harvest all images'
+    task harvest_all: :environment do
       begin
-        query = 'layer_slug_s:*'
-        layers = %w[
-          layer_slug_s
-          layer_id_s
-          dc_rights_s
-          dct_provenance_s
-          layer_geom_type_s
-          dct_references_s
-        ]
+        query = '*:*'
         index = Geoblacklight::SolrDocument.index
         results = index.send_and_receive(index.blacklight_config.solr_path,
                                          q: query,
-                                         fl: layers.join(','),
+                                         fl: "*",
                                          rows: 100_000_000)
         num_found = results.response[:numFound]
         doc_counter = 0
         results.docs.each do |document|
+          sleep(1)
           begin
-            StoreImageJob.perform_later(document.to_h)
+            StoreImageJob.perform_later(document.id)
           rescue Blacklight::Exceptions::RecordNotFound
             next
           end
         end
       end
     end
+
+    desc 'Hash of SolrDocumentSidecar image state counts'
+    task harvest_states: :environment do
+      states = [
+        :initialized,
+        :queued,
+        :processing,
+        :succeeded,
+        :failed,
+        :placeheld
+      ]
+
+      col_state = {}
+      states.each do |state|
+        sidecars = SolrDocumentSidecar.in_state(state)
+        col_state[state] = sidecars.size
+      end
+
+      col_state.each do |col,state|
+        puts "#{col} - #{state}"
+      end
+    end
+
+    desc 'Re-queues incomplete states for harvesting'
+    task harvest_retry: :environment do
+      states = [
+        :initialized,
+        :queued,
+        :processing,
+        :failed,
+        :placeheld
+      ]
+
+      states.each do |state|
+        sidecars = SolrDocumentSidecar.in_state(state)
+
+        puts "#{state} - #{sidecars.size}"
+
+        sidecars.each do |sc|
+          cat = CatalogController.new
+          begin
+            resp, doc = cat.fetch(sc.document_id)
+            StoreImageJob.perform_later(doc.id)
+          rescue
+            puts "orphaned / #{sc.document_id}"
+          end
+        end
+      end
+    end
+
+    desc 'Write harvest state report (CSV)'
+    task harvest_report: :environment do
+      # Create a CSV Dump of Results
+      file = "#{Rails.root}/public/#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}.sidecar_report.csv"
+
+      sidecars = SolrDocumentSidecar.all
+
+      CSV.open(file, 'w') do |writer|
+        header = [
+          "Sidecar ID",
+          "Document ID",
+          "Current State",
+          "Doc Data Type",
+          "Doc Title",
+          "Doc Institution",
+          "Error",
+          "Viewer Protocol",
+          "Image URL",
+          "GBLSI Thumbnail URL"
+        ]
+
+        writer << header
+
+        sidecars.each do |sc|
+          cat = CatalogController.new
+          begin
+            resp, doc = cat.fetch(sc.document_id)
+            writer << [
+              sc.id,
+              sc.document_id,
+              sc.image_state.current_state,
+              doc._source['layer_geom_type_s'],
+              doc._source['dc_title_s'],
+              doc._source['dct_provenance_s'],
+              sc.image_state.last_transition.metadata['exception'],
+              sc.image_state.last_transition.metadata['viewer_protocol'],
+              sc.image_state.last_transition.metadata['image_url'],
+              sc.image_state.last_transition.metadata['gblsi_thumbnail_uri']
+            ]
+          rescue Exception => e
+            puts "Exception: #{e.inspect}"
+            puts "orphaned / #{sc.document_id}"
+            next
+          end
+        end
+      end
+    end
+
+    desc 'Destroy all harvested images and sidecar AR objects'
+    task harvest_purge_all: :environment do
+      # Remove all images
+      sidecars = SolrDocumentSidecar.all
+      sidecars.each do |sc|
+        sc.image.purge
+      end
+
+      # Delete all Transitions and Sidecars
+      SidecarImageTransition.destroy_all
+      SolrDocumentSidecar.destroy_all
+    end
+
+    desc 'Destroy orphaned images and sidecar AR objects'
+    # When a SolrDocumentSidecar AR object exists,
+    # but it's corresponding SolrDocument is no longer in the Solr index.
+    task harvest_purge_orphans: :environment do
+      # Remove all images
+      sidecars = SolrDocumentSidecar.all
+      sidecars.each do |sc|
+        cat = CatalogController.new
+        begin
+          resp, doc = cat.fetch(sc.document_id)
+        rescue
+          sc.destroy
+          puts "orphaned / #{sc.document_id} / destroyed"
+        end
+      end
+    end
+
+    desc 'Destroy select sidecar AR objects by CSV file'
+    task harvest_destroy_batch: :environment do
+      # Expects a CSV file in Rails.root/tmp/destroy_batch.csv
+      #
+      # From your local machine, copy it up to production server like this:
+      # scp destroy_batch.csv swadm@geoprod:/swadm/var/www/geoblacklight/current/tmp/
+      CSV.foreach("#{Rails.root}/tmp/destroy_batch.csv", headers: true) do |row|
+        sc = SolrDocumentSidecar.find_by(:document_id => row[0])
+        sc.destroy
+        puts "document_id - #{row[0]} - destroyed"
+      end
+    end
+
+    desc 'Inspect failed state objects'
+    task harvest_failed_state_inspect: :environment do
+      states = [
+        :failed
+      ]
+
+      states.each do |state|
+        sidecars = SolrDocumentSidecar.in_state(state).each do |sc|
+          puts "#{state} - #{sc.document_id} - #{sc.image_state.last_transition.metadata.inspect}"
+        end
+      end
+    end
   end
 end