blacklight-spotlight 3.0.0.alpha.9 → 3.0.0.rc4

data/app/models/spotlight/resources/csv_upload.rb

@@ -3,9 +3,10 @@
 module Spotlight
   module Resources
     ##
-    # Shim object for CSV Uploads. see {Spotlight::AddUploadsFromCSV}
+    # Shim object for CSV Uploads. see {Spotlight::AddUploadsFromCsv}
     class CsvUpload
       attr_reader :url
+
       include ActiveModel::Model
       extend ActiveModel::Translation
     end

data/app/models/spotlight/resources/iiif_harvester.rb

@@ -7,7 +7,6 @@ module Spotlight
     # harvest Images from IIIF Manifest and turn them into a Spotlight::Resource
     # Note: IIIF API : http://iiif.io/api/presentation/2.0
     class IiifHarvester < Spotlight::Resource
-      self.document_builder_class = Spotlight::Resources::IiifBuilder
       self.weight = -5000
 
       validate :valid_url?
@@ -16,6 +15,16 @@ module Spotlight
         @iiif_manifests ||= IiifService.parse(url)
       end
 
+      def self.indexing_pipeline
+        @indexing_pipeline ||= super.dup.tap do |pipeline|
+          pipeline.sources = [Spotlight::Etl::Sources::SourceMethodSource(:iiif_manifests)]
+
+          pipeline.transforms = [
+            ->(data, p) { data.merge(p.source.to_solr(exhibit: p.context.resource.exhibit)) }
+          ] + pipeline.transforms
+        end
+      end
+
       private
 
       def valid_url?

data/app/models/spotlight/resources/iiif_manifest.rb

@@ -6,6 +6,7 @@ module Spotlight
     # A PORO to construct a solr hash for a given IiifManifest
     class IiifManifest
       attr_reader :collection
+
       def initialize(attrs = {})
         @url = attrs[:url]
         @manifest = attrs[:manifest]
@@ -13,7 +14,9 @@ module Spotlight
         @solr_hash = {}
       end
 
-      def to_solr
+      def to_solr(exhibit: nil)
+        @exhibit = exhibit if exhibit
+
         add_document_id
         add_label
         add_thumbnail_url
@@ -36,6 +39,7 @@ module Spotlight
       private
 
       attr_reader :url, :manifest, :exhibit, :solr_hash
+
       delegate :blacklight_config, to: :exhibit
 
       def add_document_id
@@ -67,7 +71,7 @@ module Spotlight
       end
 
       def add_label
-        return unless title_fields.present? && manifest.try(:label)
+        return unless title_fields.present? && manifest&.label
 
         Array.wrap(title_fields).each do |field|
           solr_hash[field] = metadata_class.new(manifest).label
@@ -142,7 +146,7 @@ module Spotlight
       end
 
       def thumbnail_field
-        blacklight_config.index.try(:thumbnail_field)
+        blacklight_config.index.thumbnail_field
       end
 
       def full_image_field
@@ -150,11 +154,11 @@ module Spotlight
       end
 
       def tile_source_field
-        blacklight_config.show.try(:tile_source_field)
+        blacklight_config.show.tile_source_field
       end
 
       def title_fields
-        Spotlight::Engine.config.iiif_title_fields || blacklight_config.index.try(:title_field)
+        Spotlight::Engine.config.iiif_title_fields || blacklight_config.index&.title_field
       end
 
       def sidecar
@@ -185,7 +189,7 @@ module Spotlight
         end
 
         def label
-          return unless manifest.try(:label)
+          return unless manifest&.label
 
           Array(json_ld_value(manifest.label)).map { |v| html_sanitize(v) }.first
         end
@@ -195,7 +199,7 @@ module Spotlight
         attr_reader :manifest
 
         def metadata
-          manifest.try(:metadata) || []
+          manifest&.metadata || []
         end
 
         def metadata_hash

data/app/models/spotlight/resources/iiif_service.rb

@@ -42,7 +42,7 @@ module Spotlight
       class << self
         def iiif_response(url)
           Faraday.get(url).body
-        rescue Faraday::Error::ConnectionFailed, Faraday::TimeoutError => e
+        rescue Faraday::ConnectionFailed, Faraday::TimeoutError => e
           Rails.logger.warn("HTTP GET for #{url} failed with #{e}")
           {}.to_json
         end

data/app/models/spotlight/resources/json_upload.rb

@@ -5,6 +5,18 @@ module Spotlight
     # Raw solr document uploads
     class JsonUpload < Spotlight::Resource
       store :data, accessors: :json
+
+      # The indexing pipeline for JSON uploads copies the data from the stored
+      # `#data` field directly into the indexed document.
+      def self.indexing_pipeline
+        @indexing_pipeline ||= super.dup.tap do |pipeline|
+          pipeline.sources = [Spotlight::Etl::Sources::StoredData]
+
+          pipeline.transforms = [
+            Spotlight::Etl::Transforms::IdentityTransform
+          ] + pipeline.transforms
+        end
+      end
     end
   end
 end

data/app/models/spotlight/resources/upload.rb

@@ -10,8 +10,6 @@ module Spotlight
       # we want to do this before reindexing
       after_create :update_document_sidecar
 
-      self.document_builder_class = UploadSolrDocumentBuilder
-
       def self.fields(exhibit)
         @fields ||= {}
         @fields[exhibit] ||= begin
@@ -25,6 +23,15 @@ module Spotlight
         end
       end
 
+      def self.indexing_pipeline
+        @indexing_pipeline ||= super.dup.tap do |pipeline|
+          pipeline.transforms = [
+            ->(data, p) { data.merge({ p.context.document_model.unique_key.to_sym => p.source.compound_id }) },
+            Spotlight::Etl::Transforms::SourceMethodTransform(:to_solr)
+          ] + pipeline.transforms
+        end
+      end
+
       def compound_id
         "#{exhibit_id}-#{id}"
       end
@@ -33,6 +40,22 @@ module Spotlight
         @sidecar ||= document_model.new(id: compound_id).sidecar(exhibit)
       end
 
+      def to_solr
+        return {} unless upload.file_present?
+
+        spotlight_routes = Spotlight::Engine.routes.url_helpers
+        riiif = Riiif::Engine.routes.url_helpers
+
+        dimensions = Riiif::Image.new(upload_id).info
+
+        {
+          spotlight_full_image_width_ssm: dimensions.width,
+          spotlight_full_image_height_ssm: dimensions.height,
+          Spotlight::Engine.config.thumbnail_field => riiif.image_path(upload_id, size: '!400,400'),
+          Spotlight::Engine.config.iiif_manifest_field => spotlight_routes.manifest_exhibit_solr_document_path(exhibit, compound_id)
+        }
+      end
+
       private
 
       def configured_fields

data/app/models/spotlight/search.rb

@@ -12,12 +12,17 @@ module Spotlight
 
     self.table_name = 'spotlight_searches'
     belongs_to :exhibit
+    has_many :group_memberships, class_name: 'Spotlight::GroupMember', as: :member, dependent: :delete_all
+    has_many :groups, through: :group_memberships
+    accepts_nested_attributes_for :group_memberships
+    accepts_nested_attributes_for :groups
     serialize :query_params, Hash
     default_scope { order('weight ASC') }
     scope :published, -> { where(published: true) }
+    scope :unpublished, -> { where(published: [nil, false]) }
     validates :title, presence: true
 
-    translates :title, :long_description
+    translates :title, :subtitle, :long_description
 
     has_paper_trail
 
@@ -26,6 +31,10 @@ module Spotlight
     accepts_nested_attributes_for :thumbnail, update_only: true, reject_if: proc { |attr| attr['iiif_tilesource'].blank? }
     accepts_nested_attributes_for :masthead, update_only: true, reject_if: proc { |attr| attr['iiif_tilesource'].blank? }
 
+    def full_title
+      [title, subtitle.presence].compact.join(' · ')
+    end
+
     def thumbnail_image_url
       return unless thumbnail&.iiif_url
 

data/app/models/spotlight/solr_document_sidecar.rb

@@ -33,11 +33,7 @@ module Spotlight
 
     # Roll our own polymorphism because our documents are not AREL-able
     def document
-      document_type.new document_type.unique_key => document_id
-    end
-
-    def document_type
-      (super.constantize if defined?(super)) || default_document_type
+      document_type_class.new document_type_class.unique_key => document_id
     end
 
     def default_document_type
@@ -46,6 +42,12 @@ module Spotlight
 
     private
 
+    def document_type_class
+      return default_document_type unless document_type.is_a?(String)
+
+      document_type.constantize
+    end
+
     def visibility_field
       blacklight_config.document_model.visibility_field(exhibit)
     end
@@ -72,7 +74,6 @@ module Spotlight
 
       upload_fields.each_with_object({}) do |field, solr_hash|
         field_name = field.field_name.to_s
-        next unless configured_fields && configured_fields[field_name].present?
 
         value = configured_fields[field_name]
         field_data = field.data_to_solr(convert_stored_value_to_solr(value))
@@ -83,6 +84,8 @@ module Spotlight
     end
 
     def upload_fields
+      return [] unless document.uploaded_resource? || resource.is_a?(Spotlight::Resources::Upload)
+
       Spotlight::Resources::Upload.fields(exhibit)
     end
 

data/app/models/spotlight/temporary_image.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ##
+  # Featured images for browse categories, feature pages, and exhibits
+  class TemporaryImage < Spotlight::FeaturedImage
+  end
+end

data/app/services/spotlight/etl.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # :nodoc:
+  module Etl
+  end
+end

data/app/services/spotlight/etl/context.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Contextual information for the ETL pipeline
+    class Context
+      # A hook for downstream applications to report or handle errors using external
+      # systems or services.
+      class_attribute :error_reporter
+
+      attr_reader :arguments, :additional_metadata, :additional_parameters, :logger
+
+      delegate :document_model, to: :resource
+
+      def initialize(*args, additional_metadata: {}, on_error: :log, logger: Rails.logger, **additional_parameters)
+        @arguments = args
+        @additional_metadata = additional_metadata
+        @additional_parameters = additional_parameters
+        @on_error = on_error
+        @logger = logger
+      end
+
+      # @return [Spotlight::Resource]
+      def resource
+        arguments.first
+      end
+
+      # @return [String]
+      def unique_key(data)
+        data[document_model&.unique_key&.to_sym || :id]
+      end
+
+      ##
+      # This hook receives any exceptions raised by pipeline steps and handles them
+      # appropriately.
+      def on_error(pipeline, exception, data)
+        error_reporter&.call(pipeline, exception, data)
+
+        case @on_error
+        when :log
+          logger.tagged('ETL') do
+            logger.error("Pipeline error processing resource #{resource.id}: #{exception}")
+          end
+        when :exception
+          raise exception
+        else
+          @on_error&.call(pipeline, exception, data)
+        end
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/executor.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # ETL pipeline executor
+    class Executor
+      include ActiveSupport::Benchmarkable
+
+      attr_reader :pipeline, :context, :source, :cache, :logger
+
+      delegate :sources, :pre_processes, :transforms, :post_processes, :loaders, to: :pipeline
+
+      # @param [Spotlight::Etl::Pipeline] pipeline
+      # @param [Spotlight::Etl::Context] context
+      # @param [Hash] cache a shared cache for pipeline steps to store data for the lifetime of the cache
+      def initialize(pipeline, context, cache: nil)
+        @pipeline = pipeline
+        @context = context
+
+        @provided_cache = cache.present?
+        @cache = cache || {}
+        @step_cache = {}
+      end
+
+      ##
+      # Execute the ETL pipeline
+      #
+      # @param [Hash] data the initial data structure to pass through to the transform steps
+      # @yield (optionally..) each transformed document after it is transformed but before
+      #        it is sent to the loaders
+      def call(data: {}, &block)
+        extract.with_index do |source, index|
+          with_source(source, index) do
+            catch :skip do
+              load(transform(data), &block)
+            end
+          rescue StandardError => e
+            on_error(e, data)
+          end
+        end
+
+        after_call
+      end
+
+      ##
+      # Estimate the number of documents that will be produced by the pipeline
+      #
+      # @return [Number]
+      def estimated_size
+        @estimated_size ||= begin
+          compile_steps(sources).sum { |source| source.call(context).count }
+        end
+      end
+
+      ##
+      # Tagged logger for benchmarks and data flow logging.
+      # NOTE: this is super weird to support Rails 5.2
+      # @private
+      # @yield Logger
+      def with_logger
+        logger = (context&.logger || Rails.logger)
+        logger.tagged(pipeline.class) do
+          logger.tagged("#<#{source.class} id=#{source&.id if source.respond_to?(:id)}>") do
+            @logger = logger
+            yield logger
+          end
+        end
+      end
+
+      ##
+      # @private
+      # @param [Hash] data
+      # @return [String] a simplified + truncated version of the data hash for debugging
+      def transform_data_for_debugging(data, verbose: $VERBOSE, truncate: 100)
+        return data.inspect.truncate(truncate) unless data.is_a?(Hash)
+        return "id #{context.unique_key(data) || data&.first(5)&.inspect}" unless verbose
+
+        JSON.fast_generate(data).truncate(truncate)
+      end
+
+      ##
+      # Propagate exceptions up to the context's error handler.
+      def on_error(exception, data)
+        context.on_error(self, exception, data)
+      end
+
+      private
+
+      ##
+      # Set the current source
+      # @param [Object] source
+      # @param [Number] index
+      def with_source(source, index)
+        @source = source
+
+        benchmark "Indexing item #{source.inspect.truncate(50)} in resource #{context.resource.id} (#{index} / #{estimated_size})" do
+          yield.tap { @source = nil }
+        end
+      end
+
+      ##
+      # Extract data from sources. The defined sources receive the provided context
+      # and should return an array or other enumerable of sources to pass through
+      # the pipeline.
+      #
+      # @yield [Object]
+      def extract(&block)
+        return to_enum(:extract) { estimated_size } unless block_given?
+
+        compile_steps(sources).each do |source|
+          source.call(context).each do |data|
+            block.call(data)
+          end
+        end
+      end
+
+      ##
+      # Transform the source to a document.
+      #
+      # @param [Hash] from the initial seed data used as the input to the initial transforms
+      # @return [Hash] the transformed document
+      def transform(from)
+        compile_steps(pre_processes).each { |step| step.call(from, self) }
+
+        data = compile_steps(transforms).inject(from) { |input, step| step.call(input, self) }
+
+        compile_steps(post_processes).each { |step| step.call(data, self) }
+
+        with_logger do |logger|
+          logger.debug do
+            "Transform output: #{transform_data_for_debugging(data, verbose: true, truncate: 1000)}"
+          end
+        end
+
+        data
+      end
+
+      ##
+      # Load a document into a data sink.
+      #
+      # @param [Hash] the fully transformed data
+      # @yield [Hash] the data before it is sent to any loaders
+      def load(data, &block)
+        return unless data
+
+        catch :skip do
+          block&.call(data, self)
+
+          compile_steps(loaders).each do |loader|
+            loader.call(data, self)
+          end
+        end
+      end
+
+      ##
+      # A callback run after transforming data to do any finalizing or cleanup
+      # from the run.
+      def after_call
+        finalize_loaders
+        @cache = {} unless @provided_cache
+        @step_cache = {}
+      end
+
+      ##
+      # Loaders may implement a `#finalize` method if they want to perform any work
+      # after all the data is transformed.
+      def finalize_loaders
+        compile_steps(loaders).each do |step|
+          step.finalize(self) if step.respond_to? :finalize
+        end
+      end
+
+      ##
+      # DSL convenience utility for writing compact lists of steps; this unrolls
+      # pipeline definitions to contain arrays or hashes, e.g.:
+      # `pipeline.transforms = [step_1: lambda {}, step_2: lambda {}]`
+      #
+      # @return [Enumerable<Spotlight::Etl::Step>]
+      def compile_steps(steps)
+        return to_enum(:compile_steps, steps) unless block_given?
+
+        steps.flatten.each do |step|
+          if step.is_a? Hash
+            step.each do |k, v|
+              yield(@step_cache[k] ||= Spotlight::Etl::Step.new(v, label: k, executor: self))
+            end
+          else
+            yield @step_cache[step] ||= Spotlight::Etl::Step.new(step, executor: self)
+          end
+        end
+      end
+    end
+  end
+end