blacklight-spotlight 3.0.0.rc1 → 3.0.0.rc6

data/app/models/spotlight/resources/iiif_harvester.rb

@@ -7,7 +7,6 @@ module Spotlight
     # harvest Images from IIIF Manifest and turn them into a Spotlight::Resource
     # Note: IIIF API : http://iiif.io/api/presentation/2.0
     class IiifHarvester < Spotlight::Resource
-      self.document_builder_class = Spotlight::Resources::IiifBuilder
       self.weight = -5000
 
       validate :valid_url?
@@ -16,6 +15,16 @@ module Spotlight
         @iiif_manifests ||= IiifService.parse(url)
       end
 
+      def self.indexing_pipeline
+        @indexing_pipeline ||= super.dup.tap do |pipeline|
+          pipeline.sources = [Spotlight::Etl::Sources::SourceMethodSource(:iiif_manifests)]
+
+          pipeline.transforms = [
+            ->(data, p) { data.merge(p.source.to_solr(exhibit: p.context.resource.exhibit)) }
+          ] + pipeline.transforms
+        end
+      end
+
       private
 
       def valid_url?
@@ -24,8 +33,8 @@ module Spotlight
 
       def url_is_iiif?(url)
         valid_content_types = ['application/json', 'application/ld+json']
-        req = Faraday.head(url)
-        req = Faraday.get(url) if req.status == 405
+        req = Spotlight::Resources::IiifService.http_client.head(url)
+        req = Spotlight::Resources::IiifService.http_client.get(url) if req.status == 405
         return unless req.success?
 
         valid_content_types.any? do |valid_type|

data/app/models/spotlight/resources/iiif_manifest.rb

@@ -14,7 +14,9 @@ module Spotlight
         @solr_hash = {}
       end
 
-      def to_solr
+      def to_solr(exhibit: nil)
+        @exhibit = exhibit if exhibit
+
         add_document_id
         add_label
         add_thumbnail_url
@@ -69,7 +71,7 @@ module Spotlight
       end
 
       def add_label
-        return unless title_fields.present? && manifest.try(:label)
+        return unless title_fields.present? && manifest&.label
 
         Array.wrap(title_fields).each do |field|
           solr_hash[field] = metadata_class.new(manifest).label
@@ -144,7 +146,7 @@ module Spotlight
       end
 
       def thumbnail_field
-        blacklight_config.index.try(:thumbnail_field)
+        blacklight_config.index.thumbnail_field
       end
 
       def full_image_field
@@ -152,11 +154,11 @@ module Spotlight
       end
 
       def tile_source_field
-        blacklight_config.show.try(:tile_source_field)
+        blacklight_config.show.tile_source_field
       end
 
       def title_fields
-        Spotlight::Engine.config.iiif_title_fields || blacklight_config.index.try(:title_field)
+        Spotlight::Engine.config.iiif_title_fields || blacklight_config.index&.title_field
       end
 
       def sidecar
@@ -187,7 +189,7 @@ module Spotlight
         end
 
         def label
-          return unless manifest.try(:label)
+          return unless manifest&.label
 
           Array(json_ld_value(manifest.label)).map { |v| html_sanitize(v) }.first
         end
@@ -197,7 +199,7 @@ module Spotlight
         attr_reader :manifest
 
         def metadata
-          manifest.try(:metadata) || []
+          manifest&.metadata || []
         end
 
         def metadata_hash

data/app/models/spotlight/resources/iiif_service.rb

@@ -29,6 +29,13 @@ module Spotlight
         recursive_manifests(new(url))
       end
 
+      def self.http_client
+        Faraday.new do |b|
+          b.use FaradayMiddleware::FollowRedirects
+          b.adapter Faraday.default_adapter
+        end
+      end
+
       protected
 
       def object
@@ -41,8 +48,8 @@ module Spotlight
 
       class << self
         def iiif_response(url)
-          Faraday.get(url).body
-        rescue Faraday::Error::ConnectionFailed, Faraday::TimeoutError => e
+          Spotlight::Resources::IiifService.http_client.get(url).body
+        rescue Faraday::ConnectionFailed, Faraday::TimeoutError => e
           Rails.logger.warn("HTTP GET for #{url} failed with #{e}")
           {}.to_json
         end

data/app/models/spotlight/resources/json_upload.rb

@@ -5,6 +5,18 @@ module Spotlight
     # Raw solr document uploads
     class JsonUpload < Spotlight::Resource
       store :data, accessors: :json
+
+      # The indexing pipeline for JSON uploads copies the data from the stored
+      # `#data` field directly into the indexed document.
+      def self.indexing_pipeline
+        @indexing_pipeline ||= super.dup.tap do |pipeline|
+          pipeline.sources = [Spotlight::Etl::Sources::StoredData]
+
+          pipeline.transforms = [
+            Spotlight::Etl::Transforms::IdentityTransform
+          ] + pipeline.transforms
+        end
+      end
     end
   end
 end

data/app/models/spotlight/resources/upload.rb

@@ -10,8 +10,6 @@ module Spotlight
       # we want to do this before reindexing
       after_create :update_document_sidecar
 
-      self.document_builder_class = UploadSolrDocumentBuilder
-
       def self.fields(exhibit)
         @fields ||= {}
         @fields[exhibit] ||= begin
@@ -25,6 +23,15 @@ module Spotlight
         end
       end
 
+      def self.indexing_pipeline
+        @indexing_pipeline ||= super.dup.tap do |pipeline|
+          pipeline.transforms = [
+            ->(data, p) { data.merge({ p.context.document_model.unique_key.to_sym => p.source.compound_id }) },
+            Spotlight::Etl::Transforms::SourceMethodTransform(:to_solr)
+          ] + pipeline.transforms
+        end
+      end
+
       def compound_id
         "#{exhibit_id}-#{id}"
       end
@@ -33,6 +40,22 @@ module Spotlight
         @sidecar ||= document_model.new(id: compound_id).sidecar(exhibit)
       end
 
+      def to_solr
+        return {} unless upload&.file_present?
+
+        spotlight_routes = Spotlight::Engine.routes.url_helpers
+        riiif = Riiif::Engine.routes.url_helpers
+
+        dimensions = Riiif::Image.new(upload_id).info
+
+        {
+          spotlight_full_image_width_ssm: dimensions.width,
+          spotlight_full_image_height_ssm: dimensions.height,
+          Spotlight::Engine.config.thumbnail_field => riiif.image_path(upload, size: '!400,400'),
+          Spotlight::Engine.config.iiif_manifest_field => spotlight_routes.manifest_exhibit_solr_document_path(exhibit, compound_id)
+        }
+      end
+
       private
 
       def configured_fields

data/app/models/spotlight/role.rb

@@ -4,11 +4,10 @@ module Spotlight
   ##
   # Exhibit authorization roles
   class Role < ActiveRecord::Base
-    ROLES = %w[admin curator].freeze
     belongs_to :resource, polymorphic: true, optional: true
     belongs_to :user, class_name: Spotlight::Engine.config.user_class, autosave: true, optional: false
 
-    validates :role, inclusion: { in: ROLES }
+    validates :role, inclusion: { in: Spotlight::Engine.config.exhibit_roles }
     validate :user_must_be_unique, if: :user
 
     def user_key

data/app/models/spotlight/search.rb

@@ -12,9 +12,14 @@ module Spotlight
 
     self.table_name = 'spotlight_searches'
     belongs_to :exhibit
+    has_many :group_memberships, class_name: 'Spotlight::GroupMember', as: :member, dependent: :delete_all
+    has_many :groups, through: :group_memberships
+    accepts_nested_attributes_for :group_memberships
+    accepts_nested_attributes_for :groups
     serialize :query_params, Hash
     default_scope { order('weight ASC') }
     scope :published, -> { where(published: true) }
+    scope :unpublished, -> { where(published: [nil, false]) }
     validates :title, presence: true
 
     translates :title, :subtitle, :long_description

data/app/models/spotlight/solr_document_sidecar.rb

@@ -74,7 +74,6 @@ module Spotlight
 
       upload_fields.each_with_object({}) do |field, solr_hash|
         field_name = field.field_name.to_s
-        next unless configured_fields && configured_fields[field_name].present?
 
         value = configured_fields[field_name]
         field_data = field.data_to_solr(convert_stored_value_to_solr(value))
@@ -85,6 +84,8 @@ module Spotlight
     end
 
     def upload_fields
+      return [] unless document.uploaded_resource? || resource.is_a?(Spotlight::Resources::Upload)
+
       Spotlight::Resources::Upload.fields(exhibit)
     end
 

data/app/presenters/spotlight/iiif_manifest_presenter.rb

@@ -70,7 +70,7 @@ module Spotlight
     private
 
     def presenter
-      controller.view_context.show_presenter(resource)
+      controller.view_context.document_presenter(resource)
     end
 
     def endpoint
@@ -79,7 +79,7 @@ module Spotlight
 
     def iiif_url
       # yes this is hacky, and we are appropriately ashamed.
-      controller.riiif.info_url(uploaded_resource.upload.id).sub(%r{/info\.json\Z}, '')
+      controller.riiif.info_url(uploaded_resource.upload).sub(%r{/info\.json\Z}, '')
     end
   end
 end

data/app/services/spotlight/bulk_updates_csv_template_service.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Spotlight
+  # A service to generate a CSV template suitable for re-uploading for bulk updates
+  class BulkUpdatesCsvTemplateService
+    attr_reader :exhibit
+
+    def initialize(exhibit:)
+      @exhibit = exhibit
+    end
+
+    def template(view_context:, title: true, tags: true, visibility: true)
+      return to_enum(:template, view_context: view_context, title: title, tags: tags, visibility: visibility) unless block_given?
+
+      yield ::CSV.generate_line(csv_headers(title: title, tags: tags, visibility: visibility))
+      each_document do |document|
+        sidecar = document.sidecar(exhibit)
+        yield ::CSV.generate_line([
+          document.id,
+          (title_column(view_context, document) if title),
+          (visibility_column(sidecar) if visibility),
+          (tags_column(sidecar) if tags)
+        ].flatten.compact)
+      end
+    end
+
+    private
+
+    def title_column(view_context, document)
+      CGI.unescapeHTML(view_context.document_presenter(document).heading)
+    end
+
+    def visibility_column(sidecar)
+      sidecar.public ? 'TRUE' : ' '
+    end
+
+    def tags_column(sidecar)
+      exhibit_tags.map do |tag|
+        sidecar.all_tags_list.include?(tag) ? 'TRUE' : ' '
+      end
+    end
+
+    def exhibit_tags
+      @exhibit_tags ||= exhibit.owned_tags.map(&:name)
+    end
+
+    def csv_headers(title:, tags:, visibility:)
+      headers = [bulk_updates_config.csv_id]
+      headers.append(bulk_updates_config.csv_title) if title
+      headers.append(bulk_updates_config.csv_visibility) if visibility
+      if tags
+        exhibit_tags.each do |tag|
+          headers.append(format(bulk_updates_config.csv_tags, tag))
+        end
+      end
+      headers
+    end
+
+    def bulk_updates_config
+      Spotlight::Engine.config.bulk_updates
+    end
+
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def each_document(&block)
+      return to_enum(:each_document) unless block_given?
+
+      cursor_mark = nil
+      next_cursor_mark = '*'
+
+      solr_params = exhibit&.exhibit_search_builder&.to_h || {}
+
+      until next_cursor_mark == cursor_mark || next_cursor_mark.nil?
+        cursor_mark = next_cursor_mark
+        response = exhibit.blacklight_config.repository.search(
+          solr_params.merge(
+            'q' => '*',
+            'rows' => Spotlight::Engine.config.bulk_actions_batch_size,
+            'cursorMark' => cursor_mark,
+            'sort' => "#{exhibit.blacklight_config.document_model.unique_key} asc"
+          )
+        )
+        response.documents.each do |document|
+          block.call(document)
+        end
+
+        next_cursor_mark = response['nextCursorMark']
+      end
+    end
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+  end
+end

data/app/services/spotlight/etl.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # :nodoc:
+  module Etl
+  end
+end

data/app/services/spotlight/etl/context.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Contextual information for the ETL pipeline
+    class Context
+      # A hook for downstream applications to report or handle errors using external
+      # systems or services.
+      class_attribute :error_reporter
+
+      attr_reader :arguments, :additional_metadata, :additional_parameters, :logger
+
+      delegate :document_model, to: :resource
+
+      def initialize(*args, additional_metadata: {}, on_error: :log, logger: Rails.logger, **additional_parameters)
+        @arguments = args
+        @additional_metadata = additional_metadata
+        @additional_parameters = additional_parameters
+        @on_error = on_error
+        @logger = logger
+      end
+
+      # @return [Spotlight::Resource]
+      def resource
+        arguments.first
+      end
+
+      # @return [String]
+      def unique_key(data)
+        data[document_model&.unique_key&.to_sym || :id]
+      end
+
+      ##
+      # This hook receives any exceptions raised by pipeline steps and handles them
+      # appropriately.
+      def on_error(pipeline, exception, data)
+        error_reporter&.call(pipeline, exception, data)
+
+        case @on_error
+        when :log
+          logger.tagged('ETL') do
+            logger.error("Pipeline error processing resource #{resource.id}: #{exception}")
+          end
+        when :exception
+          raise exception
+        else
+          @on_error&.call(pipeline, exception, data)
+        end
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/executor.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # ETL pipeline executor
+    class Executor
+      include ActiveSupport::Benchmarkable
+
+      attr_reader :pipeline, :context, :source, :cache, :logger
+
+      delegate :sources, :pre_processes, :transforms, :post_processes, :loaders, to: :pipeline
+
+      # @param [Spotlight::Etl::Pipeline] pipeline
+      # @param [Spotlight::Etl::Context] context
+      # @param [Hash] cache a shared cache for pipeline steps to store data for the lifetime of the cache
+      def initialize(pipeline, context, cache: nil)
+        @pipeline = pipeline
+        @context = context
+
+        @provided_cache = cache.present?
+        @cache = cache || {}
+        @step_cache = {}
+      end
+
+      ##
+      # Execute the ETL pipeline
+      #
+      # @param [Hash] data the initial data structure to pass through to the transform steps
+      # @yield (optionally..) each transformed document after it is transformed but before
+      #        it is sent to the loaders
+      def call(data: {}, &block)
+        extract.with_index do |source, index|
+          with_source(source, index) do
+            catch :skip do
+              load(transform(data), &block)
+            end
+          rescue StandardError => e
+            on_error(e, data)
+          end
+        end
+
+        after_call
+      end
+
+      ##
+      # Estimate the number of documents that will be produced by the pipeline
+      #
+      # @return [Number]
+      def estimated_size
+        @estimated_size ||= compile_steps(sources).sum { |source| source.call(context).count }
+      end
+
+      ##
+      # Tagged logger for benchmarks and data flow logging.
+      # NOTE: this is super weird to support Rails 5.2
+      # @private
+      # @yield Logger
+      def with_logger
+        logger = (context&.logger || Rails.logger)
+        logger.tagged(pipeline.class) do
+          logger.tagged("#<#{source.class} id=#{source&.id if source.respond_to?(:id)}>") do
+            @logger = logger
+            yield logger
+          end
+        end
+      end
+
+      ##
+      # @private
+      # @param [Hash] data
+      # @return [String] a simplified + truncated version of the data hash for debugging
+      def transform_data_for_debugging(data, verbose: $VERBOSE, truncate: 100)
+        return data.inspect.truncate(truncate) unless data.is_a?(Hash)
+        return "id #{context.unique_key(data) || data&.first(5)&.inspect}" unless verbose
+
+        JSON.fast_generate(data).truncate(truncate)
+      end
+
+      ##
+      # Propagate exceptions up to the context's error handler.
+      def on_error(exception, data)
+        context.on_error(self, exception, data)
+      end
+
+      private
+
+      ##
+      # Set the current source
+      # @param [Object] source
+      # @param [Number] index
+      def with_source(source, index)
+        @source = source
+
+        benchmark "Indexing item #{source.inspect.truncate(50)} in resource #{context.resource.id} (#{index} / #{estimated_size})" do
+          yield.tap { @source = nil }
+        end
+      end
+
+      ##
+      # Extract data from sources. The defined sources receive the provided context
+      # and should return an array or other enumerable of sources to pass through
+      # the pipeline.
+      #
+      # @yield [Object]
+      def extract(&block)
+        return to_enum(:extract) { estimated_size } unless block_given?
+
+        compile_steps(sources).each do |source|
+          source.call(context).each do |data|
+            block.call(data)
+          end
+        end
+      end
+
+      ##
+      # Transform the source to a document.
+      #
+      # @param [Hash] from the initial seed data used as the input to the initial transforms
+      # @return [Hash] the transformed document
+      def transform(from)
+        compile_steps(pre_processes).each { |step| step.call(from, self) }
+
+        data = compile_steps(transforms).inject(from) { |input, step| step.call(input, self) }
+
+        compile_steps(post_processes).each { |step| step.call(data, self) }
+
+        with_logger do |logger|
+          logger.debug do
+            "Transform output: #{transform_data_for_debugging(data, verbose: true, truncate: 1000)}"
+          end
+        end
+
+        data
+      end
+
+      ##
+      # Load a document into a data sink.
+      #
+      # @param [Hash] the fully transformed data
+      # @yield [Hash] the data before it is sent to any loaders
+      def load(data, &block)
+        return unless data
+
+        catch :skip do
+          block&.call(data, self)
+
+          compile_steps(loaders).each do |loader|
+            loader.call(data, self)
+          end
+        end
+      end
+
+      ##
+      # A callback run after transforming data to do any finalizing or cleanup
+      # from the run.
+      def after_call
+        finalize_loaders
+        @cache = {} unless @provided_cache
+        @step_cache = {}
+      end
+
+      ##
+      # Loaders may implement a `#finalize` method if they want to perform any work
+      # after all the data is transformed.
+      def finalize_loaders
+        compile_steps(loaders).each do |step|
+          step.finalize(self) if step.respond_to? :finalize
+        end
+      end
+
+      ##
+      # DSL convenience utility for writing compact lists of steps; this unrolls
+      # pipeline definitions to contain arrays or hashes, e.g.:
+      # `pipeline.transforms = [step_1: lambda {}, step_2: lambda {}]`
+      #
+      # @return [Enumerable<Spotlight::Etl::Step>]
+      def compile_steps(steps)
+        return to_enum(:compile_steps, steps) unless block_given?
+
+        steps.flatten.each do |step|
+          if step.is_a? Hash
+            step.each do |k, v|
+              yield(@step_cache[k] ||= Spotlight::Etl::Step.new(v, label: k, executor: self))
+            end
+          else
+            yield @step_cache[step] ||= Spotlight::Etl::Step.new(step, executor: self)
+          end
+        end
+      end
+    end
+  end
+end