blacklight-spotlight 3.0.0.alpha.10 → 3.0.0.rc5

data/app/services/spotlight/etl/loaders.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    module Loaders
+      # A loader that just prints the data to $stderr for debugging.
+      WarnLoader = lambda do |data, _context|
+        warn(JSON.pretty_generate(data))
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/pipeline.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # ETL pipeline definition
+    class Pipeline
+      include ActiveSupport::Benchmarkable
+
+      attr_reader :context, :source
+
+      # This ETL pipeline system, while somewhat generic, was implemented for Spotlight
+      # to transform Spotlight::Resource instances into Solr documents. The resources
+      # go through a series of steps (sources, transforms, loaders) to produce one or
+      # more documents in the Solr index.
+      #
+      # All of the steps below can be provided as:
+      #  - a lambda
+      #  - a ruby class (which will be initialized for each pipeline execution)
+      #  - or, a hash (of any length) with:
+      #       - a key (used only for clarity in logging, particularly useful to label lambdas)
+      #       - a value that is one of the valid step types (lambda or ruby class).
+      #
+      # Any of the transform or loader steps can `throw :skip` to skip the current source.
+      #
+      # Any exceptions raised by the pipeline's steps are sent to the context's
+      # error handler by calling `#on_error` on the context object.
+
+      # sources return enumerables that convert from the Spotlight::Etl::Context
+      # to some data structure that the transform steps can handle. The Context is provided
+      # by the implementation when the pipeline is executed.
+      class_attribute :sources, default: []
+
+      # The transform steps (pre-processes, transforms, and post-processes) receive
+      # the current data state and the pipeline. The return value from the transforms
+      # steps replaces the current data state, however the return values for pre- and
+      # post- processing is ignored (although they may mutate the provided data, pipeline, etc).
+      #
+      # Through the pipeline argument, the transform steps can access:
+      #  - `context`, the implementation-provided resource
+      #  - `source`, the current source instance
+      class_attribute :pre_processes, default: []
+      class_attribute :transforms, default: []
+      class_attribute :post_processes, default: []
+
+      # loaders receive the transformed data and.. do something with it (like load it into Solr)
+      # After all documents are transformed, the loader may also receive `#finalize` to finish any
+      # additional processing.
+      class_attribute :loaders, default: []
+
+      def initialize
+        yield(self) if block_given?
+      end
+
+      ##
+      # Execute the ETL pipeline
+      #
+      # @param [Spotlight::Etl::Context] context
+      # @param [Hash] data the initial data structure to pass through to the transform steps
+      # @yield (optioanlly..) each transformed document after it is transformed but before
+      #        it is sent to the loaders
+      def call(context, data: {}, cache: nil, &block)
+        executor(context, cache: cache).call(data: data, &block)
+      end
+
+      ##
+      # Estimate the number of documents that will be produced by the pipeline
+      #
+      # @param [Spotlight::Etl::Context] context
+      # @return [Number]
+      def estimated_size(context)
+        executor(context).estimated_size
+      end
+
+      private
+
+      def executor(context, **args)
+        Spotlight::Etl::Executor.new(self, context, **args)
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/solr_loader.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Solr data loader with a built-in buffer to combine document updates into batches
+    class SolrLoader
+      attr_reader :queue, :batch_size
+
+      delegate :size, to: :queue
+
+      def initialize(batch_size: Spotlight::Engine.config.solr_batch_size, solr_connection: nil)
+        @queue = Queue.new
+        @batch_size = batch_size
+        @blacklight_solr = solr_connection
+      end
+
+      def call(data, pipeline = nil)
+        @queue << data
+
+        write_to_index(pipeline) if @queue.size >= @batch_size
+      end
+
+      def finalize(pipeline = nil)
+        write_to_index(pipeline)
+
+        commit! if pipeline.nil? || pipeline.context.additional_parameters[:commit]
+      end
+
+      private
+
+      def write_to_index(pipeline)
+        batch = drain_queue
+
+        return unless write? && batch.any?
+
+        send_batch(batch, pipeline)
+      end
+
+      def send_batch(documents, pipeline)
+        blacklight_solr.update params: { commitWithin: 500 },
+                               data: documents.to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+      rescue StandardError => e
+        logger.warn "Error sending a batch of documents to solr: #{e}"
+
+        documents.each do |doc|
+          send_one(doc, pipeline)
+        end
+      end
+
+      def send_one(document, pipeline)
+        blacklight_solr.update params: { commitWithin: 500 },
+                               data: [document].to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+      rescue StandardError => e
+        pipeline&.on_error(self, e, document.to_json)
+      end
+
+      def blacklight_solr
+        @blacklight_solr ||= RSolr.connect(connection_config.merge(adapter: connection_config[:http_adapter]))
+      end
+
+      def connection_config
+        Blacklight.connection_config
+      end
+
+      def drain_queue
+        arr = []
+
+        begin
+          arr << @queue.deq(true) while arr.length < @batch_size && !@queue.empty?
+        rescue ThreadError
+          # @queue throws a ThreadError if it is empty...
+        end
+
+        arr
+      end
+
+      def commit!
+        return unless write?
+
+        blacklight_solr.commit
+      rescue StandardError => e
+        logger.warn "Unable to commit to solr: #{e}"
+      end
+
+      def write?
+        Spotlight::Engine.config.writable_index
+      end
+
+      def logger
+        Rails.logger
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/sources.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Basic ETL source implementations
+    module Sources
+      # A simple source that just returns the original resource(s)
+      IdentitySource = lambda do |context|
+        Array.wrap(context.resource)
+      end
+
+      # A transform step that calls a method on the resource to generate a source
+      def self.SourceMethodSource(method) # rubocop:disable Naming/MethodName
+        lambda do |context|
+          context.resource.public_send(method)
+        end
+      end
+
+      # A simple source that retrieves the stored data from a Spotlight::Resource
+      StoredData = lambda do |context, **|
+        Array.wrap(context.resource.data)
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/step.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # ETL pipeline step
+    class Step
+      attr_reader :definition, :context, :executor
+
+      # @param [Class, Proc] definition the step to run
+      # @param [String] label
+      # @param [Spotlight::Etl::Executor] executor the execution environment
+      def initialize(definition, label: nil, executor: nil)
+        @definition = definition
+        @executor = executor
+        @label = label
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def call(*args)
+        with_logger do |logger|
+          logger.debug { "Called with #{transform_data_for_debugging(args.first)}" }
+
+          catch :skip do
+            return action.call(*args).tap do |result|
+              logger.debug { "   => Returning #{transform_data_for_debugging(result)}" } if $VERBOSE
+            end
+          end
+
+          logger.debug '  => Caught skip.'
+          throw :skip
+        end
+      rescue StandardError => e
+        with_logger do |logger|
+          logger.error("Caught exception #{e}")
+        end
+        raise(e)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def finalize(*args)
+        action.finalize(*args) if action.respond_to? :finalize
+      end
+
+      private
+
+      # @return [#call]
+      def action
+        case definition
+        when Class
+          # memoize the class' instance for the lifetime of the step
+          @memoized_action ||= definition.new
+        else # Proc, etc
+          definition
+        end
+      end
+
+      # @return [#to_string]
+      def label
+        @label || definition
+      end
+
+      # NOTE: this is super weird to support Rails 5.2
+      # @return [Logger]
+      def with_logger
+        yield(Rails.logger) && return unless executor
+
+        executor.with_logger do |logger|
+          logger.tagged(label) do
+            yield logger
+          end
+        end
+      end
+
+      ##
+      # @param [Hash] data
+      # @return [String] a simplified + truncated version of the data hash for debugging
+      def transform_data_for_debugging(data)
+        executor&.transform_data_for_debugging(data) || data.inspect.truncate(100)
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/transforms.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Basic + default transform steps
+    module Transforms
+      # A transform step that "transforms" the source into the data element
+      IdentityTransform = lambda do |data, pipeline|
+        data.merge(pipeline.source)
+      end
+
+      # A transform step that calls a method on the source to generate a document
+      def self.SourceMethodTransform(method) # rubocop:disable Naming/MethodName
+        lambda do |data, pipeline|
+          data.merge(pipeline.source.public_send(method))
+        end
+      end
+
+      # A transform step that throws away blank data
+      RejectBlank = lambda do |data, _|
+        throw :skip if data.blank?
+        data
+      end
+
+      # A trasnform step that ensures data has a unique key attribute
+      RejectMissingUniqueId = lambda do |data, pipeline|
+        id = pipeline.context.unique_key(data)
+
+        throw :skip if id.blank?
+        data
+      end
+
+      # A transform that adds exhibit-specific metadata (like Spotlight sidecar data)
+      # to the document
+      ApplyExhibitMetadata = lambda do |data, pipeline|
+        resource = pipeline.context.resource
+        document_model = pipeline.context.document_model
+        id = pipeline.context.unique_key(data)
+
+        next data unless document_model && id.present?
+
+        exhibit_metadata = document_model.build_for_exhibit(id, resource.exhibit, resource: (resource if resource.persisted?))
+
+        data.reverse_merge(exhibit_metadata.to_solr)
+      end
+
+      # A transform that adds application-specific metadata (like what resource generated the solr document)
+      ApplyApplicationMetadata = lambda do |data, pipeline|
+        resource = pipeline.context.resource
+        document_model = pipeline.context.document_model
+
+        data.reverse_merge(
+          Spotlight::Engine.config.resource_global_id_field => (resource.to_global_id.to_s if resource.persisted?),
+          document_model.resource_type_field => resource.class.to_s.tableize
+        )
+      end
+
+      # A transform that adds externally-provided metadata to the document
+      ApplyPipelineMetadata = lambda do |data, pipeline|
+        data.reverse_merge(pipeline.context.additional_metadata)
+      end
+    end
+  end
+end

data/app/services/spotlight/exhibit_import_export_service.rb

@@ -11,6 +11,7 @@ module Spotlight
     class_attribute :serialization_pipeline, default: %i[
       raw_json
       add_feature_page_hierarchy
+      add_browse_group_hierarchy
       add_page_content
       attach_featured_images
       attach_attachments
@@ -37,34 +38,43 @@ module Spotlight
         attachments: {},
         languages: {},
         translations: {},
-        owned_taggings: {}
+        owned_taggings: {},
+        groups: {}
       )
 
       exhibit_attributes = hash.reject { |_k, v| v.is_a?(Array) || v.is_a?(Hash) }
-      exhibit.update_attributes(exhibit_attributes.except(:theme))
+      exhibit.update(exhibit_attributes.except(:theme))
       exhibit.theme = exhibit_attributes[:theme] if exhibit.themes.include? exhibit_attributes[:theme]
 
       deserialize_featured_image(exhibit, :masthead, hash[:masthead]) if hash[:masthead]
       deserialize_featured_image(exhibit, :thumbnail, hash[:thumbnail]) if hash[:thumbnail]
 
-      exhibit.blacklight_configuration.update_attributes hash[:blacklight_configuration].deep_stringify_keys if hash[:blacklight_configuration]
+      exhibit.blacklight_configuration.update hash[:blacklight_configuration].deep_stringify_keys if hash[:blacklight_configuration]
 
       hash[:main_navigations].each do |attr|
         ar = exhibit.main_navigations.find_or_initialize_by(nav_type: attr[:nav_type])
-        ar.update_attributes(attr)
+        ar.update(attr)
       end
 
       hash[:contact_emails].each do |attr|
         ar = exhibit.contact_emails.find_or_initialize_by(email: attr[:email])
-        ar.update_attributes(attr)
+        ar.update(attr)
+      end
+
+      hash[:groups].each do |attr|
+        gr = exhibit.groups.find_or_initialize_by(slug: attr[:slug])
+        gr.update(attr)
       end
 
       hash[:searches].each do |attr|
+        group_slugs = attr.delete(:group_slugs) || []
         masthead = attr.delete(:masthead)
         thumbnail = attr.delete(:thumbnail)
 
         ar = exhibit.searches.find_or_initialize_by(slug: attr[:slug])
-        ar.update_attributes(attr)
+        ar.update(attr)
+
+        ar.update(groups: exhibit.groups.select { |x| group_slugs.include? x.slug })
 
         deserialize_featured_image(ar, :masthead, masthead) if masthead
         deserialize_featured_image(ar, :thumbnail, thumbnail) if thumbnail
@@ -76,7 +86,7 @@ module Spotlight
         translated_pages = attr.delete(:translated_pages) || []
 
         ar = exhibit.about_pages.find_or_initialize_by(slug: attr[:slug])
-        ar.update_attributes(attr)
+        ar.update(attr)
 
         deserialize_featured_image(ar, :masthead, masthead) if masthead
         deserialize_featured_image(ar, :thumbnail, thumbnail) if thumbnail
@@ -86,7 +96,7 @@ module Spotlight
           thumbnail = tattr.delete(:thumbnail)
 
           tar = ar.translated_page_for(tattr[:locale]) || ar.clone_for_locale(tattr[:locale])
-          tar.update_attributes(tattr)
+          tar.update(tattr)
 
           deserialize_featured_image(ar, :masthead, masthead) if masthead
           deserialize_featured_image(ar, :thumbnail, thumbnail) if thumbnail
@@ -98,7 +108,7 @@ module Spotlight
         thumbnail = attr.delete(:thumbnail)
 
         ar = exhibit.feature_pages.find_or_initialize_by(slug: attr[:slug])
-        ar.update_attributes(attr.except(:parent_page_slug, :translated_pages))
+        ar.update(attr.except(:parent_page_slug, :translated_pages))
 
         deserialize_featured_image(ar, :masthead, masthead) if masthead
         deserialize_featured_image(ar, :thumbnail, thumbnail) if thumbnail
@@ -119,7 +129,7 @@ module Spotlight
           thumbnail = tattr.delete(:thumbnail)
 
           tar = ar.translated_page_for(tattr[:locale]) || ar.clone_for_locale(tattr[:locale])
-          tar.update_attributes(tattr)
+          tar.update(tattr)
 
           deserialize_featured_image(ar, :masthead, masthead) if masthead
           deserialize_featured_image(ar, :thumbnail, thumbnail) if thumbnail
@@ -128,7 +138,7 @@ module Spotlight
 
       if hash[:home_page]
         translated_pages = hash[:home_page].delete(:translated_pages) || []
-        exhibit.home_page.update_attributes(hash[:home_page].except(:thumbnail))
+        exhibit.home_page.update(hash[:home_page].except(:thumbnail))
         deserialize_featured_image(exhibit.home_page, :thumbnail, hash[:home_page][:thumbnail]) if hash[:home_page][:thumbnail]
 
         translated_pages.each do |tattr|
@@ -136,7 +146,7 @@ module Spotlight
           thumbnail = tattr.delete(:thumbnail)
 
           tar = exhibit.home_page.translated_page_for(tattr[:locale]) || exhibit.home_page.clone_for_locale(tattr[:locale])
-          tar.update_attributes(tattr)
+          tar.update(tattr)
 
           deserialize_featured_image(ar, :masthead, masthead) if masthead
           deserialize_featured_image(ar, :thumbnail, thumbnail) if thumbnail
@@ -147,26 +157,26 @@ module Spotlight
         avatar = attr.delete(:avatar)
 
         ar = exhibit.contacts.find_or_initialize_by(slug: attr[:slug])
-        ar.update_attributes(attr)
+        ar.update(attr)
 
         deserialize_featured_image(ar, :avatar, avatar) if avatar
       end
 
       hash[:custom_fields].each do |attr|
         ar = exhibit.custom_fields.find_or_initialize_by(slug: attr[:slug])
-        ar.update_attributes(attr)
+        ar.update(attr)
       end
 
       hash[:solr_document_sidecars].each do |attr|
         ar = exhibit.solr_document_sidecars.find_or_initialize_by(document_id: attr[:document_id])
-        ar.update_attributes(attr)
+        ar.update(attr)
       end
 
       hash[:resources].each do |attr|
         upload = attr.delete(:upload)
 
         ar = exhibit.resources.find_or_initialize_by(type: attr[:type], url: attr[:url])
-        ar.update_attributes(attr)
+        ar.update(attr)
 
         deserialize_featured_image(ar, :upload, upload) if upload
       end
@@ -183,12 +193,12 @@ module Spotlight
 
       hash[:languages].each do |attr|
         ar = exhibit.languages.find_or_initialize_by(locale: attr[:locale])
-        ar.update_attributes(attr)
+        ar.update(attr)
       end
 
       hash[:translations].each do |attr|
         ar = exhibit.translations.find_or_initialize_by(locale: attr[:locale], key: attr[:key])
-        ar.update_attributes(attr)
+        ar.update(attr)
       end
 
       hash[:owned_taggings].each do |attr|
@@ -200,18 +210,21 @@ module Spotlight
     def deserialize_featured_image(obj, method, data)
       file = data.delete(:image)
       image = obj.public_send("build_#{method}")
-      image.update_attributes(data)
+      image.update(data)
       if file
         image.image = CarrierWave::SanitizedFile.new tempfile: StringIO.new(Base64.decode64(file[:content])),
                                                      filename: file[:filename],
                                                      content_type: file[:content_type]
+        # Unset the iiif_tilesource field as the new image should be different, because
+        # the source has been reloaded
+        image.iiif_tilesource = nil
       end
       image.save!
       obj.update(method => image)
     end
 
-    def as_json(**args)
-      self.class.serialization_pipeline.inject(args) do |memo, step|
+    def as_json(*_args)
+      self.class.serialization_pipeline.inject({}) do |memo, step|
         method(step).call(memo)
       end
     end
@@ -292,7 +305,7 @@ module Spotlight
       if file
         img = {
           image: {
-            filename: file.identifier, content_type: file.content_type, content: Base64.encode64(file.read)
+            filename: file.filename, content_type: file.content_type, content: Base64.encode64(file.read)
           }
         }
       end
@@ -335,6 +348,18 @@ module Spotlight
       json
     end
 
+    def add_browse_group_hierarchy(json)
+      return json unless json[:groups] && json[:searches]
+
+      json[:searches].each do |attr|
+        search = exhibit.searches.find_by(slug: attr[:slug])
+
+        attr[:group_slugs] = search.groups.pluck(:slug)
+      end
+
+      json
+    end
+
     def add_page_content(json)
       (json[:feature_pages] || []).each do |page|
         p = exhibit.feature_pages.find_by(slug: page[:slug])
@@ -382,6 +407,9 @@ module Spotlight
                         searches: { # thumbnail
                           except: %i[id scope exhibit_id]
                         },
+                        groups: {
+                          except: %i[id exhibit_id]
+                        },
                         about_pages: { # thumbnail
                           except: %i[id scope exhibit_id parent_page_id content],
                           include: {