blacklight-spotlight 3.0.0.rc2 → 3.0.0

data/app/services/spotlight/bulk_updates_csv_template_service.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Spotlight
+  # A service to generate a CSV template suitable for re-uploading for bulk updates
+  class BulkUpdatesCsvTemplateService
+    attr_reader :exhibit
+
+    def initialize(exhibit:)
+      @exhibit = exhibit
+    end
+
+    def template(view_context:, title: true, tags: true, visibility: true)
+      return to_enum(:template, view_context: view_context, title: title, tags: tags, visibility: visibility) unless block_given?
+
+      yield ::CSV.generate_line(csv_headers(title: title, tags: tags, visibility: visibility))
+      each_document do |document|
+        sidecar = document.sidecar(exhibit)
+        yield ::CSV.generate_line([
+          document.id,
+          (title_column(view_context, document) if title),
+          (visibility_column(sidecar) if visibility),
+          (tags_column(sidecar) if tags)
+        ].flatten.compact)
+      end
+    end
+
+    private
+
+    def title_column(view_context, document)
+      CGI.unescapeHTML(view_context.document_presenter(document).heading)
+    end
+
+    def visibility_column(sidecar)
+      sidecar.public ? 'TRUE' : ' '
+    end
+
+    def tags_column(sidecar)
+      exhibit_tags.map do |tag|
+        sidecar.all_tags_list.include?(tag) ? 'TRUE' : ' '
+      end
+    end
+
+    def exhibit_tags
+      @exhibit_tags ||= exhibit.owned_tags.map(&:name)
+    end
+
+    def csv_headers(title:, tags:, visibility:)
+      headers = [bulk_updates_config.csv_id]
+      headers.append(bulk_updates_config.csv_title) if title
+      headers.append(bulk_updates_config.csv_visibility) if visibility
+      if tags
+        exhibit_tags.each do |tag|
+          headers.append(format(bulk_updates_config.csv_tags, tag))
+        end
+      end
+      headers
+    end
+
+    def bulk_updates_config
+      Spotlight::Engine.config.bulk_updates
+    end
+
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def each_document(&block)
+      return to_enum(:each_document) unless block_given?
+
+      cursor_mark = nil
+      next_cursor_mark = '*'
+
+      solr_params = exhibit&.exhibit_search_builder&.to_h || {}
+
+      until next_cursor_mark == cursor_mark || next_cursor_mark.nil?
+        cursor_mark = next_cursor_mark
+        response = exhibit.blacklight_config.repository.search(
+          solr_params.merge(
+            'q' => '*',
+            'rows' => Spotlight::Engine.config.bulk_actions_batch_size,
+            'cursorMark' => cursor_mark,
+            'sort' => "#{exhibit.blacklight_config.document_model.unique_key} asc"
+          )
+        )
+        response.documents.each do |document|
+          block.call(document)
+        end
+
+        next_cursor_mark = response['nextCursorMark']
+      end
+    end
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+  end
+end

data/app/services/spotlight/etl.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # :nodoc:
+  module Etl
+  end
+end

data/app/services/spotlight/etl/context.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Contextual information for the ETL pipeline
+    class Context
+      # A hook for downstream applications to report or handle errors using external
+      # systems or services.
+      class_attribute :error_reporter
+
+      attr_reader :arguments, :additional_metadata, :additional_parameters, :logger
+
+      delegate :document_model, to: :resource
+
+      def initialize(*args, additional_metadata: {}, on_error: :log, logger: Rails.logger, **additional_parameters)
+        @arguments = args
+        @additional_metadata = additional_metadata
+        @additional_parameters = additional_parameters
+        @on_error = on_error
+        @logger = logger
+      end
+
+      # @return [Spotlight::Resource]
+      def resource
+        arguments.first
+      end
+
+      # @return [String]
+      def unique_key(data)
+        data[document_model&.unique_key&.to_sym || :id]
+      end
+
+      ##
+      # This hook receives any exceptions raised by pipeline steps and handles them
+      # appropriately.
+      def on_error(pipeline, exception, data)
+        error_reporter&.call(pipeline, exception, data)
+
+        case @on_error
+        when :log
+          logger.tagged('ETL') do
+            logger.error("Pipeline error processing resource #{resource.id}: #{exception}")
+          end
+        when :exception
+          raise exception
+        else
+          @on_error&.call(pipeline, exception, data)
+        end
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/executor.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # ETL pipeline executor
+    class Executor
+      include ActiveSupport::Benchmarkable
+
+      attr_reader :pipeline, :context, :source, :cache, :logger
+
+      delegate :sources, :pre_processes, :transforms, :post_processes, :loaders, to: :pipeline
+
+      # @param [Spotlight::Etl::Pipeline] pipeline
+      # @param [Spotlight::Etl::Context] context
+      # @param [Hash] cache a shared cache for pipeline steps to store data for the lifetime of the cache
+      def initialize(pipeline, context, cache: nil)
+        @pipeline = pipeline
+        @context = context
+
+        @provided_cache = cache.present?
+        @cache = cache || {}
+        @step_cache = {}
+      end
+
+      ##
+      # Execute the ETL pipeline
+      #
+      # @param [Hash] data the initial data structure to pass through to the transform steps
+      # @yield (optionally..) each transformed document after it is transformed but before
+      #        it is sent to the loaders
+      def call(data: {}, &block)
+        extract.with_index do |source, index|
+          with_source(source, index) do
+            catch :skip do
+              load(transform(data), &block)
+            end
+          rescue StandardError => e
+            on_error(e, data)
+          end
+        end
+
+        after_call
+      end
+
+      ##
+      # Estimate the number of documents that will be produced by the pipeline
+      #
+      # @return [Number]
+      def estimated_size
+        @estimated_size ||= compile_steps(sources).sum { |source| source.call(context).count }
+      end
+
+      ##
+      # Tagged logger for benchmarks and data flow logging.
+      # NOTE: this is super weird to support Rails 5.2
+      # @private
+      # @yield Logger
+      def with_logger
+        logger = (context&.logger || Rails.logger)
+        logger.tagged(pipeline.class) do
+          logger.tagged("#<#{source.class} id=#{source&.id if source.respond_to?(:id)}>") do
+            @logger = logger
+            yield logger
+          end
+        end
+      end
+
+      ##
+      # @private
+      # @param [Hash] data
+      # @return [String] a simplified + truncated version of the data hash for debugging
+      def transform_data_for_debugging(data, verbose: $VERBOSE, truncate: 100)
+        return data.inspect.truncate(truncate) unless data.is_a?(Hash)
+        return "id #{context.unique_key(data) || data&.first(5)&.inspect}" unless verbose
+
+        JSON.fast_generate(data).truncate(truncate)
+      end
+
+      ##
+      # Propagate exceptions up to the context's error handler.
+      def on_error(exception, data)
+        context.on_error(self, exception, data)
+      end
+
+      private
+
+      ##
+      # Set the current source
+      # @param [Object] source
+      # @param [Number] index
+      def with_source(source, index)
+        @source = source
+
+        benchmark "Indexing item #{source.inspect.truncate(50)} in resource #{context.resource.id} (#{index} / #{estimated_size})" do
+          yield.tap { @source = nil }
+        end
+      end
+
+      ##
+      # Extract data from sources. The defined sources receive the provided context
+      # and should return an array or other enumerable of sources to pass through
+      # the pipeline.
+      #
+      # @yield [Object]
+      def extract(&block)
+        return to_enum(:extract) { estimated_size } unless block_given?
+
+        compile_steps(sources).each do |source|
+          source.call(context).each do |data|
+            block.call(data)
+          end
+        end
+      end
+
+      ##
+      # Transform the source to a document.
+      #
+      # @param [Hash] from the initial seed data used as the input to the initial transforms
+      # @return [Hash] the transformed document
+      def transform(from)
+        compile_steps(pre_processes).each { |step| step.call(from, self) }
+
+        data = compile_steps(transforms).inject(from) { |input, step| step.call(input, self) }
+
+        compile_steps(post_processes).each { |step| step.call(data, self) }
+
+        with_logger do |logger|
+          logger.debug do
+            "Transform output: #{transform_data_for_debugging(data, verbose: true, truncate: 1000)}"
+          end
+        end
+
+        data
+      end
+
+      ##
+      # Load a document into a data sink.
+      #
+      # @param [Hash] the fully transformed data
+      # @yield [Hash] the data before it is sent to any loaders
+      def load(data, &block)
+        return unless data
+
+        catch :skip do
+          block&.call(data, self)
+
+          compile_steps(loaders).each do |loader|
+            loader.call(data, self)
+          end
+        end
+      end
+
+      ##
+      # A callback run after transforming data to do any finalizing or cleanup
+      # from the run.
+      def after_call
+        finalize_loaders
+        @cache = {} unless @provided_cache
+        @step_cache = {}
+      end
+
+      ##
+      # Loaders may implement a `#finalize` method if they want to perform any work
+      # after all the data is transformed.
+      def finalize_loaders
+        compile_steps(loaders).each do |step|
+          step.finalize(self) if step.respond_to? :finalize
+        end
+      end
+
+      ##
+      # DSL convenience utility for writing compact lists of steps; this unrolls
+      # pipeline definitions to contain arrays or hashes, e.g.:
+      # `pipeline.transforms = [step_1: lambda {}, step_2: lambda {}]`
+      #
+      # @return [Enumerable<Spotlight::Etl::Step>]
+      def compile_steps(steps)
+        return to_enum(:compile_steps, steps) unless block_given?
+
+        steps.flatten.each do |step|
+          if step.is_a? Hash
+            step.each do |k, v|
+              yield(@step_cache[k] ||= Spotlight::Etl::Step.new(v, label: k, executor: self))
+            end
+          else
+            yield @step_cache[step] ||= Spotlight::Etl::Step.new(step, executor: self)
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/loaders.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    module Loaders
+      # A loader that just prints the data to $stderr for debugging.
+      WarnLoader = lambda do |data, _context|
+        warn(JSON.pretty_generate(data))
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/pipeline.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # ETL pipeline definition
+    class Pipeline
+      include ActiveSupport::Benchmarkable
+
+      attr_reader :context, :source
+
+      # This ETL pipeline system, while somewhat generic, was implemented for Spotlight
+      # to transform Spotlight::Resource instances into Solr documents. The resources
+      # go through a series of steps (sources, transforms, loaders) to produce one or
+      # more documents in the Solr index.
+      #
+      # All of the steps below can be provided as:
+      #  - a lambda
+      #  - a ruby class (which will be initialized for each pipeline execution)
+      #  - or, a hash (of any length) with:
+      #       - a key (used only for clarity in logging, particularly useful to label lambdas)
+      #       - a value that is one of the valid step types (lambda or ruby class).
+      #
+      # Any of the transform or loader steps can `throw :skip` to skip the current source.
+      #
+      # Any exceptions raised by the pipeline's steps are sent to the context's
+      # error handler by calling `#on_error` on the context object.
+
+      # sources return enumerables that convert from the Spotlight::Etl::Context
+      # to some data structure that the transform steps can handle. The Context is provided
+      # by the implementation when the pipeline is executed.
+      class_attribute :sources, default: []
+
+      # The transform steps (pre-processes, transforms, and post-processes) receive
+      # the current data state and the pipeline. The return value from the transforms
+      # steps replaces the current data state, however the return values for pre- and
+      # post- processing is ignored (although they may mutate the provided data, pipeline, etc).
+      #
+      # Through the pipeline argument, the transform steps can access:
+      #  - `context`, the implementation-provided resource
+      #  - `source`, the current source instance
+      class_attribute :pre_processes, default: []
+      class_attribute :transforms, default: []
+      class_attribute :post_processes, default: []
+
+      # loaders receive the transformed data and.. do something with it (like load it into Solr)
+      # After all documents are transformed, the loader may also receive `#finalize` to finish any
+      # additional processing.
+      class_attribute :loaders, default: []
+
+      def initialize
+        yield(self) if block_given?
+      end
+
+      ##
+      # Execute the ETL pipeline
+      #
+      # @param [Spotlight::Etl::Context] context
+      # @param [Hash] data the initial data structure to pass through to the transform steps
+      # @yield (optioanlly..) each transformed document after it is transformed but before
+      #        it is sent to the loaders
+      def call(context, data: {}, cache: nil, &block)
+        executor(context, cache: cache).call(data: data, &block)
+      end
+
+      ##
+      # Estimate the number of documents that will be produced by the pipeline
+      #
+      # @param [Spotlight::Etl::Context] context
+      # @return [Number]
+      def estimated_size(context)
+        executor(context).estimated_size
+      end
+
+      private
+
+      def executor(context, **args)
+        Spotlight::Etl::Executor.new(self, context, **args)
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/solr_loader.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Solr data loader with a built-in buffer to combine document updates into batches
+    class SolrLoader
+      attr_reader :queue, :batch_size
+
+      delegate :size, to: :queue
+
+      def initialize(batch_size: Spotlight::Engine.config.solr_batch_size, solr_connection: nil)
+        @queue = Queue.new
+        @batch_size = batch_size
+        @blacklight_solr = solr_connection
+      end
+
+      def call(data, pipeline = nil)
+        @queue << data
+
+        write_to_index(pipeline) if @queue.size >= @batch_size
+      end
+
+      def finalize(pipeline = nil)
+        write_to_index(pipeline)
+
+        commit! if pipeline.nil? || pipeline.context.additional_parameters[:commit]
+      end
+
+      private
+
+      def write_to_index(pipeline)
+        batch = drain_queue
+
+        return unless write? && batch.any?
+
+        send_batch(batch, pipeline)
+      end
+
+      def send_batch(documents, pipeline)
+        blacklight_solr.update params: { commitWithin: 500 },
+                               data: documents.to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+      rescue StandardError => e
+        logger.warn "Error sending a batch of documents to solr: #{e}"
+
+        documents.each do |doc|
+          send_one(doc, pipeline)
+        end
+      end
+
+      def send_one(document, pipeline)
+        blacklight_solr.update params: { commitWithin: 500 },
+                               data: [document].to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+      rescue StandardError => e
+        pipeline&.on_error(self, e, document.to_json)
+      end
+
+      def blacklight_solr
+        @blacklight_solr ||= RSolr.connect(connection_config.merge(adapter: connection_config[:http_adapter]))
+      end
+
+      def connection_config
+        Blacklight.connection_config
+      end
+
+      def drain_queue
+        arr = []
+
+        begin
+          arr << @queue.deq(true) while arr.length < @batch_size && !@queue.empty?
+        rescue ThreadError
+          # @queue throws a ThreadError if it is empty...
+        end
+
+        arr
+      end
+
+      def commit!
+        return unless write?
+
+        blacklight_solr.commit
+      rescue StandardError => e
+        logger.warn "Unable to commit to solr: #{e}"
+      end
+
+      def write?
+        Spotlight::Engine.config.writable_index
+      end
+
+      def logger
+        Rails.logger
+      end
+    end
+  end
+end