blacklight-spotlight 3.0.0.rc3 → 3.0.0.rc4

data/app/services/spotlight/etl/solr_loader.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Solr data loader with a built-in buffer to combine document updates into batches
+    class SolrLoader
+      attr_reader :queue, :batch_size
+
+      delegate :size, to: :queue
+
+      def initialize(batch_size: Spotlight::Engine.config.solr_batch_size, solr_connection: nil)
+        @queue = Queue.new
+        @batch_size = batch_size
+        @blacklight_solr = solr_connection
+      end
+
+      def call(data, pipeline = nil)
+        @queue << data
+
+        write_to_index(pipeline) if @queue.size >= @batch_size
+      end
+
+      def finalize(pipeline = nil)
+        write_to_index(pipeline)
+
+        commit! if pipeline.nil? || pipeline.context.additional_parameters[:commit]
+      end
+
+      private
+
+      def write_to_index(pipeline)
+        batch = drain_queue
+
+        return unless write? && batch.any?
+
+        send_batch(batch, pipeline)
+      end
+
+      def send_batch(documents, pipeline)
+        blacklight_solr.update params: { commitWithin: 500 },
+                               data: documents.to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+      rescue StandardError => e
+        logger.warn "Error sending a batch of documents to solr: #{e}"
+
+        documents.each do |doc|
+          send_one(doc, pipeline)
+        end
+      end
+
+      def send_one(document, pipeline)
+        blacklight_solr.update params: { commitWithin: 500 },
+                               data: [document].to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+      rescue StandardError => e
+        pipeline&.on_error(self, e, document.to_json)
+      end
+
+      def blacklight_solr
+        @blacklight_solr ||= RSolr.connect(connection_config.merge(adapter: connection_config[:http_adapter]))
+      end
+
+      def connection_config
+        Blacklight.connection_config
+      end
+
+      def drain_queue
+        arr = []
+
+        begin
+          arr << @queue.deq(true) while arr.length < @batch_size && !@queue.empty?
+        rescue ThreadError
+          # @queue throws a ThreadError if it is empty...
+        end
+
+        arr
+      end
+
+      def commit!
+        return unless write?
+
+        blacklight_solr.commit
+      rescue StandardError => e
+        logger.warn "Unable to commit to solr: #{e}"
+      end
+
+      def write?
+        Spotlight::Engine.config.writable_index
+      end
+
+      def logger
+        Rails.logger
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/sources.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Basic ETL source implementations
+    module Sources
+      # A simple source that just returns the original resource(s)
+      IdentitySource = lambda do |context|
+        Array.wrap(context.resource)
+      end
+
+      # A transform step that calls a method on the resource to generate a source
+      def self.SourceMethodSource(method) # rubocop:disable Naming/MethodName
+        lambda do |context|
+          context.resource.public_send(method)
+        end
+      end
+
+      # A simple source that retrieves the stored data from a Spotlight::Resource
+      StoredData = lambda do |context, **|
+        Array.wrap(context.resource.data)
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/step.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # ETL pipeline step
+    class Step
+      attr_reader :definition, :context, :executor
+
+      # @param [Class, Proc] definition the step to run
+      # @param [String] label
+      # @param [Spotlight::Etl::Executor] executor the execution environment
+      def initialize(definition, label: nil, executor: nil)
+        @definition = definition
+        @executor = executor
+        @label = label
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def call(*args)
+        with_logger do |logger|
+          logger.debug { "Called with #{transform_data_for_debugging(args.first)}" }
+
+          catch :skip do
+            return action.call(*args).tap do |result|
+              logger.debug { "   => Returning #{transform_data_for_debugging(result)}" } if $VERBOSE
+            end
+          end
+
+          logger.debug '  => Caught skip.'
+          throw :skip
+        end
+      rescue StandardError => e
+        with_logger do |logger|
+          logger.error("Caught exception #{e}")
+        end
+        raise(e)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def finalize(*args)
+        action.finalize(*args) if action.respond_to? :finalize
+      end
+
+      private
+
+      # @return [#call]
+      def action
+        case definition
+        when Class
+          # memoize the class' instance for the lifetime of the step
+          @memoized_action ||= definition.new
+        else # Proc, etc
+          definition
+        end
+      end
+
+      # @return [#to_string]
+      def label
+        @label || definition
+      end
+
+      # NOTE: this is super weird to support Rails 5.2
+      # @return [Logger]
+      def with_logger
+        yield(Rails.logger) && return unless executor
+
+        executor.with_logger do |logger|
+          logger.tagged(label) do
+            yield logger
+          end
+        end
+      end
+
+      ##
+      # @param [Hash] data
+      # @return [String] a simplified + truncated version of the data hash for debugging
+      def transform_data_for_debugging(data)
+        executor&.transform_data_for_debugging(data) || data.inspect.truncate(100)
+      end
+    end
+  end
+end

data/app/services/spotlight/etl/transforms.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Spotlight
+  module Etl
+    # Basic + default transform steps
+    module Transforms
+      # A transform step that "transforms" the source into the data element
+      IdentityTransform = lambda do |data, pipeline|
+        data.merge(pipeline.source)
+      end
+
+      # A transform step that calls a method on the source to generate a document
+      def self.SourceMethodTransform(method) # rubocop:disable Naming/MethodName
+        lambda do |data, pipeline|
+          data.merge(pipeline.source.public_send(method))
+        end
+      end
+
+      # A transform step that throws away blank data
+      RejectBlank = lambda do |data, _|
+        throw :skip if data.blank?
+        data
+      end
+
+      # A trasnform step that ensures data has a unique key attribute
+      RejectMissingUniqueId = lambda do |data, pipeline|
+        id = pipeline.context.unique_key(data)
+
+        throw :skip if id.blank?
+        data
+      end
+
+      # A transform that adds exhibit-specific metadata (like Spotlight sidecar data)
+      # to the document
+      ApplyExhibitMetadata = lambda do |data, pipeline|
+        resource = pipeline.context.resource
+        document_model = pipeline.context.document_model
+        id = pipeline.context.unique_key(data)
+
+        next data unless document_model && id.present?
+
+        exhibit_metadata = document_model.build_for_exhibit(id, resource.exhibit, resource: (resource if resource.persisted?))
+
+        data.reverse_merge(exhibit_metadata.to_solr)
+      end
+
+      # A transform that adds application-specific metadata (like what resource generated the solr document)
+      ApplyApplicationMetadata = lambda do |data, pipeline|
+        resource = pipeline.context.resource
+        document_model = pipeline.context.document_model
+
+        data.reverse_merge(
+          Spotlight::Engine.config.resource_global_id_field => (resource.to_global_id.to_s if resource.persisted?),
+          document_model.resource_type_field => resource.class.to_s.tableize
+        )
+      end
+
+      # A transform that adds externally-provided metadata to the document
+      ApplyPipelineMetadata = lambda do |data, pipeline|
+        data.reverse_merge(pipeline.context.additional_metadata)
+      end
+    end
+  end
+end

data/app/services/spotlight/validity_checker.rb

@@ -5,16 +5,16 @@ module Spotlight
   # check if a delayed job still needs to run
   class ValidityChecker
     # Return a validity token
-    # @param [ActiveModel::Model]
+    # @param [ActiveJob::Base]
     # @return [Object] any serializable object
-    def mint(_model); end
+    def mint(_job); end
 
     # Check if the token is still valid for the model
-    # @param [ActiveModel::Model]
+    # @param [ActiveJob::Base]
     # @param [Object] the serializable token minted by #mint
     # @return [boolean]
-    def check(_model, _token)
-      true
+    def check(_job, validity_token: nil)
+      validity_token || true
     end
   end
 end

data/app/views/spotlight/dashboards/_reindexing_activity.html.erb

@@ -12,13 +12,13 @@
     </tr>
   </thead>
   <tbody>
-    <% @recent_reindexing.each do |log_entry| %>
+    <% @recent_reindexing.each do |tracker| %>
       <tr class="d-flex">
-        <td class="col-3"><%= l log_entry.start_time, format: :long if log_entry.start_time %></td>
-        <td class="col-3"><%= log_entry.user.email if log_entry.user %></td>
-        <td class="col-1"><%= log_entry.items_reindexed_count %></td>
-        <td class="col-3"><%= distance_of_time_in_words log_entry.duration, 0, include_seconds: true if log_entry.duration %></td>
-        <td class="col-2"><%= t "reindexing_log.status.#{log_entry.job_status}" %></td>
+        <td class="col-3"><%= l tracker.created_at, format: :long %></td>
+        <td class="col-3"><%= tracker.user.email if tracker.user %></td>
+        <td class="col-1"><%= tracker.progress %></td>
+        <td class="col-3"><%= distance_of_time_in_words tracker.created_at, tracker.updated_at, include_seconds: true if tracker.completed? || tracker.failed? %></td>
+        <td class="col-2"><%= t "reindexing_log.status.#{tracker.status}" %></td>
       </tr>
     <% end %>
   </tbody>

data/app/views/spotlight/shared/_locale_picker.html.erb

@@ -6,7 +6,7 @@
     <ul class="dropdown-menu">
       <% locale_selecter_dropown_options.each do |language| %>
         <li>
-          <%= link_to language.to_native, (spotlight.url_for(locale: lanaguage.locale) rescue url_for(locale: language.locale)), class: 'dropdown-item', data: { turbolinks: false } %>
+          <%= link_to language.to_native, current_page_for_locale(language.locale), class: 'dropdown-item', data: { turbolinks: false } %>
         </li>
       <% end %>
     </ul>

data/app/views/spotlight/sir_trevor/blocks/_browse_group_categories_block.html.erb

@@ -5,15 +5,15 @@
         <div class="d-flex justify-content-between align-items-end">
           <h2 class="m-0"><%= group.title %></h2>
           <div class="pl-5">
-            <%= link_to 'View all', exhibit_browse_groups_path(current_exhibit, group) %>
+            <%= link_to t(:'.view_all'), exhibit_browse_groups_path(current_exhibit, group) %>
           </div>
         </div>
         <% if group.searches.count > 1 %>
           <ul class="browse-group-categories-controls p-0 pt-2 pt-md-0 ml-md-auto mb-0 d-flex justify-content-center align-items-end">
-            <li class="prev px-1">
+            <li class="prev px-1 overflow-hidden">
               <%= blacklight_icon 'arrow-alt-circle-left' %>
             </li>
-            <li class="next px-1">
+            <li class="next px-1 overflow-hidden">
               <%= blacklight_icon 'arrow-alt-circle-right' %>
             </li>
           </ul>
@@ -34,6 +34,7 @@
                     <span class="item-count"><%= t(:'.items', count: search.count) %></span>
                   <% end %>
                 </div>
+                <div class="hover-overlay"></div>
               </div>
             <% end %>
           </div>

data/config/locales/spotlight.ar.yml

@@ -34,6 +34,9 @@ ar:
         zero_results:
           expand_html: "تستطيع أن <a href=\"%{clear_search_url}\"> تمسح استعلام البحث </a> أو محاولة <a href=\"%{expand_search_url}\"> البحث في كل المواد المعروضة لاستعلام البحث ل\"%{browse_query}\"</a>"
           result_number: لم يتطابق بحثك مع أي عنصر في هذه الفئة.
+    exhibits:
+      groups:
+        all: الكل
     header_links:
       contact: ملاحظات واقتراحات
       login: تسجيل الدخول
@@ -52,7 +55,14 @@ ar:
             few: "%{count} عناصر"
             many: "%{count} عنصراً"
             other: "%{count} عنصراً"
-
+        browse_group_categories_block:
+          items:
+            one: "%{count} عنصر"
+            two: "%{count} عنصرين"
+            few: "%{count} عناصر"
+            many: "%{count} عنصراً"
+            other: "%{count} عنصراً"
+          view_all: مشاهدة الكل
         link_to_search_block:
           items:
             zero: "%{count} عنصر"

data/config/locales/spotlight.en.yml

@@ -167,10 +167,10 @@ en:
     zh: Chinese
   reindexing_log:
     status:
+      completed: Successful
+      enqueued: Not yet started
       failed: Failed
       in_progress: In progress
-      succeeded: Successful
-      unstarted: Not yet started
   shared:
     site_sidebar:
       documentation: Curator documentation
@@ -765,6 +765,7 @@ en:
           items:
             one: "%{count} item"
             other: "%{count} items"
+          view_all: View all
         link_to_search_block:
           items:
             one: "%{count} item"

data/db/migrate/20210122082032_create_job_trackers.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class CreateJobTrackers < ActiveRecord::Migration[5.2]
+  def change
+    create_table :spotlight_job_trackers do |t|
+      t.references :on, null: false, polymorphic: true, index: true
+      t.references :resource, null: false, polymorphic: true, index: true
+      t.string :job_id
+      t.string :job_class
+      t.string :parent_job_id
+      t.string :parent_job_class
+      t.string :status
+      t.references :user
+      t.text :log
+      t.text :data
+
+      t.timestamps
+    end
+
+    add_index :spotlight_job_trackers, :job_id
+  end
+end

data/db/migrate/20210126123041_create_events.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateEvents < ActiveRecord::Migration[5.2]
+  def change
+    create_table :spotlight_events do |t|
+      t.references :exhibit, null: false, polymorphic: true
+      t.references :resource, null: false, polymorphic: true, index: true
+      t.string :type
+      t.string :collation_key
+      t.text :data
+
+      t.timestamps
+    end
+  end
+end

data/lib/generators/spotlight/scaffold_resource_generator.rb

@@ -7,23 +7,15 @@ module Spotlight
   # spotlight:scaffold_resource generator
   class ScaffoldResource < Rails::Generators::NamedBase
     source_root File.expand_path('templates', __dir__)
-    def create_document_builder
-      create_file "app/services/#{file_name}_builder.rb", <<-FILE.strip_heredoc
-        class #{class_name}Builder < Spotlight::SolrDocumentBuilder
-          def to_solr
-            return to_enum(:to_solr) unless block_given?
-
-            # TODO: your implementation here
-            # yield { id: resource.id }
-          end
-        end
-      FILE
-    end
 
     def create_model
       create_file "app/models/#{file_name}_resource.rb", <<-FILE.strip_heredoc
         class #{class_name}Resource < Spotlight::Resource
-          self.document_builder_class = #{class_name}Builder
+          def self.indexing_pipeline
+            @indexing_pipeline ||= super.dup.tap do |pipeline|
+              # your pipeline here...
+            end
+          end
         end
       FILE
     end