blacklight-spotlight 3.0.0.rc3 → 3.0.0.rc4

data/app/jobs/spotlight/update_job_trackers_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  # Calls the #set_default_thumbnail method
+  # on the object passed in and calls save
+  ###
+  class UpdateJobTrackersJob < Spotlight::ApplicationJob
+    def perform(job_tracker)
+      reports_on = job_tracker.on
+
+      return unless reports_on.is_a? Spotlight::JobTracker
+
+      reports_on.update(status: 'completed') if reports_on.job_trackers.all?(&:completed?)
+      reports_on.update(status: 'failed') if reports_on.job_trackers.any?(&:failed?)
+
+      reports_on.update(data: { progress: reports_on.job_trackers.sum(&:progress), total: reports_on.job_trackers.sum(&:total) })
+    end
+  end
+end

data/app/models/concerns/spotlight/user.rb

@@ -8,7 +8,6 @@ module Spotlight
     included do
       has_many :roles, class_name: 'Spotlight::Role', dependent: :destroy
       has_many :exhibits, class_name: 'Spotlight::Exhibit', through: :roles, source: 'resource', source_type: 'Spotlight::Exhibit'
-      has_many :reindexing_log_entries, class_name: 'Spotlight::ReindexingLogEntry'
 
       scope :with_roles, -> { where(id: Spotlight::Role.distinct.pluck(:user_id)) }
 
@@ -28,6 +27,8 @@ module Spotlight
     end
 
     def add_default_roles
+      return unless Spotlight::Engine.config.assign_default_roles_to_first_user
+
       roles.build role: 'admin', resource: Spotlight::Site.instance unless self.class.any?
     end
 

data/app/models/spotlight/event.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # Logged events for Spotlight exhibits
+  class Event < ActiveRecord::Base
+    belongs_to :resource, polymorphic: true
+    belongs_to :exhibit, optional: true
+
+    serialize :data
+
+    self.inheritance_column = :event_class
+  end
+end

data/app/models/spotlight/exhibit.rb

@@ -4,7 +4,6 @@ require 'mail'
 module Spotlight
   ##
   # Spotlight exhibit
-  # rubocop:disable Metrics/ClassLength
   class Exhibit < ActiveRecord::Base
     class_attribute :themes_selector
     include Spotlight::ExhibitAnalytics
@@ -57,8 +56,8 @@ module Spotlight
 
     has_many :feature_pages, -> { for_default_locale }, extend: FriendlyId::FinderMethods
     has_many :groups, dependent: :delete_all
+    has_many :job_trackers, as: :on, dependent: :delete_all
     has_many :main_navigations, dependent: :delete_all
-    has_many :reindexing_log_entries, dependent: :destroy
     has_many :resources
     has_many :roles, as: :resource, dependent: :delete_all
     has_many :searches, dependent: :destroy, extend: FriendlyId::FinderMethods
@@ -119,8 +118,8 @@ module Spotlight
       end
     end
 
-    def reindex_later(user = nil)
-      Spotlight::ReindexJob.perform_later(self, new_reindexing_log_entry(user))
+    def reindex_later(current_user = nil)
+      Spotlight::ReindexExhibitJob.perform_later(self, user: current_user)
     end
 
     def uploaded_resource_fields
@@ -136,7 +135,7 @@ module Spotlight
     end
 
     def reindex_progress
-      @reindex_progress ||= ReindexProgress.new(current_reindexing_log_entry)
+      @reindex_progress ||= ReindexProgress.new(self)
     end
 
     def available_locales
@@ -149,19 +148,10 @@ module Spotlight
       self.description = ::Rails::Html::FullSanitizer.new.sanitize(description)
     end
 
-    def new_reindexing_log_entry(user = nil)
-      Spotlight::ReindexingLogEntry.create(exhibit: self, user: user, items_reindexed_count: 0, job_status: 'unstarted')
-    end
-
     private
 
-    def current_reindexing_log_entry
-      reindexing_log_entries.started_or_completed.first || reindexing_log_entries.build
-    end
-
     def move_friendly_id_error_to_slug
       errors.add :slug, *errors.delete(:friendly_id) if errors[:friendly_id].present?
     end
   end
-  # rubocop:enable Metrics/ClassLength
 end

data/app/models/spotlight/job_tracker.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # Associate background jobs with records
+  class JobTracker < ActiveRecord::Base
+    scope :recent, -> { order('updated_at DESC').limit(5) }
+    scope :in_progress, -> { where.not(status: %w[completed failed]) }
+    scope :completed, -> { where(status: %w[completed failed]) }
+
+    belongs_to :on, polymorphic: true
+    belongs_to :resource, polymorphic: true
+    belongs_to :user, optional: true, class_name: Spotlight::Engine.config.user_class # rubocop:disable Rails/ReflectionClassName
+    has_many :events, as: :resource, dependent: :delete_all
+    has_many :job_trackers, as: :on, dependent: Rails.version > '6.1' ? :destroy_async : :destroy
+
+    serialize :data
+
+    after_initialize do
+      self.data ||= {}
+    end
+
+    after_commit do
+      next unless on.is_a? Spotlight::JobTracker
+
+      UpdateJobTrackersJob.perform_later(self)
+    end
+
+    def label
+      "[#{job_class.titleize}] #{resource_label}"
+    end
+
+    def resource_label
+      return resource.filename if resource.is_a? ActiveStorage::Blob
+      return resource.name if resource.is_a? Upload
+
+      resource_id
+    end
+
+    def job_status
+      return {} unless job_id
+
+      @job_status ||= ActiveJob::Status.get(job_id)
+    end
+
+    def progress_label
+      return number_with_delimiter(progress) unless total?
+
+      "#{number_with_delimiter(progress)} / #{number_with_delimiter(total)}"
+    end
+
+    def progress
+      data[:progress] || job_status[:progress] || 0
+    end
+
+    def total(default: progress)
+      [progress, data[:total] || job_status[:total] || default].max
+    end
+
+    def total?
+      total(default: 0).positive?
+    end
+
+    def percent
+      return nil unless total?
+
+      (100.0 * progress) / total
+    end
+
+    def enqueued?
+      status == 'enqueued'
+    end
+
+    def in_progress?
+      status == 'in_progress'
+    end
+
+    def completed?
+      status == 'completed'
+    end
+
+    def failed?
+      status == 'failed'
+    end
+
+    def append_log_entry(type:, **args)
+      events.create(type: type, data: args)
+    rescue StandardError => e
+      Rails.logger.error("Unable to create log entry for job tracker #{id}: #{e}")
+    end
+
+    def top_level_job_tracker
+      if on.is_a?(Spotlight::JobTracker)
+        on.top_level_job_tracker
+      else
+        self
+      end
+    end
+
+    private
+
+    def number_with_delimiter(*args)
+      ActiveSupport::NumberHelper.number_to_delimited(*args)
+    end
+  end
+end

data/app/models/spotlight/reindex_progress.rb

@@ -4,59 +4,76 @@ module Spotlight
   ##
   # ReindexProgress is a class that models the progress of reindexing a list of resources
   class ReindexProgress
-    attr_reader :current_log_entry
+    attr_reader :exhibit
 
-    delegate :updated_at, to: :current_log_entry
+    delegate :updated_at, to: :most_relevant_job_tracker
 
-    def initialize(current_log_entry)
-      @current_log_entry = current_log_entry
+    def initialize(exhibit)
+      @exhibit = exhibit
+    end
+
+    def as_json(*)
+      {
+        recently_in_progress: recently_in_progress?,
+        started_at: localized_start_time,
+        finished_at: localized_finish_time,
+        updated_at: localized_updated_time,
+        total: [total, completed].max,
+        completed: completed,
+        finished: finished?,
+        errored: errored?
+      }
+    end
+
+    private
+
+    def job_trackers
+      @job_trackers ||= exhibit.job_trackers.where(job_class: 'Spotlight::ReindexExhibitJob').recent
+    end
+
+    def most_relevant_job_tracker
+      return @most_relevant_job_tracker if @most_relevant_job_tracker
+
+      @most_relevant_job_tracker ||= job_trackers.in_progress.first || job_trackers.completed.first || job_trackers.first || Spotlight::JobTracker.new
     end
 
     def recently_in_progress?
-      return true if current_log_entry.in_progress?
+      return false unless most_relevant_job_tracker.persisted?
+      return true if most_relevant_job_tracker.in_progress?
 
-      current_log_entry.end_time.present? && (current_log_entry.end_time > Spotlight::Engine.config.reindex_progress_window.minutes.ago)
+      finished? && most_relevant_job_tracker.updated_at >= Spotlight::Engine.config.reindex_progress_window.ago
     end
 
     def started_at
-      current_log_entry.start_time
+      most_relevant_job_tracker.created_at
     end
 
     def finished?
-      current_log_entry.succeeded? || current_log_entry.failed?
+      most_relevant_job_tracker.completed? || (errored? && most_relevant_job_tracker.job_trackers.none?(&:in_progress?))
     end
 
     def finished_at
-      current_log_entry.end_time
+      return unless finished?
+
+      most_relevant_job_tracker.updated_at
     end
 
     def total
-      current_log_entry.items_reindexed_estimate
+      return most_relevant_job_tracker.total if finished?
+
+      most_relevant_job_tracker.job_trackers.sum(&:total)
     end
 
     def completed
-      current_log_entry.items_reindexed_count
-    end
+      return most_relevant_job_tracker.progress if finished?
 
-    def errored?
-      current_log_entry.failed?
+      most_relevant_job_tracker.job_trackers.sum(&:progress)
     end
 
-    def as_json(*)
-      {
-        recently_in_progress: recently_in_progress?,
-        started_at: localized_start_time,
-        finished_at: localized_finish_time,
-        updated_at: localized_updated_time,
-        total: total,
-        completed: completed,
-        finished: finished?,
-        errored: errored?
-      }
+    def errored?
+      most_relevant_job_tracker.failed?
     end
 
-    private
-
     def localized_start_time
       return unless started_at
 

data/app/models/spotlight/resource.rb

@@ -4,10 +4,17 @@ module Spotlight
   ##
   # Exhibit resources
   class Resource < ActiveRecord::Base
-    include ActiveSupport::Benchmarkable
-
-    class_attribute :document_builder_class
-    self.document_builder_class = SolrDocumentBuilder
+    class_attribute :indexing_pipeline, default: (Spotlight::Etl::Pipeline.new do |pipeline|
+      pipeline.sources = [Spotlight::Etl::Sources::IdentitySource]
+      pipeline.transforms = [
+        reject_blank: Spotlight::Etl::Transforms::RejectBlank,
+        reject_missing: Spotlight::Etl::Transforms::RejectMissingUniqueId,
+        apply_exhibit_metadata: Spotlight::Etl::Transforms::ApplyExhibitMetadata,
+        apply_application_metadata: Spotlight::Etl::Transforms::ApplyApplicationMetadata,
+        apply_pipeline_metadata: Spotlight::Etl::Transforms::ApplyPipelineMetadata
+      ]
+      pipeline.loaders = [Spotlight::Etl::SolrLoader]
+    end)
 
     extend ActiveModel::Callbacks
     define_model_callbacks :index
@@ -16,12 +23,10 @@ module Spotlight
 
     belongs_to :exhibit
     has_many :solr_document_sidecars
+    has_many :events, as: :resource
 
     serialize :data, Hash
 
-    after_index :commit
-    after_index :touch_exhibit!
-
     ##
     # Persist the record to the database, and trigger a reindex to solr
     #
@@ -45,67 +50,28 @@ module Spotlight
       # Index the result of {#to_solr} into the index in batches of {#batch_size}
       #
       # @return [Integer] number of records indexed
-      def reindex(reindexing_log_entry = nil)
-        benchmark "Reindexing #{self} (batch size: #{batch_size})" do
-          count = 0
-
-          run_callbacks :index do
-            document_builder.documents_to_index.each_slice(batch_size) do |batch|
-              write_to_index(batch)
-              count += batch.length
-              reindexing_log_entry&.update(items_reindexed_count: count)
-            end
-
-            count
+      def reindex(touch: true, **args, &block)
+        i = 0
+        run_callbacks :index do
+          indexing_pipeline.call(Spotlight::Etl::Context.new(self, commit: true, **args)) do |data|
+            i += 1
+            block&.call(data)
           end
         end
-      end
-
-      def document_builder
-        @document_builder ||= document_builder_class.new(self)
-      end
-
-      private
-
-      def blacklight_solr
-        @solr ||= RSolr.connect(connection_config.merge(adapter: connection_config[:http_adapter]))
-      end
 
-      def connection_config
-        Blacklight.connection_config
-      end
+        touch_exhibit! if touch
 
-      def batch_size
-        Spotlight::Engine.config.solr_batch_size
+        i
       end
 
-      def write_to_index(batch)
-        documents = documents_that_have_ids(batch)
-        return unless write? && documents.present?
-
-        blacklight_solr.update params: { commitWithin: 500 },
-                               data: documents.to_json,
-                               headers: { 'Content-Type' => 'application/json' }
+      def estimated_size(**args)
+        indexing_pipeline.estimated_size(Spotlight::Etl::Context.new(self, **args))
       end
 
-      def commit
-        return unless write?
-
-        blacklight_solr.commit
-      rescue StandardError => e
-        Rails.logger.warn "Unable to commit to solr: #{e}"
-      end
+      private
 
       def touch_exhibit!
-        exhibit.touch
-      end
-
-      def write?
-        Spotlight::Engine.config.writable_index
-      end
-
-      def documents_that_have_ids(document_list)
-        document_list.reject { |d| d[document_builder.document_model.unique_key.to_sym].blank? }
+        exhibit&.touch
       end
     end
   end

data/app/models/spotlight/resources/iiif_harvester.rb

@@ -7,7 +7,6 @@ module Spotlight
     # harvest Images from IIIF Manifest and turn them into a Spotlight::Resource
     # Note: IIIF API : http://iiif.io/api/presentation/2.0
     class IiifHarvester < Spotlight::Resource
-      self.document_builder_class = Spotlight::Resources::IiifBuilder
       self.weight = -5000
 
       validate :valid_url?
@@ -16,6 +15,16 @@ module Spotlight
         @iiif_manifests ||= IiifService.parse(url)
       end
 
+      def self.indexing_pipeline
+        @indexing_pipeline ||= super.dup.tap do |pipeline|
+          pipeline.sources = [Spotlight::Etl::Sources::SourceMethodSource(:iiif_manifests)]
+
+          pipeline.transforms = [
+            ->(data, p) { data.merge(p.source.to_solr(exhibit: p.context.resource.exhibit)) }
+          ] + pipeline.transforms
+        end
+      end
+
       private
 
       def valid_url?