blacklight-spotlight 3.0.0.rc2 → 3.0.0

data/app/jobs/concerns/spotlight/job_tracking.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # Job status tracking
+  module JobTracking
+    extend ActiveSupport::Concern
+    include ActiveJob::Status
+
+    class_methods do
+      # @param resource [Proc] receives the job and returns the resource connect to the job tracking status
+      # @param reports_on [Proc] optional, receives the job and returns a Spotlight::JobTracker to "roll up" statuses to
+      # @param user [Proc] optional, receives the job and returns the User that initiated the job
+      def with_job_tracking(
+        resource:,
+        reports_on: ->(job) { job.arguments.last[:reports_on] if job.arguments.last.is_a?(Hash) },
+        user: ->(job) { job.arguments.last[:user] if job.arguments.last.is_a?(Hash) }
+      )
+        around_perform do |job, block|
+          resource_object = resource&.call(job)
+
+          job.initialize_job_tracker!(
+            resource: resource_object,
+            on: reports_on&.call(job) || resource_object,
+            user: user&.call(job)
+          )
+
+          block.call
+        ensure
+          job.finalize_job_tracker!
+        end
+      end
+    end
+
+    def mark_job_as_failed!
+      @failed = true
+    end
+
+    def job_tracker
+      @job_tracker ||= find_or_initialize_job_tracker
+    end
+
+    def initialize_job_tracker!(**params)
+      job_tracker.update(params.merge(status: 'in_progress').compact)
+    end
+
+    def finalize_job_tracker!
+      return unless job_tracker.status == 'in_progress' || job_tracker.status == 'enqueued'
+
+      job_tracker.update(
+        status: @failed ? 'failed' : 'completed',
+        data: { progress: progress.progress, total: progress.total }
+      )
+    end
+
+    private
+
+    def find_or_initialize_job_tracker
+      JobTracker.find_or_create_by(job_id: job_id) do |tracker|
+        tracker.job_class = self.class.name
+        tracker.status = 'enqueued'
+      end
+    end
+  end
+end

data/app/jobs/concerns/spotlight/limit_concurrency.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # Job status tracking
+  module LimitConcurrency
+    extend ActiveSupport::Concern
+
+    VALIDITY_TOKEN_PARAMETER = 'validity_token'
+
+    included do
+      # The validity checker is a seam for implementations to expire unnecessary
+      # indexing tasks if it becomes redundant while waiting in the job queue.
+      class_attribute :validity_checker, default: Spotlight::ValidityChecker.new
+
+      before_enqueue do |job|
+        token = job.arguments.last[VALIDITY_TOKEN_PARAMETER] if job.arguments.last.is_a?(Hash)
+        token ||= validity_checker.mint(job)
+
+        job.arguments << {} unless job.arguments.last.is_a? Hash
+        job.arguments.last[VALIDITY_TOKEN_PARAMETER] = token
+      end
+
+      before_perform do |job|
+        next unless job.arguments.last.is_a?(Hash)
+
+        token = job.arguments.last.delete(VALIDITY_TOKEN_PARAMETER)
+        throw(:abort) unless token.nil? || validity_checker.check(job, validity_token: token)
+
+        job.arguments.pop if job.arguments.last.empty?
+      end
+    end
+  end
+end

data/app/jobs/spotlight/add_tags_job.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  class AddTagsJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    include Spotlight::GatherDocuments
+    with_job_tracking(resource: ->(job) { job.arguments.last[:exhibit] })
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def perform(solr_params:, exhibit:, tags:, **)
+      @errors = 0
+
+      each_document(solr_params, exhibit) do |document|
+        sidecar = document.sidecar(exhibit)
+        all_tags = sidecar.all_tags_list
+        all_tags += tags
+        exhibit.tag(document.sidecar(exhibit), with: all_tags, on: :tags)
+        document.reindex(update_params: {})
+        progress&.increment
+      rescue StandardError => e
+        job_tracker.append_log_entry(type: :error, exhibit: exhibit, message: e.to_s)
+        @errors += 1
+        mark_job_as_failed!
+      end
+      exhibit.blacklight_config.repository.connection.commit
+      job_tracker.append_log_entry(type: :info, exhibit: exhibit, message: "#{progress.progress} of #{progress.total} (#{@errors} errors)")
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+  end
+end

data/app/jobs/spotlight/add_uploads_from_csv.rb

@@ -3,12 +3,13 @@
 module Spotlight
   ##
   # Process a CSV upload into new Spotlight::Resource::Upload objects
-  class AddUploadsFromCsv < ActiveJob::Base
+  class AddUploadsFromCsv < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    with_job_tracking(resource: ->(job) { job.arguments[1] })
+
     attr_reader :count
     attr_reader :errors
 
-    queue_as :default
-
     after_perform do |job|
       csv_data, exhibit, user = job.arguments
       Spotlight::IndexingCompleteMailer.documents_indexed(

data/app/jobs/spotlight/application_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # :nodoc:
+  class ApplicationJob < ActiveJob::Base
+    queue_as :default
+  end
+end

data/app/jobs/spotlight/change_visibility_job.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  class ChangeVisibilityJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    include Spotlight::GatherDocuments
+    with_job_tracking(resource: ->(job) { job.arguments.last[:exhibit] })
+
+    # rubocop:disable Metrics/MethodLength
+    def perform(solr_params:, exhibit:, visibility:, **)
+      @errors = 0
+
+      each_document(solr_params, exhibit) do |document|
+        case visibility
+        when 'public'
+          document.make_public!(exhibit)
+        when 'private'
+          document.make_private!(exhibit)
+        end
+        document.reindex(update_params: {})
+        progress&.increment
+      rescue StandardError => e
+        job_tracker.append_log_entry(type: :error, exhibit: exhibit, message: e.to_s)
+        @errors += 1
+        mark_job_as_failed!
+      end
+      exhibit.blacklight_config.repository.connection.commit
+      job_tracker.append_log_entry(type: :info, exhibit: exhibit, message: "#{progress.progress} of #{progress.total} (#{@errors} errors)")
+    end
+    # rubocop:enable Metrics/MethodLength
+  end
+end

data/app/jobs/spotlight/cleanup_job_trackers_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  # Calls the #set_default_thumbnail method
+  # on the object passed in and calls save
+  ###
+  class CleanupJobTrackersJob < Spotlight::ApplicationJob
+    def perform
+      Spotlight::JobTracker.where(status: 'completed', updated_at: Time.zone.at(0)...Spotlight::Engine.config.reindex_progress_window.minutes.ago).delete_all
+    end
+  end
+end

data/app/jobs/spotlight/default_thumbnail_job.rb

@@ -5,9 +5,7 @@ module Spotlight
   # Calls the #set_default_thumbnail method
   # on the object passed in and calls save
   ###
-  class DefaultThumbnailJob < ActiveJob::Base
-    queue_as :default
-
+  class DefaultThumbnailJob < Spotlight::ApplicationJob
     def perform(thumbnailable)
       thumbnailable.set_default_thumbnail
       thumbnailable.save

data/app/jobs/spotlight/process_bulk_updates_csv_job.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Spotlight
+  ###
+  class ProcessBulkUpdatesCsvJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    with_job_tracking(resource: ->(job) { job.arguments.first })
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def perform(exhibit, bulk_update)
+      errors = 0
+      header_converter = ->(header) { header } # Use raw header for columns (since they are configured)
+      csv_path = bulk_update.file.current_path
+      File.open(csv_path) do |f|
+        progress&.total = f.each_line.count(&:present?) - 1 # ignore the header
+
+        ::CSV.table(f, header_converters: header_converter).each do |row|
+          process_row(exhibit, row)
+          progress&.increment
+        rescue StandardError => e
+          job_tracker.append_log_entry(type: :error, exhibit: exhibit, message: e.to_s)
+          errors += 1
+          mark_job_as_failed!
+        end
+
+        exhibit.blacklight_config.repository.connection.commit
+        job_tracker.append_log_entry(type: :info, exhibit: exhibit, message: "#{progress&.progress} of #{progress&.total} (#{errors} errors)")
+      end
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    private
+
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+    def process_row(exhibit, row)
+      document = exhibit.blacklight_config.document_model.find(row[config.csv_id])
+      sidecar = document.sidecar(exhibit)
+
+      raise 'Unable to locate document' if document.blank?
+
+      needs_reindex = false
+
+      begin
+        if sidecar.public != to_bool(row[config.csv_visibility])
+          sidecar.update(public: to_bool(row[config.csv_visibility]))
+          needs_reindex = true
+        end
+
+        tag_cols = row.headers.select { |k| k =~ /^#{config.csv_tags_prefix}/ }
+
+        if tag_cols.any?
+          all_tags = sidecar.all_tags_list
+
+          added_tags_cols, removed_tags_cols = tag_cols.partition do |k|
+            to_bool(row[k])
+          end
+
+          existing_or_added_tags = added_tags_cols.map { |x| x.sub(/#{config.csv_tags_prefix}\s/, '') }
+          removed_tags = removed_tags_cols.map { |x| x.sub(/#{config.csv_tags_prefix}\s/, '') }
+
+          if all_tags.difference(existing_or_added_tags).any? || existing_or_added_tags.difference(all_tags).any?
+            exhibit.tag(sidecar, with: all_tags - removed_tags + existing_or_added_tags, on: :tags)
+            needs_reindex = true
+          end
+        end
+      ensure
+        document.reindex(update_params: {}) if needs_reindex
+      end
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+    def to_bool(value)
+      ActiveModel::Type::Boolean.new.cast(value.to_s.strip)
+    end
+
+    def config
+      Spotlight::Engine.config.bulk_updates
+    end
+  end
+end

data/app/jobs/spotlight/reindex_exhibit_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ##
+  # Reindex an exhibit by parallelizing resource indexing into multiple batches of reindex jobs
+  class ReindexExhibitJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    with_job_tracking(resource: ->(job) { job.arguments.first })
+
+    include Spotlight::LimitConcurrency
+
+    def perform(exhibit, batch_size: Spotlight::Engine.config.reindexing_batch_size, batch_count: Spotlight::Engine.config.reindexing_batch_count, **)
+      count = exhibit.resources.count
+
+      # Use the provided batch size, or calculate a reasonable default
+      batch_count = (count.to_f / batch_size).ceil if batch_size
+      batch_count ||= 1 + Math.log(count).round # e.g. 10 => 3, 100 => 6, 1000 => 8
+
+      return Spotlight::ReindexJob.perform_now(exhibit, reports_on: job_tracker) if batch_count == 1
+
+      batch_size ||= (count.to_f / batch_count).ceil
+
+      perform_later_in_batches(exhibit, of: batch_size)
+
+      # mark the job as 'pending' and let the UpdateJobTrackersJob finalize this status after the ReindexJobs finish
+      job_tracker.update(status: 'pending')
+    end
+
+    def perform_later_in_batches(exhibit, of:)
+      last = 0
+      exhibit.resources.select(:id).in_batches(of: of) do |batch|
+        last = batch.last.id
+        Spotlight::ReindexJob.perform_later(exhibit, reports_on: job_tracker, start: batch.first.id, finish: batch.last.id)
+      end
+
+      Spotlight::ReindexJob.perform_later(exhibit, reports_on: job_tracker, start: last)
+    end
+  end
+end

data/app/jobs/spotlight/reindex_job.rb

@@ -3,72 +3,92 @@
 module Spotlight
   ##
   # Reindex the given resources or exhibits
-  class ReindexJob < ActiveJob::Base
-    queue_as :default
+  class ReindexJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    with_job_tracking(resource: ->(job) { job.exhibit })
 
-    # The validity checker is a seam for implementations to expire unnecessary
-    # indexing tasks if it becomes redundant while waiting in the job queue.
-    class_attribute :validity_checker, default: Spotlight::ValidityChecker.new
-    self.validity_checker ||= Spotlight::ValidityChecker.new if Rails.version < '5.2'
+    include Spotlight::LimitConcurrency
 
     before_perform do |job|
-      job_log_entry = log_entry(job)
-      next unless job_log_entry
+      pagination = job.arguments.last.slice(:start, :finish) if job.arguments.last.is_a? Hash
+      pagination ||= {}
 
-      items_reindexed_estimate = resource_list(job.arguments.first).sum do |resource|
-        resource.document_builder.documents_to_index.size
-      end
-      job_log_entry.update(items_reindexed_estimate: items_reindexed_estimate)
+      progress.total = resource_list(job.arguments.first, **pagination).sum(&:estimated_size)
     end
 
-    around_perform do |job, block|
-      job_log_entry = log_entry(job)
-      job_log_entry&.in_progress!
+    after_perform do
+      exhibit&.touch # rubocop:disable Rails/SkipsModelValidations
+    end
 
-      begin
-        block.call
-      rescue StandardError
-        job_log_entry&.failed!
-        raise
+    after_perform :commit
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def perform(exhibit_or_resources, start: nil, finish: nil, **)
+      errors = 0
+
+      error_handler = lambda do |pipeline, exception, _data|
+        job_tracker.append_log_entry(
+          type: :error,
+          exhibit: exhibit,
+          message: exception.to_s,
+          backtrace: exception.backtrace.first(5).join("\n"),
+          resource_id: (pipeline.source.id if pipeline.source.respond_to?(:id))
+        )
+        mark_job_as_failed!
+        errors += 1
       end
 
-      job_log_entry&.succeeded!
-    end
-
-    def self.perform_later(exhibit_or_resources, log_entry = nil)
-      validity_token = validity_checker.mint(exhibit_or_resources)
+      resource_list(exhibit_or_resources, start: start, finish: finish).each do |resource|
+        resource.reindex(touch: false, commit: false, job_tracker: job_tracker, additional_data: job_data, on_error: error_handler) do |*|
+          progress&.increment
+        end
+      rescue StandardError => e
+        error_handler.call(Struct.new(:source).new(resource), e, nil)
+      end
 
-      super(exhibit_or_resources, log_entry, validity_token)
+      job_tracker.append_log_entry(
+        type: :summary,
+        exhibit: exhibit,
+        message: I18n.t(
+          'spotlight.job_trackers.show.messages.status.in_progress',
+          progress: progress.progress,
+          total: progress.total,
+          errors: (I18n.t('spotlight.job_trackers.show.messages.errors', count: errors) if errors.positive?)
+        ),
+        progress: progress.progress, total: progress.total, errors: errors
+      )
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-    def perform(exhibit_or_resources, log_entry = nil, validity_token = nil)
-      return unless still_valid?(exhibit_or_resources, validity_token)
+    def exhibit
+      exhibit_or_resources = arguments.first
 
-      resource_list(exhibit_or_resources).each do |resource|
-        resource.reindex(log_entry)
+      case exhibit_or_resources
+      when Spotlight::Exhibit
+        exhibit_or_resources
+      when Spotlight::Resource
+        exhibit_or_resources.exhibit
       end
     end
 
     private
 
-    def resource_list(exhibit_or_resources)
-      if exhibit_or_resources.is_a?(Spotlight::Exhibit)
-        exhibit_or_resources.resources.find_each
-      elsif exhibit_or_resources.is_a?(Enumerable)
-        exhibit_or_resources
-      else
-        Array(exhibit_or_resources)
-      end
+    def commit
+      Blacklight.default_index.connection.commit
     end
 
-    def log_entry(job)
-      job.arguments.second if job.arguments.second.is_a?(Spotlight::ReindexingLogEntry)
-    end
+    def job_data
+      return unless job_tracker
 
-    def still_valid?(exhibit_or_resources, validity_token)
-      return true unless validity_token
+      @job_data ||= { Spotlight::Engine.config.job_tracker_id_field => job_tracker.top_level_job_tracker.job_id }
+    end
 
-      validity_checker.check exhibit_or_resources, validity_token
+    def resource_list(exhibit_or_resources, start: nil, finish: nil)
+      if exhibit_or_resources.is_a?(Spotlight::Exhibit)
+        exhibit_or_resources.resources.find_each(start: start, finish: finish)
+      else
+        Array(exhibit_or_resources)
+      end
     end
   end
 end