blacklight-spotlight 3.0.0.rc3 → 3.0.0.rc4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8fb1d422302a15b42343a0f7afb8c545869d0c872e355762a7cea998469b12b
-  data.tar.gz: df63403ced5aa975097224a0e298562d216c427fcab43aba7640eeee18dc8ea9
+  metadata.gz: 55231336b3b4a0eae1abdd68bbeb8ede968d5ffbe4359f29f83a3a27124a4b05
+  data.tar.gz: b2f2bfc6230b5081493484393a26670961486217b64b7a96b9659979eab64381
 SHA512:
-  metadata.gz: 3c6261c2edc4b04d331d0e3961f0a88ce15599314add02061475349e9c86cb5dcb7c1cb5bd9d89100266bfcbab4449160b7b662d66aece1d73e5a721d6d35914
-  data.tar.gz: 96fad663f5370e18de8a345ed3bbd9f7b6e7b774a76ca2d16342d66bdfa2bde41fe3eaef70b290bde32e35828f50bc3c71f0f7f75df20e8029c61fff29193ab1
+  metadata.gz: c3d2a9d1203e0950c79844e4fad0b2083e3cc8d1b63a357ce47a2760ac48b13ec9c2f4e2f0890add8a533d71f61bf7a96a684b74710c564b3e17d919635b592a
+  data.tar.gz: 842054ed3d1c5d46c5c6f9bb3180efdf83feee8f1bf44ae607865d6bea2f9d4589e1b19990cbb53a65ba4249fd5794dca213830be71b055a28b69e188b29dde7

data/app/assets/javascripts/spotlight/admin/reindex_monitor.js

@@ -84,6 +84,7 @@ Spotlight.onLoad(function() {
     }
 
     function calculatePercentage(data) {
+      if (data.total == 0) return 0;
       return Math.floor((data.completed / data.total) * 100);
     }
 

data/app/assets/stylesheets/spotlight/browse_group_categories_block.scss

@@ -4,6 +4,24 @@
   padding-bottom: $spacer * .75;
   padding-top: $spacer * .75;
   
+  .browse-categories .browse-category {
+    .category-caption {
+      z-index: 100;
+    }
+
+    .hover-overlay {
+      height: 100%;
+      position: absolute;
+      width: 100%;
+    }
+
+    &:hover {
+      .hover-overlay {
+        background: linear-gradient(to bottom, rgba(0, 0, 0, 0.2), rgba(0, 0, 0, 0.5), rgba(0, 0, 0, 0.7));
+      }
+    }
+  }
+  
   .spotlight-flexbox.browse-categories .box {
     display: flex;
   }
@@ -66,4 +84,9 @@
   .browse-group-categories-block .blacklight-icon-arrow-alt-circle-left, .blacklight-icon-arrow-alt-circle-right{
     transform: rotate(180deg);
   }
+  // Hack to override overflow issue not fixed in RTL upstream in tiny-slider
+  .tns-visually-hidden {
+    left: 0;
+    right: -10000em;
+  }
 }

data/app/controllers/spotlight/catalog_controller.rb

@@ -31,7 +31,10 @@ module Spotlight
       blacklight_config.view.admin_table.document_actions = []
       blacklight_config.track_search_session = false
 
-      blacklight_config.add_sort_field :timestamp, sort: "#{blacklight_config.index.timestamp_field} desc" unless blacklight_config.sort_fields.key? :timestamp
+      unless blacklight_config.sort_fields.key? :timestamp
+        blacklight_config.add_sort_field :timestamp, default: true,
+                                                     sort: "#{blacklight_config.index.timestamp_field} desc"
+      end
     end
 
     before_action only: :edit do

data/app/controllers/spotlight/dashboards_controller.rb

@@ -22,7 +22,7 @@ module Spotlight
 
       @pages = @exhibit.pages.recent.limit(5)
       @solr_documents = load_recent_solr_documents 5
-      @recent_reindexing = @exhibit.reindexing_log_entries.recent
+      @recent_reindexing = @exhibit.job_trackers.recent
 
       attach_dashboard_breadcrumbs
     end

data/app/controllers/spotlight/exhibits_controller.rb

@@ -27,7 +27,7 @@ module Spotlight
     end
 
     def process_import
-      if @exhibit.import(JSON.parse(import_exhibit_params.read)) && @exhibit.reindex_later
+      if @exhibit.import(JSON.parse(import_exhibit_params.read)) && @exhibit.reindex_later(current_user)
         redirect_to spotlight.exhibit_dashboard_path(@exhibit), notice: t(:'helpers.submit.exhibit.updated', model: @exhibit.class.model_name.human.downcase)
       else
         render action: :import

data/app/helpers/spotlight/application_helper.rb

@@ -30,6 +30,25 @@ module Spotlight
       current_site.title if current_site.title.present?
     end
 
+    # Returns the url for the current page in the new locale. This may be
+    # overridden in downstream applications where our naive use of `url_for`
+    # is insufficient to generate the expected routes
+    def current_page_for_locale(locale)
+      initial_exception = nil
+
+      ([self] + additional_locale_routing_scopes).each do |scope|
+        return scope.public_send(:url_for, params.to_unsafe_h.merge(locale: locale))
+      rescue ActionController::UrlGenerationError => e
+        initial_exception ||= e
+      end
+
+      raise initial_exception
+    end
+
+    def additional_locale_routing_scopes
+      [spotlight, main_app]
+    end
+
     # Can search for named routes directly in the main app, omitting
     # the "main_app." prefix
     def method_missing(method, *args, &block)

data/app/helpers/spotlight/pages_helper.rb

@@ -21,7 +21,7 @@ module Spotlight
     # a more complete markdown rendered
     def sir_trevor_markdown(text)
       clean_text = if text
-                     text.gsub('<br>', "\n").gsub('<p>', '').gsub('</p>', "\n\n")
+                     text.gsub('<br>', "\n\n").gsub('<p>', '').gsub('</p>', "\n\n")
                    else
                      ''
                    end

data/app/jobs/concerns/spotlight/job_tracking.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # Job status tracking
+  module JobTracking
+    extend ActiveSupport::Concern
+    include ActiveJob::Status
+
+    def self.with_job_tracking
+      before_perform :find_or_initialize_job_tracker
+      after_perform :finalize_job_tracker
+    end
+
+    def job_tracker
+      @job_tracker ||= find_or_initialize_job_tracker
+    end
+
+    private
+
+    def find_or_initialize_job_tracker
+      JobTracker.find_or_create_by(job_id: job_id) do |tracker|
+        tracker.job_class = self.class.name
+        tracker.status = 'enqueued'
+        update_job_tracker_properties(tracker)
+      end
+    end
+
+    def finalize_job_tracker
+      job_tracker.update(status: 'completed') if job_tracker.status == 'enqueued'
+    end
+
+    def update_job_tracker_properties(tracker)
+      tracker.resource = job_tracking_resource
+      tracker.on = reports_on_resource || tracker.resource
+
+      tracker.user = arguments.last[:user] if arguments.last.is_a?(Hash)
+    end
+
+    def job_tracking_resource
+      arguments.first
+    end
+
+    def reports_on_resource
+      arguments.last[:reports_on] if arguments.last.is_a?(Hash)
+    end
+  end
+end

data/app/jobs/concerns/spotlight/limit_concurrency.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # Job status tracking
+  module LimitConcurrency
+    extend ActiveSupport::Concern
+
+    VALIDITY_TOKEN_PARAMETER = 'validity_token'
+
+    included do
+      # The validity checker is a seam for implementations to expire unnecessary
+      # indexing tasks if it becomes redundant while waiting in the job queue.
+      class_attribute :validity_checker, default: Spotlight::ValidityChecker.new
+
+      before_enqueue do |job|
+        token = job.arguments.last[VALIDITY_TOKEN_PARAMETER] if job.arguments.last.is_a?(Hash)
+        token ||= validity_checker.mint(job)
+
+        job.arguments << {} unless job.arguments.last.is_a? Hash
+        job.arguments.last[VALIDITY_TOKEN_PARAMETER] = token
+      end
+
+      before_perform do |job|
+        next unless job.arguments.last.is_a?(Hash)
+
+        token = job.arguments.last.delete(VALIDITY_TOKEN_PARAMETER)
+        throw(:abort) unless token.nil? || validity_checker.check(job, validity_token: token)
+
+        job.arguments.pop if job.arguments.last.empty?
+      end
+    end
+  end
+end

data/app/jobs/spotlight/add_uploads_from_csv.rb

@@ -3,12 +3,11 @@
 module Spotlight
   ##
   # Process a CSV upload into new Spotlight::Resource::Upload objects
-  class AddUploadsFromCsv < ActiveJob::Base
+  class AddUploadsFromCsv < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
     attr_reader :count
     attr_reader :errors
 
-    queue_as :default
-
     after_perform do |job|
       csv_data, exhibit, user = job.arguments
       Spotlight::IndexingCompleteMailer.documents_indexed(
@@ -59,5 +58,9 @@ module Spotlight
         end.compact.to_h
       end.compact
     end
+
+    def job_tracking_resource
+      arguments[1]
+    end
   end
 end

data/app/jobs/spotlight/application_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # :nodoc:
+  class ApplicationJob < ActiveJob::Base
+    queue_as :default
+  end
+end

data/app/jobs/spotlight/cleanup_job_trackers_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  # Calls the #set_default_thumbnail method
+  # on the object passed in and calls save
+  ###
+  class CleanupJobTrackersJob < Spotlight::ApplicationJob
+    def perform
+      Spotlight::JobTracker.where(status: 'completed', updated_at: Time.zone.at(0)...Spotlight::Engine.config.reindex_progress_window.minutes.ago).delete_all
+    end
+  end
+end

data/app/jobs/spotlight/default_thumbnail_job.rb

@@ -5,9 +5,7 @@ module Spotlight
   # Calls the #set_default_thumbnail method
   # on the object passed in and calls save
   ###
-  class DefaultThumbnailJob < ActiveJob::Base
-    queue_as :default
-
+  class DefaultThumbnailJob < Spotlight::ApplicationJob
     def perform(thumbnailable)
       thumbnailable.set_default_thumbnail
       thumbnailable.save

data/app/jobs/spotlight/reindex_exhibit_job.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ##
+  # Reindex an exhibit by parallelizing resource indexing into multiple batches of reindex jobs
+  class ReindexExhibitJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    include Spotlight::LimitConcurrency
+
+    def perform(exhibit, batch_size: Spotlight::Engine.config.reindexing_batch_size, batch_count: Spotlight::Engine.config.reindexing_batch_count, **)
+      job_tracker.update(status: 'in_progress')
+
+      count = exhibit.resources.count
+
+      # Use the provided batch size, or calculate a reasonable default
+      batch_count = (count.to_f / batch_size).ceil if batch_size
+      batch_count ||= 1 + Math.log(count).round # e.g. 10 => 3, 100 => 6, 1000 => 8
+
+      return Spotlight::ReindexJob.perform_now(exhibit, reports_on: job_tracker) if batch_count == 1
+
+      batch_size ||= (count.to_f / batch_count).ceil
+
+      perform_later_in_batches(exhibit, of: batch_size)
+    end
+
+    def perform_later_in_batches(exhibit, of:)
+      last = 0
+      exhibit.resources.select(:id).in_batches(of: of) do |batch|
+        last = batch.last.id
+        Spotlight::ReindexJob.perform_later(exhibit, reports_on: job_tracker, start: batch.first.id, finish: batch.last.id)
+      end
+
+      Spotlight::ReindexJob.perform_later(exhibit, reports_on: job_tracker, start: last)
+    end
+  end
+end

data/app/jobs/spotlight/reindex_job.rb

@@ -3,72 +3,80 @@
 module Spotlight
   ##
   # Reindex the given resources or exhibits
-  class ReindexJob < ActiveJob::Base
-    queue_as :default
-
-    # The validity checker is a seam for implementations to expire unnecessary
-    # indexing tasks if it becomes redundant while waiting in the job queue.
-    class_attribute :validity_checker, default: Spotlight::ValidityChecker.new
-    self.validity_checker ||= Spotlight::ValidityChecker.new if Rails.version < '5.2'
+  class ReindexJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    include Spotlight::LimitConcurrency
 
     before_perform do |job|
-      job_log_entry = log_entry(job)
-      next unless job_log_entry
+      pagination = job.arguments.last.slice(:start, :finish) if job.arguments.last.is_a? Hash
+      pagination ||= {}
 
-      items_reindexed_estimate = resource_list(job.arguments.first).sum do |resource|
-        resource.document_builder.documents_to_index.size
-      end
-      job_log_entry.update(items_reindexed_estimate: items_reindexed_estimate)
+      progress.total = resource_list(job.arguments.first, **pagination).sum(&:estimated_size)
+    end
+
+    after_perform do
+      exhibit&.touch # rubocop:disable Rails/SkipsModelValidations
     end
 
-    around_perform do |job, block|
-      job_log_entry = log_entry(job)
-      job_log_entry&.in_progress!
+    after_perform :commit
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def perform(exhibit_or_resources, start: nil, finish: nil, **)
+      job_tracker.update(status: 'in_progress')
+
+      errors = 0
+
+      error_handler = lambda do |pipeline, _error_context, exception, _data|
+        job_tracker.append_log_entry(type: :error, message: exception.to_s, resource_id: pipeline.source&.id)
+        errors += 1
+      end
 
-      begin
-        block.call
-      rescue StandardError
-        job_log_entry&.failed!
-        raise
+      resource_list(exhibit_or_resources, start: start, finish: finish).each do |resource|
+        resource.reindex(touch: false, commit: false, job_tracker: job_tracker, additional_data: job_data, on_error: error_handler) do |*|
+          progress&.increment
+        end
+      rescue StandardError => e
+        error_handler.call(Struct.new(:source).new(resource), self, e, nil)
       end
 
-      job_log_entry&.succeeded!
+      job_tracker.append_log_entry(type: :info, message: "#{progress.progress} of #{progress.total} (#{errors} errors)")
+      job_tracker.update(status: errors.zero? ? 'completed' : 'failed', data: { progress: progress.progress, total: progress.total })
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-    def self.perform_later(exhibit_or_resources, log_entry = nil)
-      validity_token = validity_checker.mint(exhibit_or_resources)
+    private
 
-      super(exhibit_or_resources, log_entry, validity_token)
+    def commit
+      Blacklight.default_index.connection.commit
     end
 
-    def perform(exhibit_or_resources, log_entry = nil, validity_token = nil)
-      return unless still_valid?(exhibit_or_resources, validity_token)
+    def job_data
+      return unless job_tracker
 
-      resource_list(exhibit_or_resources).each do |resource|
-        resource.reindex(log_entry)
-      end
+      { Spotlight::Engine.config.job_tracker_id_field => job_tracker.top_level_job_tracker.job_id }
     end
 
-    private
-
-    def resource_list(exhibit_or_resources)
+    def resource_list(exhibit_or_resources, start: nil, finish: nil)
       if exhibit_or_resources.is_a?(Spotlight::Exhibit)
-        exhibit_or_resources.resources.find_each
-      elsif exhibit_or_resources.is_a?(Enumerable)
-        exhibit_or_resources
+        exhibit_or_resources.resources.find_each(start: start, finish: finish)
       else
         Array(exhibit_or_resources)
       end
     end
 
-    def log_entry(job)
-      job.arguments.second if job.arguments.second.is_a?(Spotlight::ReindexingLogEntry)
+    def job_tracking_resource
+      exhibit
     end
 
-    def still_valid?(exhibit_or_resources, validity_token)
-      return true unless validity_token
+    def exhibit
+      exhibit_or_resources = arguments.first
 
-      validity_checker.check exhibit_or_resources, validity_token
+      case exhibit_or_resources
+      when Spotlight::Exhibit
+        exhibit_or_resources
+      when Spotlight::Resource
+        exhibit_or_resources.exhibit
+      end
     end
   end
 end

data/app/jobs/spotlight/rename_sidecar_field_job.rb

@@ -4,8 +4,8 @@ module Spotlight
   ##
   # After renaming an exhibit-specific field, we also
   # need to update the sidecars that may contain that field
-  class RenameSidecarFieldJob < ActiveJob::Base
-    queue_as :default
+  class RenameSidecarFieldJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
 
     def perform(exhibit, old_field, new_field, old_slug = nil, new_slug = nil)
       exhibit.solr_document_sidecars.find_each do |s|