blacklight-spotlight 3.0.0.alpha.10 → 3.0.0.rc5

data/app/jobs/spotlight/reindex_exhibit_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ##
+  # Reindex an exhibit by parallelizing resource indexing into multiple batches of reindex jobs
+  class ReindexExhibitJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    with_job_tracking(resource: ->(job) { job.arguments.first })
+
+    include Spotlight::LimitConcurrency
+
+    def perform(exhibit, batch_size: Spotlight::Engine.config.reindexing_batch_size, batch_count: Spotlight::Engine.config.reindexing_batch_count, **)
+      count = exhibit.resources.count
+
+      # Use the provided batch size, or calculate a reasonable default
+      batch_count = (count.to_f / batch_size).ceil if batch_size
+      batch_count ||= 1 + Math.log(count).round # e.g. 10 => 3, 100 => 6, 1000 => 8
+
+      return Spotlight::ReindexJob.perform_now(exhibit, reports_on: job_tracker) if batch_count == 1
+
+      batch_size ||= (count.to_f / batch_count).ceil
+
+      perform_later_in_batches(exhibit, of: batch_size)
+
+      # mark the job as 'pending' and let the UpdateJobTrackersJob finalize this status after the ReindexJobs finish
+      job_tracker.update(status: 'pending')
+    end
+
+    def perform_later_in_batches(exhibit, of:)
+      last = 0
+      exhibit.resources.select(:id).in_batches(of: of) do |batch|
+        last = batch.last.id
+        Spotlight::ReindexJob.perform_later(exhibit, reports_on: job_tracker, start: batch.first.id, finish: batch.last.id)
+      end
+
+      Spotlight::ReindexJob.perform_later(exhibit, reports_on: job_tracker, start: last)
+    end
+  end
+end

data/app/jobs/spotlight/reindex_job.rb

@@ -3,72 +3,92 @@
 module Spotlight
   ##
   # Reindex the given resources or exhibits
-  class ReindexJob < ActiveJob::Base
-    queue_as :default
+  class ReindexJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    with_job_tracking(resource: ->(job) { job.exhibit })
 
-    # The validity checker is a seam for implementations to expire unnecessary
-    # indexing tasks if it becomes redundant while waiting in the job queue.
-    class_attribute :validity_checker, default: Spotlight::ValidityChecker.new
-    self.validity_checker ||= Spotlight::ValidityChecker.new if Rails.version < '5.2'
+    include Spotlight::LimitConcurrency
 
     before_perform do |job|
-      job_log_entry = log_entry(job)
-      next unless job_log_entry
+      pagination = job.arguments.last.slice(:start, :finish) if job.arguments.last.is_a? Hash
+      pagination ||= {}
 
-      items_reindexed_estimate = resource_list(job.arguments.first).sum do |resource|
-        resource.document_builder.documents_to_index.size
-      end
-      job_log_entry.update(items_reindexed_estimate: items_reindexed_estimate)
+      progress.total = resource_list(job.arguments.first, **pagination).sum(&:estimated_size)
     end
 
-    around_perform do |job, block|
-      job_log_entry = log_entry(job)
-      job_log_entry&.in_progress!
+    after_perform do
+      exhibit&.touch # rubocop:disable Rails/SkipsModelValidations
+    end
 
-      begin
-        block.call
-      rescue StandardError
-        job_log_entry&.failed!
-        raise
+    after_perform :commit
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def perform(exhibit_or_resources, start: nil, finish: nil, **)
+      errors = 0
+
+      error_handler = lambda do |pipeline, exception, _data|
+        job_tracker.append_log_entry(
+          type: :error,
+          exhibit: exhibit,
+          message: exception.to_s,
+          backtrace: exception.backtrace.first(5).join("\n"),
+          resource_id: (pipeline.source.id if pipeline.source.respond_to?(:id))
+        )
+        mark_job_as_failed!
+        errors += 1
       end
 
-      job_log_entry&.succeeded!
-    end
-
-    def self.perform_later(exhibit_or_resources, log_entry = nil)
-      validity_token = validity_checker.mint(exhibit_or_resources)
+      resource_list(exhibit_or_resources, start: start, finish: finish).each do |resource|
+        resource.reindex(touch: false, commit: false, job_tracker: job_tracker, additional_data: job_data, on_error: error_handler) do |*|
+          progress&.increment
+        end
+      rescue StandardError => e
+        error_handler.call(Struct.new(:source).new(resource), e, nil)
+      end
 
-      super(exhibit_or_resources, log_entry, validity_token)
+      job_tracker.append_log_entry(
+        type: :summary,
+        exhibit: exhibit,
+        message: I18n.t(
+          'spotlight.job_trackers.show.messages.status.in_progress',
+          progress: progress.progress,
+          total: progress.total,
+          errors: (I18n.t('spotlight.job_trackers.show.messages.errors', count: errors) if errors.positive?)
+        ),
+        progress: progress.progress, total: progress.total, errors: errors
+      )
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-    def perform(exhibit_or_resources, log_entry = nil, validity_token = nil)
-      return unless still_valid?(exhibit_or_resources, validity_token)
+    def exhibit
+      exhibit_or_resources = arguments.first
 
-      resource_list(exhibit_or_resources).each do |resource|
-        resource.reindex(log_entry)
+      case exhibit_or_resources
+      when Spotlight::Exhibit
+        exhibit_or_resources
+      when Spotlight::Resource
+        exhibit_or_resources.exhibit
       end
     end
 
     private
 
-    def resource_list(exhibit_or_resources)
-      if exhibit_or_resources.is_a?(Spotlight::Exhibit)
-        exhibit_or_resources.resources.find_each
-      elsif exhibit_or_resources.is_a?(Enumerable)
-        exhibit_or_resources
-      else
-        Array(exhibit_or_resources)
-      end
+    def commit
+      Blacklight.default_index.connection.commit
     end
 
-    def log_entry(job)
-      job.arguments.second if job.arguments.second.is_a?(Spotlight::ReindexingLogEntry)
-    end
+    def job_data
+      return unless job_tracker
 
-    def still_valid?(exhibit_or_resources, validity_token)
-      return true unless validity_token
+      @job_data ||= { Spotlight::Engine.config.job_tracker_id_field => job_tracker.top_level_job_tracker.job_id }
+    end
 
-      validity_checker.check exhibit_or_resources, validity_token
+    def resource_list(exhibit_or_resources, start: nil, finish: nil)
+      if exhibit_or_resources.is_a?(Spotlight::Exhibit)
+        exhibit_or_resources.resources.find_each(start: start, finish: finish)
+      else
+        Array(exhibit_or_resources)
+      end
     end
   end
 end

data/app/jobs/spotlight/remove_tags_job.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  class RemoveTagsJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    include Spotlight::GatherDocuments
+    with_job_tracking(resource: ->(job) { job.arguments.last[:exhibit] })
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def perform(solr_params:, exhibit:, tags:, **)
+      @errors = 0
+
+      each_document(solr_params, exhibit) do |document|
+        sidecar = document.sidecar(exhibit)
+        all_tags = sidecar.all_tags_list
+        all_tags -= tags
+        exhibit.tag(document.sidecar(exhibit), with: all_tags, on: :tags)
+        document.reindex(update_params: {})
+        progress&.increment
+      rescue StandardError => e
+        job_tracker.append_log_entry(type: :error, exhibit: exhibit, message: e.to_s)
+        @errors += 1
+        mark_job_as_failed!
+      end
+      exhibit.blacklight_config.repository.connection.commit
+      job_tracker.append_log_entry(type: :info, exhibit: exhibit, message: "#{progress.progress} of #{progress.total} (#{@errors} errors)")
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+  end
+end

data/app/jobs/spotlight/rename_sidecar_field_job.rb

@@ -4,8 +4,9 @@ module Spotlight
   ##
   # After renaming an exhibit-specific field, we also
   # need to update the sidecars that may contain that field
-  class RenameSidecarFieldJob < ActiveJob::Base
-    queue_as :default
+  class RenameSidecarFieldJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    with_job_tracking(resource: ->(job) { job.arguments.first })
 
     def perform(exhibit, old_field, new_field, old_slug = nil, new_slug = nil)
       exhibit.solr_document_sidecars.find_each do |s|

data/app/jobs/spotlight/rename_tags_job.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  class RenameTagsJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+
+    with_job_tracking(resource: ->(job) { job.arguments.first })
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def perform(exhibit, tag, to: nil, **)
+      @errors = 0
+
+      tag.taggings.owned_by(exhibit).find_each do |tagging|
+        sidecar = tagging.taggable
+        all_tags = sidecar.all_tags_list
+        all_tags -= [tag.name]
+        all_tags += [to] unless to.nil?
+
+        exhibit.tag(sidecar, with: all_tags, on: :tags)
+        sidecar.document.reindex(update_params: {})
+      rescue StandardError => e
+        job_tracker.append_log_entry(type: :error, exhibit: exhibit, message: e.to_s)
+        @errors += 1
+        mark_job_as_failed!
+      end
+
+      exhibit.blacklight_config.repository.connection.commit
+      job_tracker.append_log_entry(type: :info, exhibit: exhibit, message: "#{progress.progress} of #{progress.total} (#{@errors} errors)")
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+  end
+end

data/app/jobs/spotlight/update_job_trackers_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  # Calls the #set_default_thumbnail method
+  # on the object passed in and calls save
+  ###
+  class UpdateJobTrackersJob < Spotlight::ApplicationJob
+    def perform(job_tracker)
+      reports_on = job_tracker.on
+
+      return unless reports_on.is_a? Spotlight::JobTracker
+
+      reports_on.update(status: 'completed') if reports_on.job_trackers.all?(&:completed?)
+      reports_on.update(status: 'failed') if reports_on.job_trackers.any?(&:failed?)
+
+      reports_on.update(data: { progress: reports_on.job_trackers.sum(&:progress), total: reports_on.job_trackers.sum(&:total) })
+    end
+  end
+end

data/app/mailers/spotlight/indexing_complete_mailer.rb

@@ -5,9 +5,10 @@ module Spotlight
   # Notify the curator that we're finished processing a
   # batch upload
   class IndexingCompleteMailer < ActionMailer::Base
-    def documents_indexed(csv_data, exhibit, user)
-      @number = csv_data.length
+    def documents_indexed(csv_data, exhibit, user, indexed_count: nil, errors: [])
+      @number = indexed_count || csv_data.length
       @exhibit = exhibit
+      @errors = errors
       mail(to: user.email, subject: 'Document indexing complete')
     end
   end

data/app/models/concerns/spotlight/exhibit_analytics.rb

@@ -8,18 +8,14 @@ module Spotlight
       return OpenStruct.new unless analytics_provider&.enabled?
 
       @analytics ||= {}
-      @analytics[start_date] ||= begin
-        analytics_provider.exhibit_data(path || self, start_date: start_date.ago)
-      end
+      @analytics[start_date] ||= analytics_provider.exhibit_data(path || self, start_date: start_date.ago)
     end
 
     def page_analytics(start_date = 1.month, path = nil)
       return [] unless analytics_provider&.enabled?
 
       @page_analytics ||= {}
-      @page_analytics[start_date] ||= begin
-        analytics_provider.page_data(path || self, start_date: start_date.ago)
-      end
+      @page_analytics[start_date] ||= analytics_provider.page_data(path || self, start_date: start_date.ago)
     end
 
     def analytics_provider

data/app/models/concerns/spotlight/exhibit_defaults.rb

@@ -29,7 +29,7 @@ module Spotlight
     def initialize_browse
       return unless searches.blank?
 
-      searches.create title: 'All Exhibit Items',
+      searches.create title: 'All exhibit items',
                       long_description: 'All items in this exhibit.'
     end
 

data/app/models/concerns/spotlight/exhibit_documents.rb

@@ -23,12 +23,12 @@ module Spotlight
       end
     end
 
-    private
-
     def exhibit_search_builder
       blacklight_config.search_builder_class.new(exhibit_search_builder_context).except(:apply_permissive_visibility_filter)
     end
 
+    private
+
     def exhibit_search_builder_context
       OpenStruct.new(blacklight_config: blacklight_config.tap { |x| x.current_exhibit = self })
     end

data/app/models/concerns/spotlight/resources/web.rb

@@ -30,7 +30,7 @@ module Spotlight
       def self.fetch(url)
         Faraday.new(url) do |b|
           b.use FaradayMiddleware::FollowRedirects
-          b.adapter :net_http
+          b.adapter Faraday.default_adapter
         end.get
       end
     end

data/app/models/concerns/spotlight/solr_document.rb

@@ -24,7 +24,7 @@ module Spotlight
 
       def reindex(id)
         find(id).reindex
-      rescue Blacklight::Exceptions::InvalidSolrID => e
+      rescue Blacklight::Exceptions::RecordNotFound => e
         Rails.logger.debug "Unable to find document #{id}: #{e}"
       end
 

data/app/models/concerns/spotlight/solr_document/atomic_updates.rb

@@ -5,12 +5,12 @@ module Spotlight
     ##
     # Solr indexing strategy using Solr's Atomic Updates
     module AtomicUpdates
-      def reindex
+      def reindex(update_params: { commitWithin: 500 })
         return unless write?
 
         data = hash_for_solr_update(to_solr)
 
-        blacklight_solr.update params: { commitWithin: 500 }, data: data.to_json, headers: { 'Content-Type' => 'application/json' } unless data.empty?
+        blacklight_solr.update params: update_params, data: data.to_json, headers: { 'Content-Type' => 'application/json' } unless data.empty?
       end
 
       def write?

data/app/models/concerns/spotlight/user.rb

@@ -8,7 +8,6 @@ module Spotlight
     included do
       has_many :roles, class_name: 'Spotlight::Role', dependent: :destroy
       has_many :exhibits, class_name: 'Spotlight::Exhibit', through: :roles, source: 'resource', source_type: 'Spotlight::Exhibit'
-      has_many :reindexing_log_entries, class_name: 'Spotlight::ReindexingLogEntry'
 
       scope :with_roles, -> { where(id: Spotlight::Role.distinct.pluck(:user_id)) }
 
@@ -20,7 +19,11 @@ module Spotlight
     end
 
     def exhibit_roles
-      roles.where(resource_type: 'Spotlight::Exhibit')
+      roles.where(resource_type: 'Spotlight::Exhibit').where.not(role: 'viewer')
+    end
+
+    def viewer_roles
+      roles.where(resource_type: 'Spotlight::Exhibit', role: 'viewer')
     end
 
     def admin_roles
@@ -28,6 +31,8 @@ module Spotlight
     end
 
     def add_default_roles
+      return unless Spotlight::Engine.config.assign_default_roles_to_first_user
+
       roles.build role: 'admin', resource: Spotlight::Site.instance unless self.class.any?
     end
 

data/app/models/sir_trevor_rails/blocks/browse_group_categories_block.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SirTrevorRails
+  module Blocks
+    ##
+    # Browese Group Categories
+    class BrowseGroupCategoriesBlock < SirTrevorRails::Block
+      include Displayable
+
+      def display_item_counts?
+        send(:'display-item-counts') == 'true'
+      end
+
+      def groups
+        @groups ||= parent.exhibit.groups.published.where(slug: item_ids).sort do |a, b|
+          ordered_items.index(a.slug) <=> ordered_items.index(b.slug)
+        end
+      end
+
+      def groups?
+        !groups.empty?
+      end
+    end
+  end
+end

data/app/models/sir_trevor_rails/blocks/uploaded_items_block.rb

@@ -11,6 +11,10 @@ module SirTrevorRails
       def files
         (item || {}).map { |_, file| file }.select { |file| file[:display].to_s == 'true' }
       end
+
+      def zpr_link?
+        zpr_link == 'true'
+      end
     end
   end
 end