blacklight-spotlight 3.0.0.alpha.9 → 3.0.0.rc4

data/app/jobs/spotlight/reindex_exhibit_job.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ##
+  # Reindex an exhibit by parallelizing resource indexing into multiple batches of reindex jobs
+  class ReindexExhibitJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    include Spotlight::LimitConcurrency
+
+    def perform(exhibit, batch_size: Spotlight::Engine.config.reindexing_batch_size, batch_count: Spotlight::Engine.config.reindexing_batch_count, **)
+      job_tracker.update(status: 'in_progress')
+
+      count = exhibit.resources.count
+
+      # Use the provided batch size, or calculate a reasonable default
+      batch_count = (count.to_f / batch_size).ceil if batch_size
+      batch_count ||= 1 + Math.log(count).round # e.g. 10 => 3, 100 => 6, 1000 => 8
+
+      return Spotlight::ReindexJob.perform_now(exhibit, reports_on: job_tracker) if batch_count == 1
+
+      batch_size ||= (count.to_f / batch_count).ceil
+
+      perform_later_in_batches(exhibit, of: batch_size)
+    end
+
+    def perform_later_in_batches(exhibit, of:)
+      last = 0
+      exhibit.resources.select(:id).in_batches(of: of) do |batch|
+        last = batch.last.id
+        Spotlight::ReindexJob.perform_later(exhibit, reports_on: job_tracker, start: batch.first.id, finish: batch.last.id)
+      end
+
+      Spotlight::ReindexJob.perform_later(exhibit, reports_on: job_tracker, start: last)
+    end
+  end
+end

data/app/jobs/spotlight/reindex_job.rb

@@ -3,72 +3,80 @@
 module Spotlight
   ##
   # Reindex the given resources or exhibits
-  class ReindexJob < ActiveJob::Base
-    queue_as :default
-
-    # The validity checker is a seam for implementations to expire unnecessary
-    # indexing tasks if it becomes redundant while waiting in the job queue.
-    class_attribute :validity_checker, default: Spotlight::ValidityChecker.new
-    self.validity_checker ||= Spotlight::ValidityChecker.new if Rails.version < '5.2'
+  class ReindexJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
+    include Spotlight::LimitConcurrency
 
     before_perform do |job|
-      job_log_entry = log_entry(job)
-      next unless job_log_entry
+      pagination = job.arguments.last.slice(:start, :finish) if job.arguments.last.is_a? Hash
+      pagination ||= {}
 
-      items_reindexed_estimate = resource_list(job.arguments.first).sum do |resource|
-        resource.document_builder.documents_to_index.size
-      end
-      job_log_entry.update(items_reindexed_estimate: items_reindexed_estimate)
+      progress.total = resource_list(job.arguments.first, **pagination).sum(&:estimated_size)
+    end
+
+    after_perform do
+      exhibit&.touch # rubocop:disable Rails/SkipsModelValidations
     end
 
-    around_perform do |job, block|
-      job_log_entry = log_entry(job)
-      job_log_entry&.in_progress!
+    after_perform :commit
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def perform(exhibit_or_resources, start: nil, finish: nil, **)
+      job_tracker.update(status: 'in_progress')
+
+      errors = 0
+
+      error_handler = lambda do |pipeline, _error_context, exception, _data|
+        job_tracker.append_log_entry(type: :error, message: exception.to_s, resource_id: pipeline.source&.id)
+        errors += 1
+      end
 
-      begin
-        block.call
-      rescue StandardError
-        job_log_entry&.failed!
-        raise
+      resource_list(exhibit_or_resources, start: start, finish: finish).each do |resource|
+        resource.reindex(touch: false, commit: false, job_tracker: job_tracker, additional_data: job_data, on_error: error_handler) do |*|
+          progress&.increment
+        end
+      rescue StandardError => e
+        error_handler.call(Struct.new(:source).new(resource), self, e, nil)
       end
 
-      job_log_entry&.succeeded!
+      job_tracker.append_log_entry(type: :info, message: "#{progress.progress} of #{progress.total} (#{errors} errors)")
+      job_tracker.update(status: errors.zero? ? 'completed' : 'failed', data: { progress: progress.progress, total: progress.total })
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-    def self.perform_later(exhibit_or_resources, log_entry = nil)
-      validity_token = validity_checker.mint(exhibit_or_resources)
+    private
 
-      super(exhibit_or_resources, log_entry, validity_token)
+    def commit
+      Blacklight.default_index.connection.commit
     end
 
-    def perform(exhibit_or_resources, log_entry = nil, validity_token = nil)
-      return unless still_valid?(exhibit_or_resources, validity_token)
+    def job_data
+      return unless job_tracker
 
-      resource_list(exhibit_or_resources).each do |resource|
-        resource.reindex(log_entry)
-      end
+      { Spotlight::Engine.config.job_tracker_id_field => job_tracker.top_level_job_tracker.job_id }
     end
 
-    private
-
-    def resource_list(exhibit_or_resources)
+    def resource_list(exhibit_or_resources, start: nil, finish: nil)
       if exhibit_or_resources.is_a?(Spotlight::Exhibit)
-        exhibit_or_resources.resources.find_each
-      elsif exhibit_or_resources.is_a?(Enumerable)
-        exhibit_or_resources
+        exhibit_or_resources.resources.find_each(start: start, finish: finish)
       else
         Array(exhibit_or_resources)
       end
     end
 
-    def log_entry(job)
-      job.arguments.second if job.arguments.second.is_a?(Spotlight::ReindexingLogEntry)
+    def job_tracking_resource
+      exhibit
     end
 
-    def still_valid?(exhibit_or_resources, validity_token)
-      return true unless validity_token
+    def exhibit
+      exhibit_or_resources = arguments.first
 
-      validity_checker.check exhibit_or_resources, validity_token
+      case exhibit_or_resources
+      when Spotlight::Exhibit
+        exhibit_or_resources
+      when Spotlight::Resource
+        exhibit_or_resources.exhibit
+      end
     end
   end
 end

data/app/jobs/spotlight/rename_sidecar_field_job.rb

@@ -4,8 +4,8 @@ module Spotlight
   ##
   # After renaming an exhibit-specific field, we also
   # need to update the sidecars that may contain that field
-  class RenameSidecarFieldJob < ActiveJob::Base
-    queue_as :default
+  class RenameSidecarFieldJob < Spotlight::ApplicationJob
+    include Spotlight::JobTracking
 
     def perform(exhibit, old_field, new_field, old_slug = nil, new_slug = nil)
       exhibit.solr_document_sidecars.find_each do |s|

data/app/jobs/spotlight/update_job_trackers_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Spotlight
+  ###
+  # Calls the #set_default_thumbnail method
+  # on the object passed in and calls save
+  ###
+  class UpdateJobTrackersJob < Spotlight::ApplicationJob
+    def perform(job_tracker)
+      reports_on = job_tracker.on
+
+      return unless reports_on.is_a? Spotlight::JobTracker
+
+      reports_on.update(status: 'completed') if reports_on.job_trackers.all?(&:completed?)
+      reports_on.update(status: 'failed') if reports_on.job_trackers.any?(&:failed?)
+
+      reports_on.update(data: { progress: reports_on.job_trackers.sum(&:progress), total: reports_on.job_trackers.sum(&:total) })
+    end
+  end
+end

data/app/mailers/spotlight/indexing_complete_mailer.rb

@@ -5,9 +5,10 @@ module Spotlight
   # Notify the curator that we're finished processing a
   # batch upload
   class IndexingCompleteMailer < ActionMailer::Base
-    def documents_indexed(csv_data, exhibit, user)
-      @number = csv_data.length
+    def documents_indexed(csv_data, exhibit, user, indexed_count: nil, errors: [])
+      @number = indexed_count || csv_data.length
       @exhibit = exhibit
+      @errors = errors
       mail(to: user.email, subject: 'Document indexing complete')
     end
   end

data/app/models/concerns/spotlight/exhibit_defaults.rb

@@ -29,7 +29,7 @@ module Spotlight
     def initialize_browse
       return unless searches.blank?
 
-      searches.create title: 'All Exhibit Items',
+      searches.create title: 'All exhibit items',
                       long_description: 'All items in this exhibit.'
     end
 

data/app/models/concerns/spotlight/translatables.rb

@@ -15,8 +15,24 @@ module Spotlight
       # Set up a reader for the specified attribute that uses the I18n backend,
       # and defaults to the ActiveRecord value
       def define_translated_attr_reader(attr_name)
+        # Define a dynamic method for translating database-backed attributes,
+        # falling back to the database information as needed.
+        #
+        # Note: the empty string is provided as the final fallback to avoid i18n blowing
+        # up on nil attributes.
         define_method(:"#{attr_name}") do
-          I18n.translate(attr_name, scope: slug, default: attr_translation(attr_name))
+          send("translated_#{attr_name}", default: [attr_translation(attr_name), ''])
+        end
+
+        # Define an accessor that gets the value of the attribute in a given locale,
+        # returning `nil` for untranslated values.
+        #
+        # Note: For the default locale, we actually want to dig into the database,
+        # because that is the source of truth for the data.
+        define_method(:"translated_#{attr_name}") do |default: [], **options|
+          default = Array.wrap(default)
+          default.prepend(attr_translation(attr_name)) if I18n.locale == I18n.default_locale
+          I18n.translate(attr_name, scope: slug, default: default, **options).presence
         end
       end
     end

data/app/models/concerns/spotlight/user.rb

@@ -8,7 +8,6 @@ module Spotlight
     included do
       has_many :roles, class_name: 'Spotlight::Role', dependent: :destroy
       has_many :exhibits, class_name: 'Spotlight::Exhibit', through: :roles, source: 'resource', source_type: 'Spotlight::Exhibit'
-      has_many :reindexing_log_entries, class_name: 'Spotlight::ReindexingLogEntry'
 
       scope :with_roles, -> { where(id: Spotlight::Role.distinct.pluck(:user_id)) }
 
@@ -28,6 +27,8 @@ module Spotlight
     end
 
     def add_default_roles
+      return unless Spotlight::Engine.config.assign_default_roles_to_first_user
+
       roles.build role: 'admin', resource: Spotlight::Site.instance unless self.class.any?
     end
 

data/app/models/sir_trevor_rails/blocks/browse_group_categories_block.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SirTrevorRails
+  module Blocks
+    ##
+    # Browese Group Categories
+    class BrowseGroupCategoriesBlock < SirTrevorRails::Block
+      include Displayable
+
+      def display_item_counts?
+        send(:'display-item-counts') == 'true'
+      end
+
+      def groups
+        @groups ||= parent.exhibit.groups.published.where(slug: item_ids).sort do |a, b|
+          ordered_items.index(a.slug) <=> ordered_items.index(b.slug)
+        end
+      end
+
+      def groups?
+        !groups.empty?
+      end
+    end
+  end
+end

data/app/models/spotlight/ability.rb

@@ -25,6 +25,7 @@ module Spotlight
       can :manage, [
         Spotlight::Attachment,
         Spotlight::Search,
+        Spotlight::Group,
         Spotlight::Resource,
         Spotlight::Page,
         Spotlight::Contact,
@@ -43,6 +44,7 @@ module Spotlight
       can :read, Spotlight::Exhibit, published: true
       can :read, Spotlight::Page, published: true
       can :read, Spotlight::Search, published: true
+      can :read, Spotlight::Group, published: true
       can :read, Spotlight::Language, public: true
     end
     # rubocop:enable Metrics/AbcSize, Metrics/MethodLength

data/app/models/spotlight/about_page.rb

@@ -5,6 +5,8 @@ module Spotlight
   # About pages
   class AboutPage < Spotlight::Page
     extend FriendlyId
-    friendly_id :title, use: %i[slugged scoped finders history], scope: %i[exhibit locale]
+    friendly_id :title, use: %i[slugged scoped finders history], scope: %i[exhibit locale] do |config|
+      config.reserved_words&.concat(%w[update_all contacts])
+    end
   end
 end

data/app/models/spotlight/blacklight_configuration.rb

@@ -76,6 +76,7 @@ module Spotlight
 
         config.show.merge! show unless show.blank?
         config.index.merge! index unless index.blank?
+        config.index.respond_to[:iiif_json] = true
 
         config.index.thumbnail_field ||= Spotlight::Engine.config.thumbnail_field
 
@@ -353,7 +354,7 @@ module Spotlight
     def title_only_by_default?(view)
       return false if %i[show enabled].include?(view)
 
-      title_only = default_blacklight_config.view.send(:[], view).try(:title_only_by_default)
+      title_only = default_blacklight_config.view.send(:[], view)&.title_only_by_default
       title_only.nil? ? false : title_only
     end
 

data/app/models/spotlight/contact.rb

@@ -19,7 +19,7 @@ module Spotlight
       self.contact_info = contact_info.symbolize_keys
     end
 
-    before_save on: :create do
+    before_create do
       self.show_in_sidebar = true if show_in_sidebar.nil?
     end
 

data/app/models/spotlight/custom_field.rb

@@ -18,10 +18,10 @@ module Spotlight
       self.field_type ||= 'text'
     end
 
-    before_save :update_field_name, on: :update, if: -> { field_type_changed? || readonly_field_changed? }
+    before_update :update_field_name, if: -> { field_type_changed? || readonly_field_changed? }
 
-    after_commit :update_blacklight_configuration_after_field_name_change, on: :update, if: -> { saved_change_to_field? || saved_change_to_slug? }
-    after_commit :update_sidecar_data_after_field_name_change, on: :update, if: -> { saved_change_to_field? || saved_change_to_slug? }
+    after_update_commit :update_blacklight_configuration_after_field_name_change, if: -> { saved_change_to_field? || saved_change_to_slug? }
+    after_update_commit :update_sidecar_data_after_field_name_change, if: -> { saved_change_to_field? || saved_change_to_slug? }
 
     def label=(label)
       configuration['label'] = label

data/app/models/spotlight/event.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Spotlight
+  # Logged events for Spotlight exhibits
+  class Event < ActiveRecord::Base
+    belongs_to :resource, polymorphic: true
+    belongs_to :exhibit, optional: true
+
+    serialize :data
+
+    self.inheritance_column = :event_class
+  end
+end

data/app/models/spotlight/exhibit.rb

@@ -22,11 +22,16 @@ module Spotlight
     paginates_per 48
 
     extend FriendlyId
-    friendly_id :title, use: %i[slugged finders]
+    friendly_id :title, use: %i[slugged finders] do |config|
+      config.reserved_words&.concat(%w[site])
+    end
+
     validates :title, presence: true, if: -> { I18n.locale == I18n.default_locale }
     validates :slug, uniqueness: true
     validates :theme, inclusion: { in: Spotlight::Engine.config.exhibit_themes }, allow_blank: true
 
+    after_validation :move_friendly_id_error_to_slug
+
     acts_as_tagger
     acts_as_taggable
     delegate :blacklight_config, to: :blacklight_configuration
@@ -50,8 +55,9 @@ module Spotlight
     has_many :custom_search_fields, dependent: :delete_all
 
     has_many :feature_pages, -> { for_default_locale }, extend: FriendlyId::FinderMethods
+    has_many :groups, dependent: :delete_all
+    has_many :job_trackers, as: :on, dependent: :delete_all
     has_many :main_navigations, dependent: :delete_all
-    has_many :reindexing_log_entries, dependent: :destroy
     has_many :resources
     has_many :roles, as: :resource, dependent: :delete_all
     has_many :searches, dependent: :destroy, extend: FriendlyId::FinderMethods
@@ -70,7 +76,7 @@ module Spotlight
     belongs_to :masthead, dependent: :destroy, optional: true
     belongs_to :thumbnail, class_name: 'Spotlight::ExhibitThumbnail', dependent: :destroy, optional: true
 
-    accepts_nested_attributes_for :about_pages, :attachments, :contacts, :custom_fields, :feature_pages, :languages,
+    accepts_nested_attributes_for :about_pages, :attachments, :contacts, :custom_fields, :feature_pages, :groups, :languages,
                                   :main_navigations, :owned_taggings, :pages, :resources, :searches, :solr_document_sidecars, :translations
     accepts_nested_attributes_for :blacklight_configuration, :home_page, :filters, update_only: true
     accepts_nested_attributes_for :masthead, :thumbnail, update_only: true, reject_if: proc { |attr| attr['iiif_tilesource'].blank? }
@@ -100,8 +106,10 @@ module Spotlight
     end
 
     def import(hash)
-      Spotlight::ExhibitExportSerializer.prepare(self).from_hash(hash)
-      self
+      ActiveRecord::Base.transaction do
+        Spotlight::ExhibitImportExportService.new(self).from_hash!(hash)
+        save
+      end
     end
 
     def solr_data
@@ -110,8 +118,8 @@ module Spotlight
       end
     end
 
-    def reindex_later(user = nil)
-      Spotlight::ReindexJob.perform_later(self, new_reindexing_log_entry(user))
+    def reindex_later(current_user = nil)
+      Spotlight::ReindexExhibitJob.perform_later(self, user: current_user)
     end
 
     def uploaded_resource_fields
@@ -127,7 +135,7 @@ module Spotlight
     end
 
     def reindex_progress
-      @reindex_progress ||= ReindexProgress.new(current_reindexing_log_entry)
+      @reindex_progress ||= ReindexProgress.new(self)
     end
 
     def available_locales
@@ -140,14 +148,10 @@ module Spotlight
       self.description = ::Rails::Html::FullSanitizer.new.sanitize(description)
     end
 
-    def new_reindexing_log_entry(user = nil)
-      Spotlight::ReindexingLogEntry.create(exhibit: self, user: user, items_reindexed_count: 0, job_status: 'unstarted')
-    end
-
     private
 
-    def current_reindexing_log_entry
-      reindexing_log_entries.started_or_completed.first || reindexing_log_entries.build
+    def move_friendly_id_error_to_slug
+      errors.add :slug, *errors.delete(:friendly_id) if errors[:friendly_id].present?
     end
   end
 end