blacklight-spotlight 0.31.0 → 0.32.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9129ce334891bf7bc19eda0bd1a59b3431e7914c
-  data.tar.gz: fdf77ce53166788e4b32db54dc9e0b0a2c577010
+  metadata.gz: d1fd2b950f32f603a3ee0f2db98c3b91e5cad5f9
+  data.tar.gz: 6cb02f70515f59a51422ef7cf1309216351b091f
 SHA512:
-  metadata.gz: 71aff72f8a8e7b63b7cd4d21871b116d9afa35f76113218e296364fe6119a840134932062c315ee0aa969289662639b33c26ce6615224058aa551c9eca4609e9
-  data.tar.gz: 8c72942251aadf9def1efa373cbbf3eb9b97a4454cbf84ac9e5745c78a3522f57b64a93c732b26bc8a5b44ac0ce195178088df6cbc4fe28272e1d77c0561064f
+  metadata.gz: 113fc7cd7d398ecde9d12c8428aeee286b0604267ac11e4222656766760620a11f800a33484d6c1878590f94f367443794d9baa49facaeca9adfb318a26c6a04
+  data.tar.gz: c29d9d9a84bedd527d3c63fdc0fa6d2c9ca648d69f54eda3b0b1be77843649329efc526832e6bdc8a98570b9ca8ac02627433c9300f675640eba1469d165b092

data/app/controllers/spotlight/dashboards_controller.rb

@@ -18,6 +18,7 @@ module Spotlight
 
       @pages = @exhibit.pages.recent.limit(5)
       @solr_documents = load_recent_solr_documents 5
+      @recent_reindexing = @exhibit.reindexing_log_entries.recent
 
       attach_dashboard_breadcrumbs
     end

data/app/controllers/spotlight/resources_controller.rb

@@ -35,7 +35,7 @@ module Spotlight
     end
 
     def reindex_all
-      @exhibit.reindex_later
+      @exhibit.reindex_later current_user
 
       redirect_to admin_exhibit_catalog_path(@exhibit), notice: t(:'spotlight.resources.reindexing_in_progress')
     end

data/app/helpers/spotlight/browse_helper.rb

@@ -3,6 +3,8 @@ module Spotlight
   # Helper for browse views
   module BrowseHelper
     include ::BlacklightConfigurationHelper
+    include Spotlight::RenderingHelper
+
     def document_index_view_type
       if @search && @search.default_index_view_type.present? && params[:view].blank?
         blacklight_config.view[@search.default_index_view_type].key

data/app/helpers/spotlight/pages_helper.rb

@@ -2,6 +2,8 @@ module Spotlight
   ##
   # Sir-trevor helpers methods
   module PagesHelper
+    include Spotlight::RenderingHelper
+
     ##
     # Override the default #sir_trevor_markdown so we can use
     # a more complete markdown rendered
@@ -12,7 +14,7 @@ module Spotlight
                      ''
                    end
 
-      GitHub::Markup.render('.md', clean_text).html_safe
+      render_markdown(clean_text)
     end
 
     def available_index_fields

data/app/helpers/spotlight/rendering_helper.rb

@@ -0,0 +1,7 @@
+module Spotlight
+  module RenderingHelper # :nodoc:
+    def render_markdown(text)
+      GitHub::Markup.render('.md', text).html_safe
+    end
+  end
+end

data/app/jobs/spotlight/reindex_job.rb

@@ -4,12 +4,34 @@ module Spotlight
   class ReindexJob < ActiveJob::Base
     queue_as :default
 
-    before_enqueue do |job|
-      resource_list(job.arguments.first).each(&:waiting!)
+    before_perform do |job|
+      job_log_entry = log_entry(job)
+      next unless job_log_entry
+
+      items_reindexed_estimate = resource_list(job.arguments.first).sum do |resource|
+        resource.document_builder.documents_to_index.size
+      end
+      job_log_entry.update(items_reindexed_estimate: items_reindexed_estimate)
+    end
+
+    around_perform do |job, block|
+      job_log_entry = log_entry(job)
+      job_log_entry.in_progress! if job_log_entry
+
+      begin
+        block.call
+      rescue
+        job_log_entry.failed! if job_log_entry
+        raise
+      end
+
+      job_log_entry.succeeded! if job_log_entry
     end
 
-    def perform(exhibit_or_resources)
-      resource_list(exhibit_or_resources).each(&:reindex)
+    def perform(exhibit_or_resources, log_entry = nil)
+      resource_list(exhibit_or_resources).each do |resource|
+        resource.reindex(log_entry)
+      end
     end
 
     private
@@ -23,5 +45,9 @@ module Spotlight
         Array(exhibit_or_resources)
       end
     end
+
+    def log_entry(job)
+      job.arguments.second if job.arguments.second.is_a?(Spotlight::ReindexingLogEntry)
+    end
   end
 end

data/app/models/concerns/spotlight/solr_document.rb

@@ -86,7 +86,7 @@ module Spotlight
     end
 
     def to_solr
-      { self.class.unique_key.to_sym => id }.reverse_merge(sidecars.inject({}) { |a, e| a.merge(e.to_solr) }).merge(tags_to_solr)
+      { self.class.unique_key.to_sym => id }.reverse_merge(sidecars.inject({}) { |acc, elem| acc.merge(elem.to_solr) }).merge(tags_to_solr)
     end
 
     def make_public!(exhibit)

data/app/models/concerns/spotlight/user.rb

@@ -6,6 +6,7 @@ module Spotlight
     included do
       has_many :roles, class_name: 'Spotlight::Role', dependent: :destroy
       has_many :exhibits, class_name: 'Spotlight::Exhibit', through: :roles, source: 'resource', source_type: 'Spotlight::Exhibit'
+      has_many :reindexing_log_entries, class_name: 'Spotlight::ReindexingLogEntry'
 
       scope :with_roles, -> { where(id: Spotlight::Role.distinct.pluck(:user_id)) }
 

data/app/models/spotlight/exhibit.rb

@@ -32,6 +32,7 @@ module Spotlight
     has_many :feature_pages, extend: FriendlyId::FinderMethods
     has_many :main_navigations, dependent: :delete_all
     has_many :owned_taggings, class_name: 'ActsAsTaggableOn::Tagging', as: :tagger
+    has_many :reindexing_log_entries, dependent: :destroy
     has_many :resources
     has_many :roles, as: :resource, dependent: :delete_all
     has_many :searches, dependent: :destroy, extend: FriendlyId::FinderMethods
@@ -79,8 +80,8 @@ module Spotlight
       end
     end
 
-    def reindex_later
-      Spotlight::ReindexJob.perform_later(self)
+    def reindex_later(user = nil)
+      Spotlight::ReindexJob.perform_later(self, new_reindexing_log_entry(user))
     end
 
     def uploaded_resource_fields
@@ -100,7 +101,7 @@ module Spotlight
     end
 
     def reindex_progress
-      @reindex_progress ||= ReindexProgress.new(resources) if resources
+      @reindex_progress ||= ReindexProgress.new(self)
     end
 
     protected
@@ -108,5 +109,9 @@ module Spotlight
     def sanitize_description
       self.description = ::Rails::Html::FullSanitizer.new.sanitize(description)
     end
+
+    def new_reindexing_log_entry(user = nil)
+      Spotlight::ReindexingLogEntry.create(exhibit: self, user: user, items_reindexed_count: 0, job_status: 'unstarted')
+    end
   end
 end

data/app/models/spotlight/reindex_progress.rb

@@ -2,51 +2,47 @@ module Spotlight
   ##
   # ReindexProgress is a class that models the progress of reindexing a list of resources
   class ReindexProgress
-    def initialize(resource_list)
-      @resources = if resource_list.present?
-                     resource_list.order('updated_at')
-                   else
-                     Spotlight::Resource.none
-                   end
+    attr_reader :exhibit
+
+    def initialize(exhibit)
+      @exhibit = exhibit
     end
 
     def recently_in_progress?
-      any_waiting? || (!!finished_at && finished_at > Spotlight::Engine.config.reindex_progress_window.minutes.ago)
+      return false if current_log_entry.blank?
+      return true if current_log_entry.in_progress?
+
+      current_log_entry.end_time.present? && (current_log_entry.end_time > Spotlight::Engine.config.reindex_progress_window.minutes.ago)
     end
 
     def started_at
-      return unless resources.present?
-
-      enqueued_resources = resources.select(&:enqueued_at?)
-
-      return unless enqueued_resources.any?
-
-      @started ||= enqueued_resources.min_by(&:enqueued_at).enqueued_at
+      current_log_entry.try(:start_time)
     end
 
     def updated_at
-      @updated ||= resources.maximum(:updated_at) || started_at
+      current_log_entry.try(:updated_at)
     end
 
     def finished?
-      completed_resources.present? && !any_waiting?
+      return false if current_log_entry.blank?
+      current_log_entry.succeeded? || current_log_entry.failed?
     end
 
     def finished_at
-      return unless finished?
-      @finished ||= completed_resources.max_by(&:last_indexed_finished).last_indexed_finished
+      current_log_entry.try(:end_time)
     end
 
     def total
-      @total ||= resources.map(&:last_indexed_estimate).compact.sum
+      current_log_entry.try(:items_reindexed_estimate)
     end
 
     def completed
-      @completed ||= resources.map(&:last_indexed_count).compact.sum
+      current_log_entry.try(:items_reindexed_count)
     end
 
     def errored?
-      resources.any?(&:errored?)
+      return false if current_log_entry.blank?
+      current_log_entry.failed?
     end
 
     def as_json(*)
@@ -63,10 +59,8 @@ module Spotlight
 
     private
 
-    attr_reader :resources
-
-    def any_waiting?
-      resources.any?(&:waiting?)
+    def current_log_entry
+      exhibit.reindexing_log_entries.where.not(job_status: 'unstarted').first
     end
 
     def localized_start_time
@@ -83,9 +77,5 @@ module Spotlight
       return unless updated_at
       I18n.l(updated_at, format: :short)
     end
-
-    def completed_resources
-      resources.completed
-    end
   end
 end

data/app/models/spotlight/reindexing_log_entry.rb

@@ -0,0 +1,39 @@
+module Spotlight
+  ##
+  # a log entry representing an attempt to reindex some number of records in an exhibit
+  class ReindexingLogEntry < ActiveRecord::Base
+    enum job_status: { unstarted: 0, in_progress: 1, succeeded: 2, failed: 3 }
+
+    belongs_to :exhibit, class_name: 'Spotlight::Exhibit'
+    belongs_to :user, class_name: '::User'
+
+    # null start times sort to the top, to more easily surface pending reindexing
+    default_scope { order('start_time IS NOT NULL, start_time DESC') }
+    scope :recent, -> { limit(5) }
+
+    def duration
+      end_time - start_time if end_time
+    end
+
+    def in_progress!
+      self.start_time = Time.zone.now
+      super
+    rescue
+      Rails.logger.error "unexpected error updating log entry to :in_progress from #{caller}"
+    end
+
+    def succeeded!
+      self.end_time = Time.zone.now
+      super
+    rescue
+      Rails.logger.error "unexpected error updating log entry to :succeeded from #{caller}"
+    end
+
+    def failed!
+      self.end_time = Time.zone.now
+      super
+    rescue
+      Rails.logger.error "unexpected error updating log entry to :failed from #{caller}"
+    end
+  end
+end

data/app/models/spotlight/resource.rb

@@ -16,19 +16,9 @@ module Spotlight
     has_many :solr_document_sidecars
 
     serialize :data, Hash
-    store :metadata, accessors: [
-      :enqueued_at,
-      :last_indexed_estimate,
-      :last_indexed_count,
-      :last_index_elapsed_time,
-      :last_indexed_finished
-    ], coder: JSON
 
-    enum index_status: [:waiting, :completed, :errored]
-
-    around_index :reindex_with_logging
     after_index :commit
-    after_index :completed!
+    after_index :touch_exhibit!
 
     ##
     # Persist the record to the database, and trigger a reindex to solr
@@ -41,27 +31,9 @@ module Spotlight
     ##
     # Enqueue an asynchronous reindexing job for this resource
     def reindex_later
-      waiting!
       Spotlight::ReindexJob.perform_later(self)
     end
 
-    def waiting!
-      update(enqueued_at: Time.zone.now)
-      super
-    end
-
-    def enqueued_at
-      cast_to_date_time(super)
-    end
-
-    def enqueued_at?
-      enqueued_at.present?
-    end
-
-    def last_indexed_finished
-      cast_to_date_time(super)
-    end
-
     def document_model
       exhibit.blacklight_config.document_model if exhibit
     end
@@ -71,43 +43,28 @@ module Spotlight
       # Index the result of {#to_solr} into the index in batches of {#batch_size}
       #
       # @return [Integer] number of records indexed
-      def reindex
+      # rubocop:disable Metrics/MethodLength
+      def reindex(reindexing_log_entry = nil)
         benchmark "Reindexing #{self} (batch size: #{batch_size})" do
           count = 0
 
           run_callbacks :index do
             document_builder.documents_to_index.each_slice(batch_size) do |batch|
               write_to_index(batch)
-              update(last_indexed_count: (count += batch.length))
+              count += batch.length
+              reindexing_log_entry.update(items_reindexed_count: count) if reindexing_log_entry
             end
 
             count
           end
         end
       end
+      # rubocop:enable Metrics/MethodLength
 
       def document_builder
         @document_builder ||= document_builder_class.new(self)
       end
 
-      protected
-
-      def reindex_with_logging
-        time_start = Time.zone.now
-
-        update(indexed_at: time_start,
-               last_indexed_estimate: document_builder.documents_to_index.size,
-               last_indexed_finished: nil,
-               last_index_elapsed_time: nil)
-
-        count = yield
-
-        time_end = Time.zone.now
-        update(last_indexed_count: count,
-               last_indexed_finished: time_end,
-               last_index_elapsed_time: time_end - time_start)
-      end
-
       private
 
       def blacklight_solr
@@ -136,18 +93,12 @@ module Spotlight
         Rails.logger.warn "Unable to commit to solr: #{e}"
       end
 
-      def write?
-        Spotlight::Engine.config.writable_index
+      def touch_exhibit!
+        exhibit.touch
       end
 
-      def cast_to_date_time(value)
-        return unless value
-
-        if defined? ActiveModel::Type::DateTime
-          ActiveModel::Type::DateTime.new.cast(value)
-        else
-          ActiveRecord::Type::DateTime.new.type_cast_from_database(value)
-        end
+      def write?
+        Spotlight::Engine.config.writable_index
       end
     end
   end

data/app/serializers/spotlight/exhibit_export_serializer.rb

@@ -33,7 +33,7 @@ module Spotlight
       property prop
     end
 
-    collection :searches, parse_strategy: ->(fragment, _i, options) { options.represented.searches.find_or_initialize_by(slug: fragment['slug']) },
+    collection :searches, populator: ->(fragment, options) { options[:represented].searches.find_or_initialize_by(slug: fragment['slug']) },
                           class: Spotlight::Search do
       (Spotlight::Search.attribute_names - %w(id scope exhibit_id masthead_id thumbnail_id)).each do |prop|
         property prop
@@ -44,16 +44,16 @@ module Spotlight
       property :thumbnail, class: Spotlight::FeaturedImage, decorator: FeaturedImageRepresenter
     end
 
-    collection :about_pages, parse_strategy: ->(fragment, _i, options) { options.represented.about_pages.find_or_initialize_by(slug: fragment['slug']) },
+    collection :about_pages, populator: ->(fragment, options) { options[:represented].about_pages.find_or_initialize_by(slug: fragment['slug']) },
                              class: Spotlight::AboutPage,
                              decorator: PageRepresenter
 
-    collection :feature_pages, parse_strategy: ->(fragment, _i, options) { options.represented.feature_pages.find_or_initialize_by(slug: fragment['slug']) },
+    collection :feature_pages, populator: ->(fragment, options) { options[:represented].feature_pages.find_or_initialize_by(slug: fragment['slug']) },
                                getter: ->(_opts) { feature_pages.at_top_level },
                                class: Spotlight::FeaturePage,
                                decorator: NestedPageRepresenter
 
-    property :home_page, parse_strategy: ->(_fragment, options) { options.represented.home_page },
+    property :home_page, populator: ->(_fragment, options) { options[:represented].home_page },
                          class: Spotlight::HomePage,
                          decorator: PageRepresenter
 
@@ -65,14 +65,14 @@ module Spotlight
 
     property :blacklight_configuration, class: Spotlight::BlacklightConfiguration, decorator: ConfigurationRepresenter
 
-    collection :custom_fields, parse_strategy: ->(fragment, _i, options) { options.represented.custom_fields.find_or_initialize_by(slug: fragment['slug']) },
+    collection :custom_fields, populator: ->(fragment, options) { options[:represented].custom_fields.find_or_initialize_by(slug: fragment['slug']) },
                                class: Spotlight::CustomField do
       (Spotlight::CustomField.attribute_names - %w(id exhibit_id)).each do |prop|
         property prop
       end
     end
 
-    collection :contacts, parse_strategy: ->(fragment, _i, options) { options.represented.contacts.find_or_initialize_by(slug: fragment['slug']) },
+    collection :contacts, populator: ->(fragment, options) { options[:represented].contacts.find_or_initialize_by(slug: fragment['slug']) },
                           class: Spotlight::Contact do
       (Spotlight::Contact.attribute_names - %w(id exhibit_id avatar)).each do |prop|
         property prop
@@ -150,7 +150,7 @@ module Spotlight
       end
     end
 
-    collection :resources, class: ->(fragment, *) { fragment.key?('type') ? fragment['type'].constantize : Spotlight::Resource } do
+    collection :resources, class: ->(options) { options[:fragment].key?('type') ? options[:fragment]['type'].constantize : Spotlight::Resource } do
       (Spotlight::Resource.attribute_names - %w(id url exhibit_id)).each do |prop|
         property prop
       end