blacklight-spotlight 3.0.0.rc3 → 3.0.0.rc4

data/spec/services/spotlight/etl/pipeline_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Pipeline do
+  let(:mock_executor) { instance_double(Spotlight::Etl::Executor, call: 'result', estimated_size: 10) }
+  let(:context) { instance_double(Spotlight::Etl::Context) }
+
+  describe '#call' do
+    it 'forwards the call to the executor' do
+      allow(Spotlight::Etl::Executor).to receive(:new).with(subject, context, cache: nil).and_return(mock_executor)
+
+      expect(subject.call(context)).to eq 'result'
+    end
+  end
+
+  describe '#estimated_size' do
+    it 'forwards the call to the executor' do
+      allow(Spotlight::Etl::Executor).to receive(:new).with(subject, context).and_return(mock_executor)
+
+      expect(subject.estimated_size(context)).to eq 10
+    end
+  end
+end

data/spec/services/spotlight/etl/solr_loader_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::SolrLoader do
+  subject(:loader) { described_class.new(batch_size: 5, solr_connection: mock_connection) }
+
+  let(:mock_connection) { instance_double(RSolr::Client, commit: nil, update: nil) }
+
+  describe '#call' do
+    it 'queues data' do
+      loader.call({})
+      loader.call({})
+      loader.call({})
+
+      expect(loader.size).to eq 3
+    end
+
+    it 'writes data to the index when the queue is long enough' do
+      5.times { loader.call({}) }
+
+      expect(mock_connection).to have_received(:update) do |data:, **|
+        expect(JSON.parse(data).length).to eq 5
+      end
+    end
+  end
+
+  describe '#finalize' do
+    let(:pipeline) do
+      Spotlight::Etl::Executor.new(nil, Spotlight::Etl::Context.new(commit: false))
+    end
+
+    it 'flushes the remaining queue to the index' do
+      3.times { loader.call({}) }
+
+      loader.finalize
+
+      expect(mock_connection).to have_received(:update) do |data:, **|
+        expect(JSON.parse(data).length).to eq 3
+      end
+
+      expect(mock_connection).to have_received(:commit)
+    end
+
+    it 'uses the pipeline configuration to determine whether to send a commit' do
+      loader.finalize(pipeline)
+
+      expect(mock_connection).not_to have_received(:commit)
+    end
+
+    context 'error handling' do
+      it 'tries documents individually if the whole batch fails' do
+        allow(mock_connection).to receive(:update) do |data:, **|
+          raise '???' if JSON.parse(data).length > 1
+        end
+
+        3.times { loader.call({}) }
+        loader.finalize
+
+        expect(mock_connection).to have_received(:update).exactly(4).times
+      end
+
+      it 'logs errors from trying individual documents' do
+        allow(pipeline).to receive(:on_error)
+
+        allow(mock_connection).to receive(:update) do |**|
+          raise '???'
+        end
+
+        loader.call({})
+
+        loader.finalize(pipeline)
+
+        expect(pipeline).to have_received(:on_error).once
+      end
+    end
+  end
+end

data/spec/services/spotlight/etl/step_spec.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Step do
+  subject(:step) { described_class.new(definition, **step_args) }
+
+  let(:definition) { ->(a = nil) { a || 'value' } }
+  let(:step_args) { {} }
+
+  describe '#call' do
+    it 'returns the value from the defined step' do
+      expect(step.call).to eq('value')
+    end
+
+    it 'passes through arguments' do
+      expect(step.call('a')).to eq('a')
+    end
+
+    context 'with a class instance' do
+      let(:definition) { Spotlight::Etl::SolrLoader.new }
+
+      it 'runs the call method' do
+        allow(definition).to receive(:call)
+
+        step.call('a')
+
+        expect(definition).to have_received(:call).with('a')
+      end
+    end
+
+    context 'with a class' do
+      let(:definition) { Spotlight::Etl::SolrLoader }
+      let(:mock) { instance_double(definition, call: nil) }
+
+      it 'runs the call method' do
+        allow(definition).to receive(:new).and_return(mock)
+
+        step.call('a', 'b')
+
+        expect(mock).to have_received(:call).with('a', 'b')
+      end
+
+      it 'uses the same instance for the lifetime of the step' do
+        allow(definition).to receive(:new).once.and_return(mock)
+
+        step.call('a', 'b')
+        step.call('a', 'b')
+
+        expect(mock).to have_received(:call).twice.with('a', 'b')
+      end
+    end
+  end
+
+  describe '#finalize' do
+    it 'does nothing if the definition does not have a #finalize method' do
+      expect { step.finalize }.not_to raise_exception
+    end
+
+    context 'with a loader' do
+      let(:definition) { Spotlight::Etl::SolrLoader.new }
+
+      it 'runs the finalize method' do
+        allow(definition).to receive(:finalize)
+
+        step.finalize
+
+        expect(definition).to have_received(:finalize)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -48,6 +48,8 @@ require 'spotlight'
 
 # configure spotlight with all the settings necessary to test functionality
 Spotlight::Engine.config.exhibit_themes = %w[default modern]
+Spotlight::Engine.config.reindexing_batch_count = 1
+Spotlight::Engine.config.assign_default_roles_to_first_user = false
 
 Dir['./spec/support/**/*.rb'].sort.each { |f| require f }
 
@@ -62,11 +64,6 @@ RSpec.configure do |config|
 
   config.use_transactional_fixtures = true
 
-  config.before do
-    # The first user is automatically granted admin privileges; we don't want that behavior for many of our tests
-    Spotlight::Engine.user_class.create email: 'initial+admin@example.com', password: 'password', password_confirmation: 'password'
-  end
-
   if defined? Devise::Test::ControllerHelpers
     config.include Devise::Test::ControllerHelpers, type: :controller
     config.include Devise::Test::ControllerHelpers, type: :view

data/spec/views/spotlight/dashboards/_reindexing_activity.html.erb_spec.rb

@@ -1,21 +1,16 @@
 # frozen_string_literal: true
 
 describe 'spotlight/dashboards/_reindexing_activity.html.erb', type: :view do
-  # recent reindexing entries should be sorted by start_time in descending order, so mock that behavior
-  let(:recent_reindexing) do
-    [FactoryBot.build(:unstarted_reindexing_log_entry)] + # nil start_time is trouble for the sort_by used to create the rest of the fixture's rows
-      [
-        FactoryBot.build(:reindexing_log_entry),
-        FactoryBot.build(:in_progress_reindexing_log_entry),
-        FactoryBot.build(:recent_reindexing_log_entry),
-        FactoryBot.build(:failed_reindexing_log_entry)
-      ].sort_by(&:start_time).reverse
-  end
   let(:p) { 'spotlight/dashboards/reindexing_activity' }
 
+  before do
+    assign(:recent_reindexing, recent_reindexing)
+  end
+
   context 'the reindexing log is empty' do
+    let(:recent_reindexing) { [] }
+
     before do
-      assign(:recent_reindexing, [])
       render p
     end
 
@@ -29,8 +24,18 @@ describe 'spotlight/dashboards/_reindexing_activity.html.erb', type: :view do
   end
 
   context 'the reindexing log has entries' do
+    # recent reindexing entries should be sorted by start_time in descending order, so mock that behavior
+    let(:recent_reindexing) do
+      [
+        FactoryBot.build(:job_tracker, status: 'enqueued'),
+        FactoryBot.build(:job_tracker, status: 'completed', data: { progress: 10 }),
+        FactoryBot.build(:job_tracker, status: 'in_progress', created_at: Time.zone.parse('January 10, 2017 23:00')),
+        FactoryBot.build(:job_tracker, status: 'completed', data: { progress: 100 }),
+        FactoryBot.build(:job_tracker, status: 'failed', data: { progress: 10 })
+      ]
+    end
+
     before do
-      assign(:recent_reindexing, recent_reindexing)
       render p
     end
 
@@ -43,7 +48,7 @@ describe 'spotlight/dashboards/_reindexing_activity.html.erb', type: :view do
     end
 
     it 'formats the start time correctly' do
-      expect(rendered).to have_css('table.table-striped td', text: 'January 05, 2017 23:00', count: 1)
+      expect(rendered).to have_css('table.table-striped td', text: 'January 05, 2017 23:00', count: 4)
       expect(rendered).to have_css('table.table-striped td', text: 'January 10, 2017 23:00', count: 1)
     end
 
@@ -60,11 +65,6 @@ describe 'spotlight/dashboards/_reindexing_activity.html.erb', type: :view do
       expect(rendered).to have_css('table.table-striped td', text: '5 minutes', count: 3)
     end
 
-    it 'displays nothing in the duration column or start time column when the info is unavailable (e.g. unstarted or in_progress attempts)' do
-      # we expect 2 blank durations, and 1 blank start time (1 unstarted log entry w/ blank start and duration, 1 in_progress w/ blank duration)
-      expect(rendered).to have_css('table.table-striped td', text: /^$/, count: 3)
-    end
-
     it 'displays the status of the reindexing attempt using localized text' do
       expect(rendered).to have_css('table.table-striped td', text: 'Not yet started', count: 1)
       expect(rendered).to have_css('table.table-striped td', text: 'Successful', count: 2)
@@ -74,8 +74,11 @@ describe 'spotlight/dashboards/_reindexing_activity.html.erb', type: :view do
   end
 
   context 'a reindexing log entry has a null user' do
+    let(:recent_reindexing) do
+      [FactoryBot.build(:job_tracker, status: 'completed', data: { progress: 10 }, user: nil)]
+    end
+
     it 'displays blank in the user field and renders without error' do
-      assign(:recent_reindexing, [FactoryBot.build(:reindexing_log_entry_no_user)])
       expect { render p }.not_to raise_error
 
       # we expect one blank table cell for the user, and values for everything else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: blacklight-spotlight
 version: !ruby/object:Gem::Version
-  version: 3.0.0.rc3
+  version: 3.0.0.rc4
 platform: ruby
 authors:
 - Chris Beer
@@ -11,8 +11,22 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-02 00:00:00.000000000 Z
+date: 2021-02-04 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob-status
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: acts-as-taggable-on
   requirement: !ruby/object:Gem::Requirement
@@ -425,6 +439,9 @@ dependencies:
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.2'
@@ -432,6 +449,9 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.2'
@@ -1025,10 +1045,16 @@ files:
 - app/helpers/spotlight/searches_helper.rb
 - app/helpers/spotlight/title_helper.rb
 - app/helpers/spotlight/translations_helper.rb
+- app/jobs/concerns/spotlight/job_tracking.rb
+- app/jobs/concerns/spotlight/limit_concurrency.rb
 - app/jobs/spotlight/add_uploads_from_csv.rb
+- app/jobs/spotlight/application_job.rb
+- app/jobs/spotlight/cleanup_job_trackers_job.rb
 - app/jobs/spotlight/default_thumbnail_job.rb
+- app/jobs/spotlight/reindex_exhibit_job.rb
 - app/jobs/spotlight/reindex_job.rb
 - app/jobs/spotlight/rename_sidecar_field_job.rb
+- app/jobs/spotlight/update_job_trackers_job.rb
 - app/mailers/spotlight/confirmation_mailer.rb
 - app/mailers/spotlight/contact_mailer.rb
 - app/mailers/spotlight/indexing_complete_mailer.rb
@@ -1040,7 +1066,6 @@ files:
 - app/models/concerns/spotlight/exhibit_analytics.rb
 - app/models/concerns/spotlight/exhibit_defaults.rb
 - app/models/concerns/spotlight/exhibit_documents.rb
-- app/models/concerns/spotlight/resources/open_graph.rb
 - app/models/concerns/spotlight/resources/web.rb
 - app/models/concerns/spotlight/sitemap.rb
 - app/models/concerns/spotlight/solr_document.rb
@@ -1075,6 +1100,7 @@ files:
 - app/models/spotlight/contact_image.rb
 - app/models/spotlight/custom_field.rb
 - app/models/spotlight/custom_search_field.rb
+- app/models/spotlight/event.rb
 - app/models/spotlight/exhibit.rb
 - app/models/spotlight/exhibit_thumbnail.rb
 - app/models/spotlight/feature_page.rb
@@ -1084,6 +1110,7 @@ files:
 - app/models/spotlight/group.rb
 - app/models/spotlight/group_member.rb
 - app/models/spotlight/home_page.rb
+- app/models/spotlight/job_tracker.rb
 - app/models/spotlight/language.rb
 - app/models/spotlight/lock.rb
 - app/models/spotlight/main_navigation.rb
@@ -1093,7 +1120,6 @@ files:
 - app/models/spotlight/page_content.rb
 - app/models/spotlight/page_content/sir_trevor.rb
 - app/models/spotlight/reindex_progress.rb
-- app/models/spotlight/reindexing_log_entry.rb
 - app/models/spotlight/resource.rb
 - app/models/spotlight/resources/csv_upload.rb
 - app/models/spotlight/resources/iiif_harvester.rb
@@ -1109,12 +1135,18 @@ files:
 - app/presenters/spotlight/iiif_manifest_presenter.rb
 - app/services/spotlight/carrierwave_file_resolver.rb
 - app/services/spotlight/clone_translated_page_from_locale.rb
+- app/services/spotlight/etl.rb
+- app/services/spotlight/etl/context.rb
+- app/services/spotlight/etl/executor.rb
+- app/services/spotlight/etl/loaders.rb
+- app/services/spotlight/etl/pipeline.rb
+- app/services/spotlight/etl/solr_loader.rb
+- app/services/spotlight/etl/sources.rb
+- app/services/spotlight/etl/step.rb
+- app/services/spotlight/etl/transforms.rb
 - app/services/spotlight/exhibit_import_export_service.rb
 - app/services/spotlight/iiif_resource_resolver.rb
 - app/services/spotlight/invite_users_service.rb
-- app/services/spotlight/resources/iiif_builder.rb
-- app/services/spotlight/solr_document_builder.rb
-- app/services/spotlight/upload_solr_document_builder.rb
 - app/services/spotlight/validity_checker.rb
 - app/uploaders/spotlight/attachment_uploader.rb
 - app/uploaders/spotlight/featured_image_uploader.rb
@@ -1386,6 +1418,8 @@ files:
 - db/migrate/20191205112300_add_unique_index_to_sidecars.rb
 - db/migrate/20200403161512_add_subtitle_to_searches.rb
 - db/migrate/20210113092223_create_spotlight_groups.rb
+- db/migrate/20210122082032_create_job_trackers.rb
+- db/migrate/20210126123041_create_events.rb
 - lib/blacklight/spotlight.rb
 - lib/generators/spotlight/install_generator.rb
 - lib/generators/spotlight/scaffold_resource_generator.rb
@@ -1454,10 +1488,10 @@ files:
 - spec/factories/exhibits.rb
 - spec/factories/featured_images.rb
 - spec/factories/group.rb
+- spec/factories/job_trackers.rb
 - spec/factories/language.rb
 - spec/factories/main_navigation.rb
 - spec/factories/pages.rb
-- spec/factories/reindexing_log_entries.rb
 - spec/factories/resources.rb
 - spec/factories/roles.rb
 - spec/factories/searches.rb
@@ -1547,6 +1581,7 @@ files:
 - spec/i18n_spec.rb
 - spec/jobs/spotlight/add_uploads_from_csv_spec.rb
 - spec/jobs/spotlight/default_thumbnail_job_spec.rb
+- spec/jobs/spotlight/reindex_exhibit_job_spec.rb
 - spec/jobs/spotlight/reindex_job_spec.rb
 - spec/jobs/spotlight/rename_sidecar_field_job_spec.rb
 - spec/lib/migration/iiif_spec.rb
@@ -1588,12 +1623,10 @@ files:
 - spec/models/spotlight/page_configurations_spec.rb
 - spec/models/spotlight/page_spec.rb
 - spec/models/spotlight/reindex_progress_spec.rb
-- spec/models/spotlight/reindexing_log_entry_spec.rb
 - spec/models/spotlight/resource_spec.rb
 - spec/models/spotlight/resources/iiif_harvester_spec.rb
 - spec/models/spotlight/resources/iiif_manifest_spec.rb
 - spec/models/spotlight/resources/iiif_service_spec.rb
-- spec/models/spotlight/resources/open_graph_spec.rb
 - spec/models/spotlight/resources/upload_spec.rb
 - spec/models/spotlight/resources/web_spec.rb
 - spec/models/spotlight/role_spec.rb
@@ -1611,10 +1644,14 @@ files:
 - spec/routing/spotlight/pages_routing_spec.rb
 - spec/services/spotlight/carrierwave_file_resolver_spec.rb
 - spec/services/spotlight/clone_translated_page_from_locale_spec.rb
+- spec/services/spotlight/etl/context_spec.rb
+- spec/services/spotlight/etl/executor_spec.rb
+- spec/services/spotlight/etl/pipeline_spec.rb
+- spec/services/spotlight/etl/solr_loader_spec.rb
+- spec/services/spotlight/etl/step_spec.rb
 - spec/services/spotlight/exhibit_import_export_service_spec.rb
 - spec/services/spotlight/iiif_resource_resolver_spec.rb
 - spec/services/spotlight/invite_users_service_spec.rb
-- spec/services/spotlight/solr_document_builder_spec.rb
 - spec/spec_helper.rb
 - spec/support/controllers/engine_helpers.rb
 - spec/support/disable_friendly_id_deprecation_warnings.rb
@@ -1795,10 +1832,10 @@ test_files:
 - spec/factories/exhibits.rb
 - spec/factories/featured_images.rb
 - spec/factories/group.rb
+- spec/factories/job_trackers.rb
 - spec/factories/language.rb
 - spec/factories/main_navigation.rb
 - spec/factories/pages.rb
-- spec/factories/reindexing_log_entries.rb
 - spec/factories/resources.rb
 - spec/factories/roles.rb
 - spec/factories/searches.rb
@@ -1888,6 +1925,7 @@ test_files:
 - spec/i18n_spec.rb
 - spec/jobs/spotlight/add_uploads_from_csv_spec.rb
 - spec/jobs/spotlight/default_thumbnail_job_spec.rb
+- spec/jobs/spotlight/reindex_exhibit_job_spec.rb
 - spec/jobs/spotlight/reindex_job_spec.rb
 - spec/jobs/spotlight/rename_sidecar_field_job_spec.rb
 - spec/lib/migration/iiif_spec.rb
@@ -1929,12 +1967,10 @@ test_files:
 - spec/models/spotlight/page_configurations_spec.rb
 - spec/models/spotlight/page_spec.rb
 - spec/models/spotlight/reindex_progress_spec.rb
-- spec/models/spotlight/reindexing_log_entry_spec.rb
 - spec/models/spotlight/resource_spec.rb
 - spec/models/spotlight/resources/iiif_harvester_spec.rb
 - spec/models/spotlight/resources/iiif_manifest_spec.rb
 - spec/models/spotlight/resources/iiif_service_spec.rb
-- spec/models/spotlight/resources/open_graph_spec.rb
 - spec/models/spotlight/resources/upload_spec.rb
 - spec/models/spotlight/resources/web_spec.rb
 - spec/models/spotlight/role_spec.rb
@@ -1952,10 +1988,14 @@ test_files:
 - spec/routing/spotlight/pages_routing_spec.rb
 - spec/services/spotlight/carrierwave_file_resolver_spec.rb
 - spec/services/spotlight/clone_translated_page_from_locale_spec.rb
+- spec/services/spotlight/etl/context_spec.rb
+- spec/services/spotlight/etl/executor_spec.rb
+- spec/services/spotlight/etl/pipeline_spec.rb
+- spec/services/spotlight/etl/solr_loader_spec.rb
+- spec/services/spotlight/etl/step_spec.rb
 - spec/services/spotlight/exhibit_import_export_service_spec.rb
 - spec/services/spotlight/iiif_resource_resolver_spec.rb
 - spec/services/spotlight/invite_users_service_spec.rb
-- spec/services/spotlight/solr_document_builder_spec.rb
 - spec/spec_helper.rb
 - spec/support/controllers/engine_helpers.rb
 - spec/support/disable_friendly_id_deprecation_warnings.rb