blacklight-spotlight 3.0.0.rc3 → 3.0.0.rc4

data/spec/models/spotlight/resource_spec.rb

@@ -1,126 +1,105 @@
 # frozen_string_literal: true
 
 describe Spotlight::Resource, type: :model do
-  before do
-    allow_any_instance_of(described_class).to receive(:update_index)
-  end
+  subject(:resource) { described_class.create(id: 123, exhibit: exhibit) }
 
   let(:exhibit) { FactoryBot.create(:exhibit) }
 
-  describe '#reindex' do
-    context 'with a provider that generates ids' do
-      subject do
-        Class.new(described_class).new(exhibit: exhibit)
-      end
+  describe '#save_and_index' do
+    before do
+      allow(subject).to receive(:save)
+      allow(subject).to receive(:reindex_later)
+    end
 
-      let(:solr_response) { { id: 123 } }
+    it 'saves the object' do
+      expect(subject).to receive(:save).and_return(true)
+      subject.save_and_index
+    end
 
-      before do
-        SolrDocument.new(id: 123).sidecars.create!(exhibit: exhibit, data: { document_data: true })
-        allow(subject).to receive_messages(to_global_id: '')
+    it 'reindexes after save' do
+      expect(subject).to receive(:save).and_return(true)
+      expect(subject).to receive(:reindex_later)
+      subject.save_and_index
+    end
 
-        allow(subject.document_builder).to receive(:to_solr).and_return(solr_response)
+    context 'if the save fails' do
+      it 'does not reindex' do
+        expect(subject).to receive(:save).and_return(false)
+        expect(subject).not_to receive(:reindex_later)
+        subject.save_and_index
       end
+    end
+  end
 
-      it 'includes exhibit document-specific data' do
-        allow(subject.send(:blacklight_solr)).to receive(:update) do |options|
-          data = JSON.parse(options[:data], symbolize_names: true)
-
-          expect(data.length).to eq 1
-          doc = data.first
-
-          break if doc.first == :commit
-
-          expect(doc).to include document_data: true
-        end
-
-        subject.reindex
-      end
+  describe '#reindex' do
+    before do
+      # sneak some data into the pipeline
+      subject.indexing_pipeline.transforms = [->(*) { { id: '123' } }] + subject.indexing_pipeline.transforms
+    end
 
-      context 'when a document does not have an identifier' do
-        let(:solr_response) { { other_field: 'Content' } }
+    let(:indexed_document) do
+      result = nil
 
-        it 'is not indexed (but a commit can be sent)' do
-          allow(subject.send(:blacklight_solr)).to receive(:commit)
-          expect(subject.send(:blacklight_solr)).not_to receive(:update)
+      subject.reindex(**index_args) do |data, *|
+        result = data
 
-          subject.reindex
-        end
+        # skip actually indexing the document into the solr index
+        throw :skip
       end
 
-      context 'reindexing_log_entry is provided' do
-        before do
-          allow(subject.send(:blacklight_solr)).to receive(:update)
-        end
-
-        it 'updates the count of reindexed items in the log entry' do
-          reindexing_log_entry = double(Spotlight::ReindexingLogEntry)
-          expect(reindexing_log_entry).to receive(:update).with(items_reindexed_count: 1)
-          subject.reindex reindexing_log_entry
-        end
-      end
+      result
+    end
 
-      context 'when the index is not writable' do
-        before do
-          allow(Spotlight::Engine.config).to receive_messages(writable_index: false)
-        end
+    let(:index_args) { {} }
 
-        it "doesn't write" do
-          expect(subject.send(:blacklight_solr)).not_to receive(:update)
-          subject.reindex
-        end
-      end
+    it 'returns the number of items indexed' do
+      expect(subject.reindex { |*| throw :skip }).to eq 1
+    end
 
-      context 'with a resource that creates multiple solr documents' do
-        let(:solr_response) { [{ id: 1 }, { id: 2 }] }
+    it 'applies exhibit-specific metadata from the sidecar' do
+      expect(indexed_document).to include Spotlight::SolrDocumentSidecar.new(document: SolrDocument.new(id: '123'), exhibit: exhibit).to_solr
+    end
 
-        before do
-          allow(subject.send(:blacklight_solr)).to receive(:update)
-        end
+    it 'includes metata from each sidecar' do
+      a = Spotlight::SolrDocumentSidecar.create(document: SolrDocument.new(id: '123'), exhibit: exhibit)
+      b = Spotlight::SolrDocumentSidecar.create(document: SolrDocument.new(id: '123'), exhibit: FactoryBot.build(:exhibit))
 
-        it 'returns the number of indexed objects' do
-          expect(subject.reindex).to eq 2
-        end
+      expect(indexed_document).to include(a.to_solr).and(include(b.to_solr))
+    end
 
-        it 'triggers a solr commit' do
-          expect(subject.send(:blacklight_solr)).to receive(:commit).once
+    it 'persists a sidecar document' do
+      expect { indexed_document }.to change(Spotlight::SolrDocumentSidecar, :count).by(1)
 
-          subject.reindex
-        end
+      expect(Spotlight::SolrDocumentSidecar.last).to have_attributes(document_id: '123', exhibit: exhibit)
+    end
 
-        it 'touches the exhibit to clear any caches' do
-          allow(subject.exhibit).to receive(:touch)
+    it 'applies application metadata' do
+      expect(indexed_document).to include(spotlight_resource_id_ssim: resource.to_global_id.to_s, spotlight_resource_type_ssim: 'spotlight/resources')
+    end
 
-          subject.reindex
+    context 'with some provided metadata' do
+      let(:index_args) { { additional_metadata: { a: 1 } } }
 
-          expect(subject.exhibit).to have_received(:touch)
-        end
+      it 'applies externally provided metadata' do
+        expect(indexed_document).to include a: 1
       end
     end
-  end
 
-  describe '#save_and_index' do
-    before do
-      allow(subject.send(:blacklight_solr)).to receive(:update)
-      allow(subject).to receive(:reindex_later)
-    end
+    it 'touches the exhibit to bust any caches' do
+      allow(exhibit).to receive(:touch)
 
-    it 'saves the object' do
-      expect(subject).to receive(:save).and_return(true)
-      subject.save_and_index
-    end
+      indexed_document
 
-    it 'reindexes after save' do
-      expect(subject).to receive(:save).and_return(true)
-      expect(subject).to receive(:reindex_later)
-      subject.save_and_index
+      expect(exhibit).to have_received(:touch)
     end
 
-    context 'if the save fails' do
-      it 'does not reindex' do
-        expect(subject).to receive(:save).and_return(false)
-        expect(subject).not_to receive(:reindex_later)
-        subject.save_and_index
+    context 'with touch: false' do
+      it 'does not touch the exhibit' do
+        allow(exhibit).to receive(:touch)
+
+        expect(subject.reindex(touch: false) { |*| throw :skip }).to eq 1
+
+        expect(exhibit).not_to have_received(:touch)
       end
     end
   end

data/spec/models/spotlight/resources/iiif_harvester_spec.rb

@@ -3,12 +3,11 @@
 require 'spec_helper'
 
 describe Spotlight::Resources::IiifHarvester do
+  subject(:harvester) { described_class.create(exhibit_id: exhibit.id, url: url) }
+
   let(:exhibit) { FactoryBot.create(:exhibit) }
-  let(:harvester) { described_class.create(exhibit_id: exhibit.id, url: url) }
 
   describe 'Validation' do
-    subject { harvester }
-
     context 'when given an invalid URL' do
       before do
         stub_request(:head, 'http://example.com').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
@@ -38,16 +37,16 @@ describe Spotlight::Resources::IiifHarvester do
     end
   end
 
-  describe '#documents_to_index' do
-    subject { harvester.document_builder }
-
+  describe '#reindex' do
     let(:url) { 'uri://for-top-level-collection' }
 
-    before { stub_default_collection }
+    before do
+      stub_default_collection
+      allow(Spotlight::Engine.config).to receive(:writable_index).and_return(false)
+    end
 
-    it 'returns an Enumerator of all the solr documents' do
-      expect(subject.documents_to_index).to be_a(Enumerator)
-      expect(subject.documents_to_index.count).to eq 8
+    it 'indexes all the solr documents' do
+      expect(subject.reindex).to eq 8
     end
   end
 end

data/spec/models/spotlight/solr_document_sidecar_spec.rb

@@ -20,6 +20,7 @@ describe Spotlight::SolrDocumentSidecar, type: :model do
     context 'with an uploaded item' do
       before do
         subject.data = { 'configured_fields' => { 'some_configured_field' => 'some value' } }
+        subject.resource = Spotlight::Resources::Upload.new
         allow(Spotlight::Resources::Upload).to receive(:fields).with(exhibit).and_return([uploaded_field_config])
       end
 

data/spec/services/spotlight/etl/context_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Context do
+  subject(:context) { described_class.new(resource, **context_args) }
+
+  let(:resource) { instance_double(Spotlight::Resource, id: 123, document_model: SolrDocument) }
+  let(:context_args) { {} }
+
+  describe '#resource' do
+    it 'extracts the Spotlight::Resource from the argument list' do
+      expect(context.resource).to eq resource
+    end
+  end
+
+  describe '#unique_key' do
+    let(:data) { { id: '123' } }
+
+    it 'tries to get a usable unique key for a transformed document' do
+      expect(context.unique_key(data)).to eq '123'
+    end
+  end
+
+  describe '#on_error' do
+    context 'with a class-level handler' do
+      let(:handler) { instance_double(Proc, call: nil) }
+
+      it 'calls the class-level handler' do
+        allow(described_class).to receive(:error_reporter).and_return(handler)
+
+        subject.on_error(nil, nil, {})
+
+        expect(handler).to have_received(:call)
+      end
+    end
+
+    context 'with an instance-level handler' do
+      let(:context_args) { { on_error: handler } }
+      let(:handler) { instance_double(Proc, call: nil) }
+
+      it 'calls the instance-level handler' do
+        subject.on_error(nil, nil, {})
+
+        expect(handler).to have_received(:call)
+      end
+    end
+
+    context 'with :log' do
+      it 'logs an error' do
+        allow(Rails.logger).to receive(:error)
+
+        subject.on_error(nil, nil, {})
+
+        expect(Rails.logger).to have_received(:error).with(/Pipeline error/)
+      end
+    end
+
+    context 'with :exception' do
+      let(:context_args) { { on_error: :exception } }
+      let(:e) { StandardError.new('asdf') }
+
+      it 'raises an exception' do
+        expect { subject.on_error(nil, e, {}) }.to raise_exception(e)
+      end
+    end
+  end
+end

data/spec/services/spotlight/etl/executor_spec.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Executor do
+  subject(:executor) { described_class.new(pipeline, context) }
+
+  let(:pipeline) do
+    Spotlight::Etl::Pipeline.new do |pipeline|
+      pipeline.sources = [Spotlight::Etl::Sources::IdentitySource]
+      pipeline.transforms = [Spotlight::Etl::Transforms::IdentityTransform]
+      pipeline.loaders = [->(result, *) { arr << result }]
+    end
+  end
+  let(:context) { Spotlight::Etl::Context.new(resource) }
+  let(:resource) { Spotlight::Resource.new }
+  let(:arr) { [] }
+
+  describe '#call' do
+    it 'provides the context to the sources' do
+      pipeline.sources = [->(context, *) { [{ context: context }] }]
+
+      executor.call
+
+      expect(arr).to eq [{ context: context }]
+    end
+
+    it 'makes the current source available to transforms' do
+      pipeline.transforms = [->(_data, pipeline) { pipeline.source }]
+
+      executor.call
+
+      expect(arr).to eq [resource]
+    end
+
+    it 'chains the calls to the transforms' do
+      pipeline.transforms = [
+        ->(data, *) { data.merge(a: 1) },
+        ->(data, *) { data.merge(b: 2) },
+        ->(data, *) { data.merge(c: 3) }
+      ]
+
+      executor.call
+
+      expect(arr).to eq [{ a: 1, b: 2, c: 3 }]
+    end
+
+    it 'caches the step instances across sources' do
+      pipeline.sources = [->(*) { [1, 2, 3] }]
+
+      pipeline.transforms = [
+        Class.new do
+          def initialize
+            @sum = 0
+          end
+
+          def call(data, pipeline)
+            @sum += pipeline.source
+
+            data.merge(sum: @sum)
+          end
+        end
+      ]
+
+      executor.call
+
+      expect(arr).to eq [{ sum: 1 }, { sum: 3 }, { sum: 6 }]
+    end
+
+    it 'loads resulting documents' do
+      loader = Spotlight::Etl::SolrLoader.new
+      pipeline.sources = [->(*) { [{}, {}, {}] }]
+      pipeline.loaders = [loader]
+
+      allow(loader).to receive(:call)
+
+      executor.call
+
+      expect(loader).to have_received(:call).exactly(3).times
+    end
+
+    it 'allows steps to throw :skip to skip the current source' do
+      pipeline.pre_processes = [
+        ->(*) { throw(:skip) }
+      ]
+
+      executor.call
+
+      expect(arr).to be_blank
+    end
+
+    it 'forwards errors to the context error handler' do
+      e = StandardError.new
+      pipeline.transforms = [->(*) { raise e }]
+
+      allow(context).to receive(:on_error)
+      executor.call
+
+      expect(context).to have_received(:on_error).with(executor, e, {})
+    end
+
+    context 'after processing the data' do
+      it 'calls finalize on the loaders' do
+        loader = Spotlight::Etl::SolrLoader.new
+        pipeline.sources = [->(*) { [{}, {}, {}] }]
+        pipeline.loaders = [loader]
+
+        allow(loader).to receive(:call)
+        allow(loader).to receive(:finalize)
+
+        executor.call
+
+        expect(loader).to have_received(:call).exactly(3).times
+        expect(loader).to have_received(:finalize).once
+      end
+
+      it 'resets the step cache' do
+        pipeline.transforms = [
+          Class.new do
+            def initialize
+              @count = 0
+            end
+
+            def call(data, *)
+              @count += 1
+
+              data.merge(count: @count)
+            end
+          end
+        ]
+
+        executor.call
+        executor.call
+
+        expect(arr).to eq [{ count: 1 }, { count: 1 }]
+      end
+    end
+  end
+
+  describe '#estimated_size' do
+    let(:source) { ->(*) { [1, 2, 3, 4] } }
+
+    before do
+      pipeline.sources = [source]
+    end
+
+    it 'estimates the final size from the sources' do
+      expect(executor.estimated_size).to eq 4
+    end
+  end
+end