blacklight-spotlight 3.0.0.rc3 → 3.0.1

data/spec/models/spotlight/resources/iiif_harvester_spec.rb

@@ -3,12 +3,11 @@
 require 'spec_helper'
 
 describe Spotlight::Resources::IiifHarvester do
+  subject(:harvester) { described_class.create(exhibit_id: exhibit.id, url: url) }
+
   let(:exhibit) { FactoryBot.create(:exhibit) }
-  let(:harvester) { described_class.create(exhibit_id: exhibit.id, url: url) }
 
   describe 'Validation' do
-    subject { harvester }
-
     context 'when given an invalid URL' do
       before do
         stub_request(:head, 'http://example.com').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
@@ -38,16 +37,16 @@ describe Spotlight::Resources::IiifHarvester do
     end
   end
 
-  describe '#documents_to_index' do
-    subject { harvester.document_builder }
-
+  describe '#reindex' do
     let(:url) { 'uri://for-top-level-collection' }
 
-    before { stub_default_collection }
+    before do
+      stub_default_collection
+      allow(Spotlight::Engine.config).to receive(:writable_index).and_return(false)
+    end
 
-    it 'returns an Enumerator of all the solr documents' do
-      expect(subject.documents_to_index).to be_a(Enumerator)
-      expect(subject.documents_to_index.count).to eq 8
+    it 'indexes all the solr documents' do
+      expect(subject.reindex).to eq 8
     end
   end
 end

data/spec/models/spotlight/solr_document/atomic_updates_spec.rb

@@ -37,6 +37,16 @@ describe Spotlight::SolrDocument::AtomicUpdates, type: :model do
       subject.reindex
     end
 
+    it 'update parameters can be specified to modify commitWithin' do
+      expected = {
+        params: { commitWithin: 5001 },
+        data: [{ id: 'doc_id', a: { set: 1 }, b: { set: 2 }, timestamp: { set: nil } }].to_json,
+        headers: { 'Content-Type' => 'application/json' }
+      }
+      expect(blacklight_solr).to receive(:update).with(expected)
+      subject.reindex(update_params: { commitWithin: 5001 })
+    end
+
     it 'cowardlies refuse to index a document if the only value is an id' do
       allow(subject).to receive_messages(to_solr: { id: 'doc_id' }, timestamp: { set: nil })
       expect(blacklight_solr).not_to receive(:update)

data/spec/models/spotlight/solr_document_sidecar_spec.rb

@@ -20,6 +20,7 @@ describe Spotlight::SolrDocumentSidecar, type: :model do
     context 'with an uploaded item' do
       before do
         subject.data = { 'configured_fields' => { 'some_configured_field' => 'some value' } }
+        subject.resource = Spotlight::Resources::Upload.new
         allow(Spotlight::Resources::Upload).to receive(:fields).with(exhibit).and_return([uploaded_field_config])
       end
 

data/spec/presenters/spotlight/iiif_manifest_presenter_spec.rb

@@ -16,7 +16,7 @@ describe Spotlight::IiifManifestPresenter do
   end
 
   describe 'public methods' do
-    let(:helpers) { double(show_presenter: presenter) }
+    let(:helpers) { double(document_presenter: presenter) }
     let(:presenter) { instance_double(Blacklight::ShowPresenter, heading: title_field_value) }
 
     let(:iiif_url) { 'https://iiif.test/images/1-1' }

data/spec/services/spotlight/bulk_updates_csv_template_service_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Spotlight::BulkUpdatesCsvTemplateService do
+  subject(:service) { described_class.new(exhibit: exhibit) }
+
+  let(:exhibit) { FactoryBot.create(:exhibit) }
+  let!(:tag1) { FactoryBot.create(:tagging, tagger: exhibit, taggable: exhibit) }
+  let!(:tag2) { FactoryBot.create(:tagging, tagger: exhibit, taggable: exhibit) }
+
+  describe '#template' do
+    let(:view_context) { double('ViewContext', document_presenter: double('DocumentPresenter', heading: 'Document Title')) }
+
+    it 'has a row for every document (+ the header)' do
+      template = CSV.parse(service.template(view_context: view_context).to_a.join)
+      expect(template).to have_at_least(56).items
+      expect(template[0].join(',')).to match(/Item ID,Item Title,Visibility,Tag: tagging\d,Tag: tagging\d/)
+    end
+
+    it 'only has requested columns' do
+      template = CSV.parse(service.template(view_context: view_context, tags: false).to_a.join)
+      expect(template[0].join(',')).to eq 'Item ID,Item Title,Visibility'
+    end
+  end
+end

data/spec/services/spotlight/etl/context_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Context do
+  subject(:context) { described_class.new(resource, **context_args) }
+
+  let(:resource) { instance_double(Spotlight::Resource, id: 123, document_model: SolrDocument) }
+  let(:context_args) { {} }
+
+  describe '#resource' do
+    it 'extracts the Spotlight::Resource from the argument list' do
+      expect(context.resource).to eq resource
+    end
+  end
+
+  describe '#unique_key' do
+    let(:data) { { id: '123' } }
+
+    it 'tries to get a usable unique key for a transformed document' do
+      expect(context.unique_key(data)).to eq '123'
+    end
+  end
+
+  describe '#on_error' do
+    context 'with a class-level handler' do
+      let(:handler) { instance_double(Proc, call: nil) }
+
+      it 'calls the class-level handler' do
+        allow(described_class).to receive(:error_reporter).and_return(handler)
+
+        subject.on_error(nil, nil, {})
+
+        expect(handler).to have_received(:call)
+      end
+    end
+
+    context 'with an instance-level handler' do
+      let(:context_args) { { on_error: handler } }
+      let(:handler) { instance_double(Proc, call: nil) }
+
+      it 'calls the instance-level handler' do
+        subject.on_error(nil, nil, {})
+
+        expect(handler).to have_received(:call)
+      end
+    end
+
+    context 'with :log' do
+      it 'logs an error' do
+        allow(Rails.logger).to receive(:error)
+
+        subject.on_error(nil, nil, {})
+
+        expect(Rails.logger).to have_received(:error).with(/Pipeline error/)
+      end
+    end
+
+    context 'with :exception' do
+      let(:context_args) { { on_error: :exception } }
+      let(:e) { StandardError.new('asdf') }
+
+      it 'raises an exception' do
+        expect { subject.on_error(nil, e, {}) }.to raise_exception(e)
+      end
+    end
+  end
+end

data/spec/services/spotlight/etl/executor_spec.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Executor do
+  subject(:executor) { described_class.new(pipeline, context) }
+
+  let(:pipeline) do
+    Spotlight::Etl::Pipeline.new do |pipeline|
+      pipeline.sources = [Spotlight::Etl::Sources::IdentitySource]
+      pipeline.transforms = [Spotlight::Etl::Transforms::IdentityTransform]
+      pipeline.loaders = [->(result, *) { arr << result }]
+    end
+  end
+  let(:context) { Spotlight::Etl::Context.new(resource) }
+  let(:resource) { Spotlight::Resource.new }
+  let(:arr) { [] }
+
+  describe '#call' do
+    it 'provides the context to the sources' do
+      pipeline.sources = [->(context, *) { [{ context: context }] }]
+
+      executor.call
+
+      expect(arr).to eq [{ context: context }]
+    end
+
+    it 'makes the current source available to transforms' do
+      pipeline.transforms = [->(_data, pipeline) { pipeline.source }]
+
+      executor.call
+
+      expect(arr).to eq [resource]
+    end
+
+    it 'chains the calls to the transforms' do
+      pipeline.transforms = [
+        ->(data, *) { data.merge(a: 1) },
+        ->(data, *) { data.merge(b: 2) },
+        ->(data, *) { data.merge(c: 3) }
+      ]
+
+      executor.call
+
+      expect(arr).to eq [{ a: 1, b: 2, c: 3 }]
+    end
+
+    it 'caches the step instances across sources' do
+      pipeline.sources = [->(*) { [1, 2, 3] }]
+
+      pipeline.transforms = [
+        Class.new do
+          def initialize
+            @sum = 0
+          end
+
+          def call(data, pipeline)
+            @sum += pipeline.source
+
+            data.merge(sum: @sum)
+          end
+        end
+      ]
+
+      executor.call
+
+      expect(arr).to eq [{ sum: 1 }, { sum: 3 }, { sum: 6 }]
+    end
+
+    it 'loads resulting documents' do
+      loader = Spotlight::Etl::SolrLoader.new
+      pipeline.sources = [->(*) { [{}, {}, {}] }]
+      pipeline.loaders = [loader]
+
+      allow(loader).to receive(:call)
+
+      executor.call
+
+      expect(loader).to have_received(:call).exactly(3).times
+    end
+
+    it 'allows steps to throw :skip to skip the current source' do
+      pipeline.pre_processes = [
+        ->(*) { throw(:skip) }
+      ]
+
+      executor.call
+
+      expect(arr).to be_blank
+    end
+
+    it 'forwards errors to the context error handler' do
+      e = StandardError.new
+      pipeline.transforms = [->(*) { raise e }]
+
+      allow(context).to receive(:on_error)
+      executor.call
+
+      expect(context).to have_received(:on_error).with(executor, e, {})
+    end
+
+    context 'after processing the data' do
+      it 'calls finalize on the loaders' do
+        loader = Spotlight::Etl::SolrLoader.new
+        pipeline.sources = [->(*) { [{}, {}, {}] }]
+        pipeline.loaders = [loader]
+
+        allow(loader).to receive(:call)
+        allow(loader).to receive(:finalize)
+
+        executor.call
+
+        expect(loader).to have_received(:call).exactly(3).times
+        expect(loader).to have_received(:finalize).once
+      end
+
+      it 'resets the step cache' do
+        pipeline.transforms = [
+          Class.new do
+            def initialize
+              @count = 0
+            end
+
+            def call(data, *)
+              @count += 1
+
+              data.merge(count: @count)
+            end
+          end
+        ]
+
+        executor.call
+        executor.call
+
+        expect(arr).to eq [{ count: 1 }, { count: 1 }]
+      end
+    end
+  end
+
+  describe '#estimated_size' do
+    let(:source) { ->(*) { [1, 2, 3, 4] } }
+
+    before do
+      pipeline.sources = [source]
+    end
+
+    it 'estimates the final size from the sources' do
+      expect(executor.estimated_size).to eq 4
+    end
+  end
+end

data/spec/services/spotlight/etl/pipeline_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Pipeline do
+  let(:mock_executor) { instance_double(Spotlight::Etl::Executor, call: 'result', estimated_size: 10) }
+  let(:context) { instance_double(Spotlight::Etl::Context) }
+
+  describe '#call' do
+    it 'forwards the call to the executor' do
+      allow(Spotlight::Etl::Executor).to receive(:new).with(subject, context, cache: nil).and_return(mock_executor)
+
+      expect(subject.call(context)).to eq 'result'
+    end
+  end
+
+  describe '#estimated_size' do
+    it 'forwards the call to the executor' do
+      allow(Spotlight::Etl::Executor).to receive(:new).with(subject, context).and_return(mock_executor)
+
+      expect(subject.estimated_size(context)).to eq 10
+    end
+  end
+end

data/spec/services/spotlight/etl/solr_loader_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::SolrLoader do
+  subject(:loader) { described_class.new(batch_size: 5, solr_connection: mock_connection) }
+
+  let(:mock_connection) { instance_double(RSolr::Client, commit: nil, update: nil) }
+
+  describe '#call' do
+    it 'queues data' do
+      loader.call({})
+      loader.call({})
+      loader.call({})
+
+      expect(loader.size).to eq 3
+    end
+
+    it 'writes data to the index when the queue is long enough' do
+      5.times { loader.call({}) }
+
+      expect(mock_connection).to have_received(:update) do |data:, **|
+        expect(JSON.parse(data).length).to eq 5
+      end
+    end
+  end
+
+  describe '#finalize' do
+    let(:pipeline) do
+      Spotlight::Etl::Executor.new(nil, Spotlight::Etl::Context.new(commit: false))
+    end
+
+    it 'flushes the remaining queue to the index' do
+      3.times { loader.call({}) }
+
+      loader.finalize
+
+      expect(mock_connection).to have_received(:update) do |data:, **|
+        expect(JSON.parse(data).length).to eq 3
+      end
+
+      expect(mock_connection).to have_received(:commit)
+    end
+
+    it 'uses the pipeline configuration to determine whether to send a commit' do
+      loader.finalize(pipeline)
+
+      expect(mock_connection).not_to have_received(:commit)
+    end
+
+    context 'error handling' do
+      it 'tries documents individually if the whole batch fails' do
+        allow(mock_connection).to receive(:update) do |data:, **|
+          raise '???' if JSON.parse(data).length > 1
+        end
+
+        3.times { loader.call({}) }
+        loader.finalize
+
+        expect(mock_connection).to have_received(:update).exactly(4).times
+      end
+
+      it 'logs errors from trying individual documents' do
+        allow(pipeline).to receive(:on_error)
+
+        allow(mock_connection).to receive(:update) do |**|
+          raise '???'
+        end
+
+        loader.call({})
+
+        loader.finalize(pipeline)
+
+        expect(pipeline).to have_received(:on_error).once
+      end
+    end
+  end
+end

data/spec/services/spotlight/etl/step_spec.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Step do
+  subject(:step) { described_class.new(definition, **step_args) }
+
+  let(:definition) { ->(a = nil) { a || 'value' } }
+  let(:step_args) { {} }
+
+  describe '#call' do
+    it 'returns the value from the defined step' do
+      expect(step.call).to eq('value')
+    end
+
+    it 'passes through arguments' do
+      expect(step.call('a')).to eq('a')
+    end
+
+    context 'with a class instance' do
+      let(:definition) { Spotlight::Etl::SolrLoader.new }
+
+      it 'runs the call method' do
+        allow(definition).to receive(:call)
+
+        step.call('a')
+
+        expect(definition).to have_received(:call).with('a')
+      end
+    end
+
+    context 'with a class' do
+      let(:definition) { Spotlight::Etl::SolrLoader }
+      let(:mock) { instance_double(definition, call: nil) }
+
+      it 'runs the call method' do
+        allow(definition).to receive(:new).and_return(mock)
+
+        step.call('a', 'b')
+
+        expect(mock).to have_received(:call).with('a', 'b')
+      end
+
+      it 'uses the same instance for the lifetime of the step' do
+        allow(definition).to receive(:new).once.and_return(mock)
+
+        step.call('a', 'b')
+        step.call('a', 'b')
+
+        expect(mock).to have_received(:call).twice.with('a', 'b')
+      end
+    end
+  end
+
+  describe '#finalize' do
+    it 'does nothing if the definition does not have a #finalize method' do
+      expect { step.finalize }.not_to raise_exception
+    end
+
+    context 'with a loader' do
+      let(:definition) { Spotlight::Etl::SolrLoader.new }
+
+      it 'runs the finalize method' do
+        allow(definition).to receive(:finalize)
+
+        step.finalize
+
+        expect(definition).to have_received(:finalize)
+      end
+    end
+  end
+end