blacklight-spotlight 3.0.0.alpha.10 → 3.0.0.rc5

data/spec/models/spotlight/solr_document/atomic_updates_spec.rb

@@ -37,6 +37,16 @@ describe Spotlight::SolrDocument::AtomicUpdates, type: :model do
       subject.reindex
     end
 
+    it 'update parameters can be specified to modify commitWithin' do
+      expected = {
+        params: { commitWithin: 5001 },
+        data: [{ id: 'doc_id', a: { set: 1 }, b: { set: 2 }, timestamp: { set: nil } }].to_json,
+        headers: { 'Content-Type' => 'application/json' }
+      }
+      expect(blacklight_solr).to receive(:update).with(expected)
+      subject.reindex(update_params: { commitWithin: 5001 })
+    end
+
     it 'cowardlies refuse to index a document if the only value is an id' do
       allow(subject).to receive_messages(to_solr: { id: 'doc_id' }, timestamp: { set: nil })
       expect(blacklight_solr).not_to receive(:update)

data/spec/models/spotlight/solr_document_sidecar_spec.rb

@@ -20,6 +20,7 @@ describe Spotlight::SolrDocumentSidecar, type: :model do
     context 'with an uploaded item' do
       before do
         subject.data = { 'configured_fields' => { 'some_configured_field' => 'some value' } }
+        subject.resource = Spotlight::Resources::Upload.new
         allow(Spotlight::Resources::Upload).to receive(:fields).with(exhibit).and_return([uploaded_field_config])
       end
 

data/spec/services/spotlight/bulk_updates_csv_template_service_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Spotlight::BulkUpdatesCsvTemplateService do
+  subject(:service) { described_class.new(exhibit: exhibit) }
+
+  let(:exhibit) { FactoryBot.create(:exhibit) }
+  let!(:tag1) { FactoryBot.create(:tagging, tagger: exhibit, taggable: exhibit) }
+  let!(:tag2) { FactoryBot.create(:tagging, tagger: exhibit, taggable: exhibit) }
+
+  describe '#template' do
+    let(:view_context) { double('ViewContext', document_presenter: double('DocumentPresenter', heading: 'Document Title')) }
+
+    it 'has a row for every document (+ the header)' do
+      template = CSV.parse(service.template(view_context: view_context).to_a.join)
+      expect(template).to have_at_least(56).items
+      expect(template[0].join(',')).to match(/Item ID,Item Title,Visibility,Tag: tagging\d,Tag: tagging\d/)
+    end
+
+    it 'only has requested columns' do
+      template = CSV.parse(service.template(view_context: view_context, tags: false).to_a.join)
+      expect(template[0].join(',')).to eq 'Item ID,Item Title,Visibility'
+    end
+  end
+end

data/spec/services/spotlight/etl/context_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Context do
+  subject(:context) { described_class.new(resource, **context_args) }
+
+  let(:resource) { instance_double(Spotlight::Resource, id: 123, document_model: SolrDocument) }
+  let(:context_args) { {} }
+
+  describe '#resource' do
+    it 'extracts the Spotlight::Resource from the argument list' do
+      expect(context.resource).to eq resource
+    end
+  end
+
+  describe '#unique_key' do
+    let(:data) { { id: '123' } }
+
+    it 'tries to get a usable unique key for a transformed document' do
+      expect(context.unique_key(data)).to eq '123'
+    end
+  end
+
+  describe '#on_error' do
+    context 'with a class-level handler' do
+      let(:handler) { instance_double(Proc, call: nil) }
+
+      it 'calls the class-level handler' do
+        allow(described_class).to receive(:error_reporter).and_return(handler)
+
+        subject.on_error(nil, nil, {})
+
+        expect(handler).to have_received(:call)
+      end
+    end
+
+    context 'with an instance-level handler' do
+      let(:context_args) { { on_error: handler } }
+      let(:handler) { instance_double(Proc, call: nil) }
+
+      it 'calls the instance-level handler' do
+        subject.on_error(nil, nil, {})
+
+        expect(handler).to have_received(:call)
+      end
+    end
+
+    context 'with :log' do
+      it 'logs an error' do
+        allow(Rails.logger).to receive(:error)
+
+        subject.on_error(nil, nil, {})
+
+        expect(Rails.logger).to have_received(:error).with(/Pipeline error/)
+      end
+    end
+
+    context 'with :exception' do
+      let(:context_args) { { on_error: :exception } }
+      let(:e) { StandardError.new('asdf') }
+
+      it 'raises an exception' do
+        expect { subject.on_error(nil, e, {}) }.to raise_exception(e)
+      end
+    end
+  end
+end

data/spec/services/spotlight/etl/executor_spec.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Executor do
+  subject(:executor) { described_class.new(pipeline, context) }
+
+  let(:pipeline) do
+    Spotlight::Etl::Pipeline.new do |pipeline|
+      pipeline.sources = [Spotlight::Etl::Sources::IdentitySource]
+      pipeline.transforms = [Spotlight::Etl::Transforms::IdentityTransform]
+      pipeline.loaders = [->(result, *) { arr << result }]
+    end
+  end
+  let(:context) { Spotlight::Etl::Context.new(resource) }
+  let(:resource) { Spotlight::Resource.new }
+  let(:arr) { [] }
+
+  describe '#call' do
+    it 'provides the context to the sources' do
+      pipeline.sources = [->(context, *) { [{ context: context }] }]
+
+      executor.call
+
+      expect(arr).to eq [{ context: context }]
+    end
+
+    it 'makes the current source available to transforms' do
+      pipeline.transforms = [->(_data, pipeline) { pipeline.source }]
+
+      executor.call
+
+      expect(arr).to eq [resource]
+    end
+
+    it 'chains the calls to the transforms' do
+      pipeline.transforms = [
+        ->(data, *) { data.merge(a: 1) },
+        ->(data, *) { data.merge(b: 2) },
+        ->(data, *) { data.merge(c: 3) }
+      ]
+
+      executor.call
+
+      expect(arr).to eq [{ a: 1, b: 2, c: 3 }]
+    end
+
+    it 'caches the step instances across sources' do
+      pipeline.sources = [->(*) { [1, 2, 3] }]
+
+      pipeline.transforms = [
+        Class.new do
+          def initialize
+            @sum = 0
+          end
+
+          def call(data, pipeline)
+            @sum += pipeline.source
+
+            data.merge(sum: @sum)
+          end
+        end
+      ]
+
+      executor.call
+
+      expect(arr).to eq [{ sum: 1 }, { sum: 3 }, { sum: 6 }]
+    end
+
+    it 'loads resulting documents' do
+      loader = Spotlight::Etl::SolrLoader.new
+      pipeline.sources = [->(*) { [{}, {}, {}] }]
+      pipeline.loaders = [loader]
+
+      allow(loader).to receive(:call)
+
+      executor.call
+
+      expect(loader).to have_received(:call).exactly(3).times
+    end
+
+    it 'allows steps to throw :skip to skip the current source' do
+      pipeline.pre_processes = [
+        ->(*) { throw(:skip) }
+      ]
+
+      executor.call
+
+      expect(arr).to be_blank
+    end
+
+    it 'forwards errors to the context error handler' do
+      e = StandardError.new
+      pipeline.transforms = [->(*) { raise e }]
+
+      allow(context).to receive(:on_error)
+      executor.call
+
+      expect(context).to have_received(:on_error).with(executor, e, {})
+    end
+
+    context 'after processing the data' do
+      it 'calls finalize on the loaders' do
+        loader = Spotlight::Etl::SolrLoader.new
+        pipeline.sources = [->(*) { [{}, {}, {}] }]
+        pipeline.loaders = [loader]
+
+        allow(loader).to receive(:call)
+        allow(loader).to receive(:finalize)
+
+        executor.call
+
+        expect(loader).to have_received(:call).exactly(3).times
+        expect(loader).to have_received(:finalize).once
+      end
+
+      it 'resets the step cache' do
+        pipeline.transforms = [
+          Class.new do
+            def initialize
+              @count = 0
+            end
+
+            def call(data, *)
+              @count += 1
+
+              data.merge(count: @count)
+            end
+          end
+        ]
+
+        executor.call
+        executor.call
+
+        expect(arr).to eq [{ count: 1 }, { count: 1 }]
+      end
+    end
+  end
+
+  describe '#estimated_size' do
+    let(:source) { ->(*) { [1, 2, 3, 4] } }
+
+    before do
+      pipeline.sources = [source]
+    end
+
+    it 'estimates the final size from the sources' do
+      expect(executor.estimated_size).to eq 4
+    end
+  end
+end

data/spec/services/spotlight/etl/pipeline_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Pipeline do
+  let(:mock_executor) { instance_double(Spotlight::Etl::Executor, call: 'result', estimated_size: 10) }
+  let(:context) { instance_double(Spotlight::Etl::Context) }
+
+  describe '#call' do
+    it 'forwards the call to the executor' do
+      allow(Spotlight::Etl::Executor).to receive(:new).with(subject, context, cache: nil).and_return(mock_executor)
+
+      expect(subject.call(context)).to eq 'result'
+    end
+  end
+
+  describe '#estimated_size' do
+    it 'forwards the call to the executor' do
+      allow(Spotlight::Etl::Executor).to receive(:new).with(subject, context).and_return(mock_executor)
+
+      expect(subject.estimated_size(context)).to eq 10
+    end
+  end
+end

data/spec/services/spotlight/etl/solr_loader_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::SolrLoader do
+  subject(:loader) { described_class.new(batch_size: 5, solr_connection: mock_connection) }
+
+  let(:mock_connection) { instance_double(RSolr::Client, commit: nil, update: nil) }
+
+  describe '#call' do
+    it 'queues data' do
+      loader.call({})
+      loader.call({})
+      loader.call({})
+
+      expect(loader.size).to eq 3
+    end
+
+    it 'writes data to the index when the queue is long enough' do
+      5.times { loader.call({}) }
+
+      expect(mock_connection).to have_received(:update) do |data:, **|
+        expect(JSON.parse(data).length).to eq 5
+      end
+    end
+  end
+
+  describe '#finalize' do
+    let(:pipeline) do
+      Spotlight::Etl::Executor.new(nil, Spotlight::Etl::Context.new(commit: false))
+    end
+
+    it 'flushes the remaining queue to the index' do
+      3.times { loader.call({}) }
+
+      loader.finalize
+
+      expect(mock_connection).to have_received(:update) do |data:, **|
+        expect(JSON.parse(data).length).to eq 3
+      end
+
+      expect(mock_connection).to have_received(:commit)
+    end
+
+    it 'uses the pipeline configuration to determine whether to send a commit' do
+      loader.finalize(pipeline)
+
+      expect(mock_connection).not_to have_received(:commit)
+    end
+
+    context 'error handling' do
+      it 'tries documents individually if the whole batch fails' do
+        allow(mock_connection).to receive(:update) do |data:, **|
+          raise '???' if JSON.parse(data).length > 1
+        end
+
+        3.times { loader.call({}) }
+        loader.finalize
+
+        expect(mock_connection).to have_received(:update).exactly(4).times
+      end
+
+      it 'logs errors from trying individual documents' do
+        allow(pipeline).to receive(:on_error)
+
+        allow(mock_connection).to receive(:update) do |**|
+          raise '???'
+        end
+
+        loader.call({})
+
+        loader.finalize(pipeline)
+
+        expect(pipeline).to have_received(:on_error).once
+      end
+    end
+  end
+end

data/spec/services/spotlight/etl/step_spec.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+describe Spotlight::Etl::Step do
+  subject(:step) { described_class.new(definition, **step_args) }
+
+  let(:definition) { ->(a = nil) { a || 'value' } }
+  let(:step_args) { {} }
+
+  describe '#call' do
+    it 'returns the value from the defined step' do
+      expect(step.call).to eq('value')
+    end
+
+    it 'passes through arguments' do
+      expect(step.call('a')).to eq('a')
+    end
+
+    context 'with a class instance' do
+      let(:definition) { Spotlight::Etl::SolrLoader.new }
+
+      it 'runs the call method' do
+        allow(definition).to receive(:call)
+
+        step.call('a')
+
+        expect(definition).to have_received(:call).with('a')
+      end
+    end
+
+    context 'with a class' do
+      let(:definition) { Spotlight::Etl::SolrLoader }
+      let(:mock) { instance_double(definition, call: nil) }
+
+      it 'runs the call method' do
+        allow(definition).to receive(:new).and_return(mock)
+
+        step.call('a', 'b')
+
+        expect(mock).to have_received(:call).with('a', 'b')
+      end
+
+      it 'uses the same instance for the lifetime of the step' do
+        allow(definition).to receive(:new).once.and_return(mock)
+
+        step.call('a', 'b')
+        step.call('a', 'b')
+
+        expect(mock).to have_received(:call).twice.with('a', 'b')
+      end
+    end
+  end
+
+  describe '#finalize' do
+    it 'does nothing if the definition does not have a #finalize method' do
+      expect { step.finalize }.not_to raise_exception
+    end
+
+    context 'with a loader' do
+      let(:definition) { Spotlight::Etl::SolrLoader.new }
+
+      it 'runs the finalize method' do
+        allow(definition).to receive(:finalize)
+
+        step.finalize
+
+        expect(definition).to have_received(:finalize)
+      end
+    end
+  end
+end

data/spec/services/spotlight/exhibit_import_export_service_spec.rb

@@ -24,6 +24,10 @@ describe Spotlight::ExhibitImportExportService do
     expect(subject['searches']).to have(source_exhibit.searches.count).searches
   end
 
+  it 'has group attributes' do
+    expect(subject['groups']).to have(source_exhibit.groups.count).groups
+  end
+
   it 'has home page attributes' do
     expect(subject).to have_key 'home_page'
     expect(subject['home_page']).not_to have_key 'id'
@@ -329,10 +333,26 @@ describe Spotlight::ExhibitImportExportService do
         end
       end
 
+      context 'remote thumbnail with existing tilesource' do
+        it do
+          search.thumbnail.iiif_tilesource
+          search.thumbnail.save
+          source_exhibit.reload
+        end
+
+        it 'unsets the iiif_tilesource' do
+          subject
+          existing_search.reload
+          expect(existing_search.thumbnail.iiif_tilesource).not_to eq search.thumbnail.iiif_tilesource
+        end
+      end
+
       context 'with a thumbnail from an uploaded resource' do
         before do
           search.masthead.document_global_id = SolrDocument.new(id: 'xyz').to_global_id
           search.masthead.source = 'exhibit'
+          search.masthead.iiif_tilesource = 'foo'
+          search.masthead.image = nil # Setting to nil to mimick an exhibit source
           search.masthead.save
 
           source_exhibit.reload
@@ -343,6 +363,33 @@ describe Spotlight::ExhibitImportExportService do
           existing_search.reload
           expect(existing_search.masthead).not_to be_blank
         end
+
+        it 'does not unset the iiif_tilesource' do
+          subject
+          existing_search.reload
+          expect(existing_search.masthead.iiif_tilesource).to eq search.masthead.iiif_tilesource
+        end
+      end
+
+      context 'in a group' do
+        let!(:group) { FactoryBot.create(:group, title: 'blah', exhibit: source_exhibit) }
+
+        before do
+          search.groups << group
+          search.save!
+        end
+
+        it 'copies the group' do
+          subject
+          expect(destination_exhibit.groups.length).to eq 1
+          expect(destination_exhibit.groups.first.as_json).to include(group.as_json.slice('slug', 'title', 'weight', 'published'))
+        end
+
+        it 'copies the group membership' do
+          subject
+          expect(existing_search.reload.groups.length).to eq 1
+          expect(existing_search.groups.pluck(:slug)).to match_array [group.slug]
+        end
       end
     end
 
@@ -412,10 +459,22 @@ describe Spotlight::ExhibitImportExportService do
       new_export = described_class.new(e).as_json
 
       actual = _deep_transform_values_in_object(new_export) do |v|
-        v.is_a?(Time) ? nil : v
+        if DateTime.parse(v.to_s)
+          nil
+        else
+          v
+        end
+      rescue ArgumentError
+        nil
       end
       expected = _deep_transform_values_in_object(export) do |v|
-        v.is_a?(Time) ? nil : v
+        if DateTime.parse(v.to_s)
+          nil
+        else
+          v
+        end
+      rescue ArgumentError
+        nil
       end
       expect(actual).to include expected
     end
@@ -504,7 +563,7 @@ describe Spotlight::ExhibitImportExportService do
     end
 
     it 'includes only the page-related data' do
-      expect(subject.keys).to match_array %w[searches about_pages feature_pages home_page contacts]
+      expect(subject.keys).to match_array %w[searches groups about_pages feature_pages home_page contacts]
     end
   end
 end