RubyGems - blacklight-spotlight - Versions diffs - 0.6.0 → 0.7.0 - Mend

blacklight-spotlight 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (15) hide show

checksums.yaml +4 -4
data/app/controllers/spotlight/resources/upload_controller.rb +1 -1
data/app/controllers/spotlight/resources_controller.rb +1 -1
data/app/models/concerns/spotlight/solr_document/uploaded_resource.rb +1 -1
data/app/models/spotlight/resource.rb +133 -88
data/app/models/spotlight/resource_provider.rb +24 -0
data/db/migrate/20151110082345_add_metadata_to_spotlight_resource.rb +5 -0
data/lib/spotlight/engine.rb +2 -0
data/lib/spotlight/version.rb +1 -1
data/spec/controllers/spotlight/resources/upload_controller_spec.rb +2 -3
data/spec/controllers/spotlight/resources_controller_spec.rb +1 -2
data/spec/features/add_item_bookmarklet_spec.rb +1 -1
data/spec/models/spotlight/resource_provider_spec.rb +27 -0
data/spec/models/spotlight/resource_spec.rb +59 -51
metadata +7 -3

checksums.yaml CHANGED

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19ff3a47f366456895a647229c3c9fea67ec3b76
-  data.tar.gz: 7a59c82abf7fcc1b25bebc9319867779b31082f4
+  metadata.gz: c321ae7af7b5264e59eff345211ac2760a96cb5b
+  data.tar.gz: 2048a70d4aac48da5effa59328e3e7f73e132b2b
 SHA512:
-  metadata.gz: 059110ca3e0365f26885a4ca692b4ba405b29f0d469f1ebfc34122d45ce71aa3313860f381f963e05da98810795614a55937e0eeea64841f47392773c131ffba
-  data.tar.gz: 4b546801b5cc566c9440a49cebd4b6b78aac9e8abcf208f24edf05b0acc83ba0edb91e3934c4b19ef7bf8d1057d82183d9cfe0cec5e0946be9aa424d5f35ce93
+  metadata.gz: 95dd157f70fd3b9f4d670a17c611d6e649d004a5fee7fa71d526c2b7f75fab528a21326e66ac3c45c2268152c7e7f0cb57f5a6a620ccea2ca84e912e1c2f842d
+  data.tar.gz: 7e43fa7eca9cacee4e224e8a5974732578bbd00df20130b7df44f464b58d66505e568aeca905440abeaff6afe87459c2059710c6db0e94f15a8f9275f975e5d9

data/app/controllers/spotlight/resources/upload_controller.rb CHANGED

@@ -26,7 +26,7 @@ module Spotlight
       def create
         @resource.attributes = resource_params
-        if @resource.save_and_commit
+        if @resource.save_and_index
           flash[:notice] = t('spotlight.resources.upload.success')
           if params['add-and-continue']
             redirect_to new_exhibit_resources_upload_path(@resource.exhibit)

data/app/controllers/spotlight/resources_controller.rb CHANGED

@@ -27,7 +27,7 @@ module Spotlight
       @resource.attributes = resource_params
       @resource = @resource.becomes_provider
-      if @resource.save_and_commit
+      if @resource.save_and_index
         if from_popup?
           render layout: false, text: '<html><script>window.close();</script></html>'
         else

data/app/models/concerns/spotlight/solr_document/uploaded_resource.rb CHANGED

@@ -10,7 +10,7 @@ module Spotlight
       end
       def uploaded_resource
-        @uploaded_resource ||= GlobalID::Locator.locate first(Spotlight::Resource.resource_global_id_field)
+        @uploaded_resource ||= GlobalID::Locator.locate first(Spotlight::Engine.config.resource_global_id_field)
       rescue => e
         Rails.logger.info("Unable to locate uploaded resource: #{e}")
         nil

data/app/models/spotlight/resource.rb CHANGED

@@ -3,138 +3,183 @@ module Spotlight
   # Exhibit resources
   class Resource < ActiveRecord::Base
     include Spotlight::SolrDocument::AtomicUpdates
+    include ActiveSupport::Benchmarkable
     extend ActiveModel::Callbacks
     define_model_callbacks :index
-    class_attribute :providers
     class_attribute :weight
     belongs_to :exhibit
     serialize :data, Hash
-    after_save :reindex
+    store :metadata, accessors: [
+      :last_indexed_estimate,
+      :last_indexed_count,
+      :last_index_elapsed_time,
+      :last_indexed_finished], coder: JSON
     around_index :reindex_with_lock
+    around_index :reindex_with_logging
+    after_index :commit
-    after_index :update_index_time!
+    def becomes_provider
+      klass = Spotlight::ResourceProvider.for_resource(self)
-    def self.providers
-      Spotlight::Engine.config.resource_providers
+      if klass
+        self.becomes! klass
+      else
+        self
+      end
     end
-    def self.class_for_resource(r)
-      providers.select { |provider| provider.can_provide? r }.sort_by(&:weight).first
+    def needs_provider?
+      type.blank?
     end
     ##
-    # @abstract
-    # Convert this resource into zero-to-many new solr documents. The data here
-    # should be merged into the resource-specific {#to_solr} data.
+    # Persist the record to the database, and trigger a reindex to solr
     #
-    # @return [Hash] a single solr document hash
-    # @return [Enumerator<Hash>] multiple solr document hashes. This can be a
-    #   simple array, or an lazy enumerator
-    def to_solr
-      (exhibit_specific_solr_data || {})
-        .merge(spotlight_resource_metadata_for_solr || {})
-    end
-    def self.resource_global_id_field
-      :"#{Spotlight::Engine.config.solr_fields.prefix}spotlight_resource_id#{Spotlight::Engine.config.solr_fields.string_suffix}"
+    # @param [Hash] All arguments will be passed through to ActiveRecord's #save method
+    def save_and_index(*args)
+      save(*args) && reindex_later
     end
-    def reindex_with_lock
-      with_lock do
-        yield
+    ##
+    # Enqueue an asynchronous reindexing job for this resource
+    def reindex_later
+      Spotlight::ReindexJob.perform_later(self)
+    end
+    concerning :GeneratingSolrDocuments do
+      ##
+      # @abstract
+      # Convert this resource into zero-to-many new solr documents. The data here
+      # should be merged into the resource-specific {#to_solr} data.
+      #
+      # @return [Hash] a single solr document hash
+      # @return [Enumerator<Hash>] multiple solr document hashes. This can be a
+      #   simple array, or an lazy enumerator
+      def to_solr
+        (exhibit_specific_solr_data || {}).merge(spotlight_resource_metadata_for_solr || {})
       end
-    end
-    ##
-    # Index the result of {#to_solr} into the index in batches of {#batch_size}
-    def reindex
-      run_callbacks :index do
-        data = to_solr
-        return if data.blank?
+      protected
+      ##
+      # @return an enumerator of all the indexable documents for this resource
+      def documents_to_index
+        data = to_solr
+        return [] if data.blank?
         data &&= [data] if data.is_a? Hash
-        data.each_slice(batch_size) do |batch|
-          blacklight_solr.update params: { commitWithin: 500 },
-                                 data: batch.reject(&:blank?).map { |doc| doc.reverse_merge(existing_solr_doc_hash(doc[unique_key]) || {}) }.to_json,
-                                 headers: { 'Content-Type' => 'application/json' }
+        return to_enum(:documents_to_index) { data.size } unless block_given?
+        data.reject(&:blank?).each do |doc|
+          yield doc.reverse_merge(existing_solr_doc_hash(doc[unique_key]) || {})
         end
       end
-    end
-    def update_index_time!
-      update_columns indexed_at: Time.current
-    end
+      private
-    def becomes_provider
-      klass = Spotlight::Resource.class_for_resource(self)
+      ##
+      # Get any exhibit-specific metadata stored in e.g. sidecars, tags, etc
+      # This needs the generated solr document
+      def existing_solr_doc_hash(id)
+        document_model.new(unique_key => id).to_solr if document_model && id.present?
+      end
-      if klass
-        self.becomes! klass
-      else
-        self
+      def unique_key
+        if document_model
+          document_model.unique_key.to_sym
+        else
+          :id
+        end
       end
-    end
-    def needs_provider?
-      type.blank?
+      def exhibit_specific_solr_data
+        exhibit.solr_data if exhibit
+      end
+      def spotlight_resource_metadata_for_solr
+        {
+          Spotlight::Engine.config.resource_global_id_field => (to_global_id.to_s if persisted?),
+          Spotlight::SolrDocument.resource_type_field => self.class.to_s.tableize
+        }
+      end
+      def document_model
+        exhibit.blacklight_config.document_model if exhibit
+      end
     end
-    def save_and_commit
-      save.tap do
-        begin
-          blacklight_solr.commit
-        rescue => e
-          Rails.logger.warn "Unable to commit to solr: #{e}"
+    concerning :Indexing do
+      ##
+      # Index the result of {#to_solr} into the index in batches of {#batch_size}
+      #
+      # @return [Integer] number of records indexed
+      def reindex
+        benchmark "Reindexing #{self} (batch size: #{batch_size})" do
+          count = 0
+          run_callbacks :index do
+            documents_to_index.each_slice(batch_size) do |batch|
+              write_to_index(batch)
+              update(last_indexed_count: (count += batch.length))
+            end
+            count
+          end
         end
       end
-    end
-    protected
+      protected
-    def blacklight_solr
-      @solr ||= RSolr.connect(connection_config)
-    end
+      def reindex_with_lock
+        with_lock do
+          yield
+        end
+      end
-    def connection_config
-      Blacklight.connection_config
-    end
+      def reindex_with_logging
+        time_start = Time.zone.now
-    def document_model
-      exhibit.blacklight_config.document_model if exhibit
-    end
+        update(indexed_at: time_start,
+               last_indexed_estimate: documents_to_index.size,
+               last_indexed_finished: nil,
+               last_index_elapsed_time: nil)
-    def batch_size
-      Spotlight::Engine.config.solr_batch_size
-    end
+        count = yield
-    def exhibit_specific_solr_data
-      exhibit.solr_data if exhibit
-    end
+        time_end = Time.zone.now
+        update(last_indexed_count: count,
+               last_indexed_finished: time_end,
+               last_index_elapsed_time: time_end - time_start)
+      end
-    def spotlight_resource_metadata_for_solr
-      {
-        Spotlight::Resource.resource_global_id_field => (to_global_id.to_s if persisted?),
-        Spotlight::SolrDocument.resource_type_field => self.class.to_s.tableize
-      }
-    end
+      private
-    ##
-    # Get any exhibit-specific metadata stored in e.g. sidecars, tags, etc
-    # This needs the generated solr document
-    def existing_solr_doc_hash(id)
-      document_model.new(unique_key => id).to_solr if document_model && id.present?
-    end
+      def blacklight_solr
+        @solr ||= RSolr.connect(connection_config)
+      end
-    def unique_key
-      if document_model
-        document_model.unique_key.to_sym
-      else
-        :id
+      def connection_config
+        Blacklight.connection_config
+      end
+      def batch_size
+        Spotlight::Engine.config.solr_batch_size
+      end
+      def write_to_index(batch)
+        blacklight_solr.update params: { commitWithin: 500 },
+                               data: batch.to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+      end
+      def commit
+        blacklight_solr.commit
+      rescue => e
+        Rails.logger.warn "Unable to commit to solr: #{e}"
       end
     end
   end

data/app/models/spotlight/resource_provider.rb ADDED

@@ -0,0 +1,24 @@
+module Spotlight
+  ##
+  # Detect which Spotlight::Resource subclasses can provide indexing routines for
+  # a given resource
+  class ResourceProvider
+    class <<self
+      ##
+      # @return [Class] the class that can best provide indexing for the resource
+      def for_resource(resource)
+        providers_for_resource(resource).first
+      end
+      private
+      def providers
+        Spotlight::Engine.config.resource_providers
+      end
+      def providers_for_resource(resource)
+        providers.select { |provider| provider.can_provide? resource }.sort_by(&:weight)
+      end
+    end
+  end
+end

data/db/migrate/20151110082345_add_metadata_to_spotlight_resource.rb ADDED

@@ -0,0 +1,5 @@
+class AddMetadataToSpotlightResource < ActiveRecord::Migration
+  def up
+    add_column :spotlight_resources, :metadata, :blob
+  end
+end

data/lib/spotlight/engine.rb CHANGED

@@ -70,6 +70,8 @@ module Spotlight
     Spotlight::Engine.config.solr_fields.string_suffix = '_ssim'.freeze
     Spotlight::Engine.config.solr_fields.text_suffix = '_tesim'.freeze
+    Spotlight::Engine.config.resource_global_id_field = :"#{config.solr_fields.prefix}spotlight_resource_id#{config.solr_fields.string_suffix}"
     # The solr field that original (largest) images will be stored.
     Spotlight::Engine.config.full_image_field = :full_image_url_ssm
     Spotlight::Engine.config.thumbnail_field = :thumbnail_url_ssm

data/lib/spotlight/version.rb CHANGED

@@ -1,4 +1,4 @@
 #
 module Spotlight
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

data/spec/controllers/spotlight/resources/upload_controller_spec.rb CHANGED

@@ -71,12 +71,11 @@ describe Spotlight::Resources::UploadController, type: :controller do
       let(:blacklight_solr) { double }
       before do
-        allow(blacklight_solr).to receive(:commit)
-        allow_any_instance_of(Spotlight::Resource).to receive(:reindex)
+        allow_any_instance_of(Spotlight::Resource).to receive(:reindex).and_return(true)
         allow_any_instance_of(Spotlight::Resource).to receive(:blacklight_solr).and_return blacklight_solr
       end
       it 'create a Spotlight::Resources::Upload resource' do
-        expect(blacklight_solr).to receive(:commit)
+        expect_any_instance_of(Spotlight::Resource).to receive(:reindex_later)
         post :create, exhibit_id: exhibit, resources_upload: { url: 'url-data' }
         expect(assigns[:resource]).to be_persisted
         expect(assigns[:resource]).to be_a(Spotlight::Resources::Upload)

data/spec/controllers/spotlight/resources_controller_spec.rb CHANGED

@@ -53,8 +53,7 @@ describe Spotlight::ResourcesController, type: :controller do
     describe 'POST create' do
       let(:blacklight_solr) { double }
       it 'create a resource' do
-        allow_any_instance_of(Spotlight::Resource).to receive(:reindex)
-        expect(blacklight_solr).to receive(:commit)
+        expect_any_instance_of(Spotlight::Resource).to receive(:reindex_later)
         allow_any_instance_of(Spotlight::Resource).to receive(:blacklight_solr).and_return blacklight_solr
         post :create, exhibit_id: exhibit, resource: { url: 'info:uri' }
         expect(assigns[:resource]).to be_persisted

data/spec/features/add_item_bookmarklet_spec.rb CHANGED

@@ -23,7 +23,7 @@ describe 'adding an item using the provided bookmarklet', type: :feature do
   end
   it 'submits the form to create a new item' do
-    allow_any_instance_of(Spotlight::Resource).to receive(:reindex)
+    allow_any_instance_of(Spotlight::Resource).to receive(:reindex_later)
     visit spotlight.new_exhibit_resource_path(exhibit, popup: true, resource: { url: 'info:url' })
     expect(page).to have_content 'Add Resource'
     expect(first('#resource_url', visible: false).value).to eq 'info:url'

data/spec/models/spotlight/resource_provider_spec.rb ADDED

@@ -0,0 +1,27 @@
+require 'spec_helper'
+describe Spotlight::ResourceProvider do
+  describe '.for_resource' do
+    let(:thing) { double }
+    let(:type_a) { double('TypeA', weight: 10) }
+    let(:type_b) { double('TypeB', weight: 5) }
+    let(:providers) { [type_a, type_b] }
+    subject { described_class.for_resource(thing) }
+    before do
+      allow(described_class).to receive_messages(providers: providers)
+    end
+    it 'returns a class that can provide indexing for the resource' do
+      expect(type_a).to receive(:can_provide?).with(thing).and_return(true)
+      expect(type_b).to receive(:can_provide?).with(thing).and_return(false)
+      expect(subject).to eq type_a
+    end
+    it 'returns the lowest weighted class that can provide indexing for the resource' do
+      expect(type_a).to receive(:can_provide?).with(thing).and_return(true)
+      expect(type_b).to receive(:can_provide?).with(thing).and_return(true)
+      expect(subject).to eq type_b
+    end
+  end
+end

data/spec/models/spotlight/resource_spec.rb CHANGED

@@ -6,30 +6,6 @@ describe Spotlight::Resource, type: :model do
   end
   let(:exhibit) { FactoryGirl.create(:exhibit) }
-  describe '.class_for_resource' do
-    let(:thing) { double }
-    let(:type_a) { double('TypeA', weight: 10) }
-    let(:type_b) { double('TypeB', weight: 5) }
-    let(:providers) { [type_a, type_b] }
-    subject { described_class.class_for_resource(thing) }
-    before do
-      allow(described_class).to receive_messages(providers: providers)
-    end
-    it 'returns a class that can provide indexing for the resource' do
-      expect(type_a).to receive(:can_provide?).with(thing).and_return(true)
-      expect(type_b).to receive(:can_provide?).with(thing).and_return(false)
-      expect(subject).to eq type_a
-    end
-    it 'returns the lowest weighted class that can provide indexing for the resource' do
-      expect(type_a).to receive(:can_provide?).with(thing).and_return(true)
-      expect(type_b).to receive(:can_provide?).with(thing).and_return(true)
-      expect(subject).to eq type_b
-    end
-  end
   describe '#to_solr' do
     before do
       allow(subject).to receive(:exhibit).and_return(exhibit)
@@ -48,20 +24,20 @@ describe Spotlight::Resource, type: :model do
   describe '#reindex' do
     context 'with a provider that generates ids' do
       subject do
-        Class.new(described_class) do
-          def to_solr
-            super.merge(id: 123)
-          end
-        end.new(exhibit: exhibit)
+        Class.new(described_class).new(exhibit: exhibit)
       end
+      let(:solr_response) { { id: 123 } }
       before do
         SolrDocument.new(id: 123).sidecars.create!(exhibit: exhibit, data: { document_data: true })
-        allow(subject).to receive_messages(to_global_id: '', update_index_time!: nil)
+        allow(subject).to receive_messages(to_global_id: '')
+        allow(subject).to receive(:to_solr).and_return(solr_response)
       end
       it 'includes exhibit document-specific data' do
-        expect(subject.send(:blacklight_solr)).to receive(:update) do |options|
+        allow(subject.send(:blacklight_solr)).to receive(:update) do |options|
           data = JSON.parse(options[:data], symbolize_names: true)
           expect(data.length).to eq 1
@@ -72,22 +48,69 @@ describe Spotlight::Resource, type: :model do
         subject.reindex
       end
+      context 'with a resource that creates multiple solr documents' do
+        let(:solr_response) { [{ id: 1 }, { id: 2 }] }
+        before do
+          allow(subject.send(:blacklight_solr)).to receive(:update)
+        end
+        it 'returns the number of indexed objects' do
+          expect(subject.reindex).to eq 2
+        end
+        it 'triggers a solr commit' do
+          expect(subject.send(:blacklight_solr)).to receive(:commit).once
+          subject.reindex
+        end
+        it 'records indexing metadata as document attributes' do
+          subject.reindex
+          expect(subject.indexed_at).to be > Time.zone.now - 5.seconds
+          expect(subject.last_indexed_estimate).to eq 2
+          expect(subject.last_indexed_count).to eq 2
+          expect(subject.last_index_elapsed_time).to be < 1
+        end
+      end
     end
   end
   describe '#becomes_provider' do
     it 'converts the resource to a provider-specific resource' do
       SomeClass = Class.new(described_class)
-      allow(described_class).to receive_messages(class_for_resource: SomeClass)
+      allow(Spotlight::ResourceProvider).to receive_messages(for_resource: SomeClass)
       expect(subject.becomes_provider).to be_a_kind_of(SomeClass)
       expect(subject.becomes_provider.type).to eq 'SomeClass'
     end
   end
-  it 'reindexs after save' do
-    expect(subject).to receive(:reindex)
-    subject.data_will_change!
-    subject.save!
+  describe '#save_and_index' do
+    before do
+      allow(subject.send(:blacklight_solr)).to receive(:update)
+      allow(subject).to receive(:reindex_later)
+    end
+    it 'saves the object' do
+      expect(subject).to receive(:save).and_return(true)
+      subject.save_and_index
+    end
+    it 'reindexes after save' do
+      expect(subject).to receive(:save).and_return(true)
+      expect(subject).to receive(:reindex_later)
+      subject.save_and_index
+    end
+    context 'if the save fails' do
+      it 'does not reindex' do
+        expect(subject).to receive(:save).and_return(false)
+        expect(subject).not_to receive(:reindex_later)
+        subject.save_and_index
+      end
+    end
   end
   it 'stores arbitrary data' do
@@ -97,19 +120,4 @@ describe Spotlight::Resource, type: :model do
     expect(subject.data[:a]).to eq 1
     expect(subject.data[:b]).to eq 2
   end
-  describe '#update_index_time!' do
-    it 'updates the index_time column' do
-      expect(subject).to receive(:update_columns).with(hash_including(:indexed_at))
-      subject.update_index_time!
-    end
-  end
-  describe '#save_and_commit' do
-    it 'saves the object and commit to solr' do
-      expect(subject).to receive(:save)
-      expect(subject.send(:blacklight_solr)).to receive(:commit)
-      subject.save_and_commit
-    end
-  end
 end

metadata CHANGED

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: blacklight-spotlight
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Chris Beer
@@ -11,7 +11,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-09 00:00:00.000000000 Z
+date: 2015-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -845,6 +845,7 @@ files:
 - app/models/spotlight/masthead.rb
 - app/models/spotlight/page.rb
 - app/models/spotlight/resource.rb
+- app/models/spotlight/resource_provider.rb
 - app/models/spotlight/resources/csv_upload.rb
 - app/models/spotlight/resources/upload.rb
 - app/models/spotlight/role.rb
@@ -1036,6 +1037,7 @@ files:
 - db/migrate/20150410180016_change_collation_for_tag_names.acts_as_taggable_on_engine.rb
 - db/migrate/20150713160101_change_spotlight_searches_to_published.rb
 - db/migrate/20151016092343_remove_searchable_from_exhibit.rb
+- db/migrate/20151110082345_add_metadata_to_spotlight_resource.rb
 - lib/blacklight/spotlight.rb
 - lib/generators/spotlight/install_generator.rb
 - lib/generators/spotlight/templates/catalog_controller.rb
@@ -1174,6 +1176,7 @@ files:
 - spec/models/spotlight/main_navigation_spec.rb
 - spec/models/spotlight/masthead_spec.rb
 - spec/models/spotlight/page_spec.rb
+- spec/models/spotlight/resource_provider_spec.rb
 - spec/models/spotlight/resource_spec.rb
 - spec/models/spotlight/resources/open_graph_spec.rb
 - spec/models/spotlight/resources/upload_spec.rb
@@ -1270,7 +1273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.6
+rubygems_version: 2.4.5.1
 signing_key:
 specification_version: 4
 summary: Enable librarians, curators, and others who are responsible for digital collections
@@ -1394,6 +1397,7 @@ test_files:
 - spec/models/spotlight/main_navigation_spec.rb
 - spec/models/spotlight/masthead_spec.rb
 - spec/models/spotlight/page_spec.rb
+- spec/models/spotlight/resource_provider_spec.rb
 - spec/models/spotlight/resource_spec.rb
 - spec/models/spotlight/resources/open_graph_spec.rb
 - spec/models/spotlight/resources/upload_spec.rb