RubyGems - blacklight-spotlight - Versions diffs - 0.6.0 → 0.7.0 - Mend

blacklight-spotlight 0.6.0 → 0.7.0

Files changed (15) hide show

checksums.yaml +4 -4
data/app/controllers/spotlight/resources/upload_controller.rb +1 -1
data/app/controllers/spotlight/resources_controller.rb +1 -1
data/app/models/concerns/spotlight/solr_document/uploaded_resource.rb +1 -1
data/app/models/spotlight/resource.rb +133 -88
data/app/models/spotlight/resource_provider.rb +24 -0
data/db/migrate/20151110082345_add_metadata_to_spotlight_resource.rb +5 -0
data/lib/spotlight/engine.rb +2 -0
data/lib/spotlight/version.rb +1 -1
data/spec/controllers/spotlight/resources/upload_controller_spec.rb +2 -3
data/spec/controllers/spotlight/resources_controller_spec.rb +1 -2
data/spec/features/add_item_bookmarklet_spec.rb +1 -1
data/spec/models/spotlight/resource_provider_spec.rb +27 -0
data/spec/models/spotlight/resource_spec.rb +59 -51
metadata +7 -3

checksums.yaml CHANGED

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19ff3a47f366456895a647229c3c9fea67ec3b76
-  data.tar.gz: 7a59c82abf7fcc1b25bebc9319867779b31082f4
+  metadata.gz: c321ae7af7b5264e59eff345211ac2760a96cb5b
+  data.tar.gz: 2048a70d4aac48da5effa59328e3e7f73e132b2b
 SHA512:
-  metadata.gz: 059110ca3e0365f26885a4ca692b4ba405b29f0d469f1ebfc34122d45ce71aa3313860f381f963e05da98810795614a55937e0eeea64841f47392773c131ffba
-  data.tar.gz: 4b546801b5cc566c9440a49cebd4b6b78aac9e8abcf208f24edf05b0acc83ba0edb91e3934c4b19ef7bf8d1057d82183d9cfe0cec5e0946be9aa424d5f35ce93
+  metadata.gz: 95dd157f70fd3b9f4d670a17c611d6e649d004a5fee7fa71d526c2b7f75fab528a21326e66ac3c45c2268152c7e7f0cb57f5a6a620ccea2ca84e912e1c2f842d
+  data.tar.gz: 7e43fa7eca9cacee4e224e8a5974732578bbd00df20130b7df44f464b58d66505e568aeca905440abeaff6afe87459c2059710c6db0e94f15a8f9275f975e5d9

data/app/controllers/spotlight/resources/upload_controller.rb CHANGED

@@ -26,7 +26,7 @@ module Spotlight
       def create
         @resource.attributes = resource_params
-        if @resource.save_and_commit
+        if @resource.save_and_index
           flash[:notice] = t('spotlight.resources.upload.success')
           if params['add-and-continue']
             redirect_to new_exhibit_resources_upload_path(@resource.exhibit)

data/app/controllers/spotlight/resources_controller.rb CHANGED

@@ -27,7 +27,7 @@ module Spotlight
       @resource.attributes = resource_params
       @resource = @resource.becomes_provider
-      if @resource.save_and_commit
+      if @resource.save_and_index
         if from_popup?
           render layout: false, text: '<html><script>window.close();</script></html>'
         else

data/app/models/concerns/spotlight/solr_document/uploaded_resource.rb CHANGED

@@ -10,7 +10,7 @@ module Spotlight
       end
       def uploaded_resource
-        @uploaded_resource ||= GlobalID::Locator.locate first(Spotlight::Resource.resource_global_id_field)
+        @uploaded_resource ||= GlobalID::Locator.locate first(Spotlight::Engine.config.resource_global_id_field)
       rescue => e
         Rails.logger.info("Unable to locate uploaded resource: #{e}")
         nil

data/app/models/spotlight/resource.rb CHANGED

@@ -3,138 +3,183 @@ module Spotlight
   # Exhibit resources
   class Resource < ActiveRecord::Base
     include Spotlight::SolrDocument::AtomicUpdates
+    include ActiveSupport::Benchmarkable
     extend ActiveModel::Callbacks
     define_model_callbacks :index
-    class_attribute :providers
     class_attribute :weight
     belongs_to :exhibit
     serialize :data, Hash
-    after_save :reindex
+    store :metadata, accessors: [
+      :last_indexed_estimate,
+      :last_indexed_count,
+      :last_index_elapsed_time,
+      :last_indexed_finished], coder: JSON
     around_index :reindex_with_lock
+    around_index :reindex_with_logging
+    after_index :commit
-    after_index :update_index_time!
+    def becomes_provider
+      klass = Spotlight::ResourceProvider.for_resource(self)
-    def self.providers
-      Spotlight::Engine.config.resource_providers
+      if klass
+        self.becomes! klass
+      else
+        self
+      end
     end
-    def self.class_for_resource(r)
-      providers.select { |provider| provider.can_provide? r }.sort_by(&:weight).first
+    def needs_provider?
+      type.blank?
     end
     ##
-    # @abstract
-    # Convert this resource into zero-to-many new solr documents. The data here
-    # should be merged into the resource-specific {#to_solr} data.
+    # Persist the record to the database, and trigger a reindex to solr
     #
-    # @return [Hash] a single solr document hash
-    # @return [Enumerator<Hash>] multiple solr document hashes. This can be a
-    #   simple array, or an lazy enumerator
-    def to_solr
-      (exhibit_specific_solr_data || {})
-        .merge(spotlight_resource_metadata_for_solr || {})
-    end
-    def self.resource_global_id_field
-      :"#{Spotlight::Engine.config.solr_fields.prefix}spotlight_resource_id#{Spotlight::Engine.config.solr_fields.string_suffix}"
+    # @param [Hash] All arguments will be passed through to ActiveRecord's #save method
+    def save_and_index(*args)
+      save(*args) && reindex_later
     end
-    def reindex_with_lock
-      with_lock do
-        yield
+    ##
+    # Enqueue an asynchronous reindexing job for this resource
+    def reindex_later
+      Spotlight::ReindexJob.perform_later(self)
+    end
+    concerning :GeneratingSolrDocuments do
+      ##
+      # @abstract
+      # Convert this resource into zero-to-many new solr documents. The data here
+      # should be merged into the resource-specific {#to_solr} data.
+      #
+      # @return [Hash] a single solr document hash
+      # @return [Enumerator<Hash>] multiple solr document hashes. This can be a
+      #   simple array, or an lazy enumerator
+      def to_solr
+        (exhibit_specific_solr_data || {}).merge(spotlight_resource_metadata_for_solr || {})
       end
-    end
-    ##
-    # Index the result of {#to_solr} into the index in batches of {#batch_size}
-    def reindex
-      run_callbacks :index do
-        data = to_solr
-        return if data.blank?
+      protected
+      ##
+      # @return an enumerator of all the indexable documents for this resource
+      def documents_to_index
+        data = to_solr
+        return [] if data.blank?
         data &&= [data] if data.is_a? Hash
-        data.each_slice(batch_size) do |batch|
-          blacklight_solr.update params: { commitWithin: 500 },
-                                 data: batch.reject(&:blank?).map { |doc| doc.reverse_merge(existing_solr_doc_hash(doc[unique_key]) || {}) }.to_json,
-                                 headers: { 'Content-Type' => 'application/json' }
+        return to_enum(:documents_to_index) { data.size } unless block_given?
+        data.reject(&:blank?).each do |doc|
+          yield doc.reverse_merge(existing_solr_doc_hash(doc[unique_key]) || {})
         end
       end
-    end
-    def update_index_time!
-      update_columns indexed_at: Time.current
-    end
+      private
-    def becomes_provider
-      klass = Spotlight::Resource.class_for_resource(self)
+      ##
+      # Get any exhibit-specific metadata stored in e.g. sidecars, tags, etc
+      # This needs the generated solr document
+      def existing_solr_doc_hash(id)
+        document_model.new(unique_key => id).to_solr if document_model && id.present?
+      end
-      if klass
-        self.becomes! klass
-      else
-        self
+      def unique_key
+        if document_model
+          document_model.unique_key.to_sym
+        else
+          :id
+        end
       end
-    end
-    def needs_provider?
-      type.blank?
+      def exhibit_specific_solr_data
+        exhibit.solr_data if exhibit
+      end
+      def spotlight_resource_metadata_for_solr
+        {
+          Spotlight::Engine.config.resource_global_id_field => (to_global_id.to_s if persisted?),
+          Spotlight::SolrDocument.resource_type_field => self.class.to_s.tableize
+        }
+      end
+      def document_model
+        exhibit.blacklight_config.document_model if exhibit
+      end
     end
-    def save_and_commit
-      save.tap do
-        begin
-          blacklight_solr.commit
-        rescue => e
-          Rails.logger.warn "Unable to commit to solr: #{e}"
+    concerning :Indexing do
+      ##
+      # Index the result of {#to_solr} into the index in batches of {#batch_size}
+      #
+      # @return [Integer] number of records indexed
+      def reindex
+        benchmark "Reindexing #{self} (batch size: #{batch_size})" do
+          count = 0
+          run_callbacks :index do
+            documents_to_index.each_slice(batch_size) do |batch|
+              write_to_index(batch)
+              update(last_indexed_count: (count += batch.length))
+            end
+            count
+          end
         end
       end
-    end
-    protected
+      protected
-    def blacklight_solr
-      @solr ||= RSolr.connect(connection_config)
-    end
+      def reindex_with_lock
+        with_lock do
+          yield
+        end
+      end
-    def connection_config
-      Blacklight.connection_config
-    end
+      def reindex_with_logging
+        time_start = Time.zone.now
-    def document_model
-      exhibit.blacklight_config.document_model if exhibit
-    end
+        update(indexed_at: time_start,
+               last_indexed_estimate: documents_to_index.size,
+               last_indexed_finished: nil,
+               last_index_elapsed_time: nil)
-    def batch_size
-      Spotlight::Engine.config.solr_batch_size
-    end
+        count = yield
-    def exhibit_specific_solr_data
-      exhibit.solr_data if exhibit
-    end
+        time_end = Time.zone.now
+        update(last_indexed_count: count,
+               last_indexed_finished: time_end,
+               last_index_elapsed_time: time_end - time_start)
+      end
-    def spotlight_resource_metadata_for_solr
-      {
-        Spotlight::Resource.resource_global_id_field => (to_global_id.to_s if persisted?),
-        Spotlight::SolrDocument.resource_type_field => self.class.to_s.tableize
-      }
-    end
+      private
-    ##
-    # Get any exhibit-specific metadata stored in e.g. sidecars, tags, etc
-    # This needs the generated solr document
-    def existing_solr_doc_hash(id)
-      document_model.new(unique_key => id).to_solr if document_model && id.present?
-    end
+      def blacklight_solr
+        @solr ||= RSolr.connect(connection_config)
+      end
-    def unique_key
-      if document_model
-        document_model.unique_key.to_sym
-      else
-        :id
+      def connection_config
+        Blacklight.connection_config
+      end
+      def batch_size
+        Spotlight::Engine.config.solr_batch_size
+      end
+      def write_to_index(batch)
+        blacklight_solr.update params: { commitWithin: 500 },
+                               data: batch.to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+      end
+      def commit
+        blacklight_solr.commit
+      rescue => e
+        Rails.logger.warn "Unable to commit to solr: #{e}"
       end
     end
   end

data/app/models/spotlight/resource_provider.rb ADDED

@@ -0,0 +1,24 @@
+module Spotlight
+  ##
+  # Detect which Spotlight::Resource subclasses can provide indexing routines for
+  # a given resource
+  class ResourceProvider
+    class <<self
+      ##
+      # @return [Class] the class that can best provide indexing for the resource
+      def for_resource(resource)
+        providers_for_resource(resource).first
+      end
+      private
+      def providers
+        Spotlight::Engine.config.resource_providers
+      end
+      def providers_for_resource(resource)
+        providers.select { |provider| provider.can_provide? resource }.sort_by(&:weight)
+      end
+    end
+  end
+end

data/db/migrate/20151110082345_add_metadata_to_spotlight_resource.rb ADDED

@@ -0,0 +1,5 @@
+class AddMetadataToSpotlightResource < ActiveRecord::Migration
+  def up
+    add_column :spotlight_resources, :metadata, :blob
+  end
+end

data/lib/spotlight/engine.rb CHANGED

@@ -70,6 +70,8 @@ module Spotlight
     Spotlight::Engine.config.solr_fields.string_suffix = '_ssim'.freeze
     Spotlight::Engine.config.solr_fields.text_suffix = '_tesim'.freeze
+    Spotlight::Engine.config.resource_global_id_field = :"#{config.solr_fields.prefix}spotlight_resource_id#{config.solr_fields.string_suffix}"
     # The solr field that original (largest) images will be stored.
     Spotlight::Engine.config.full_image_field = :full_image_url_ssm
     Spotlight::Engine.config.thumbnail_field = :thumbnail_url_ssm

data/lib/spotlight/version.rb CHANGED

@@ -1,4 +1,4 @@
 #
 module Spotlight
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

data/spec/controllers/spotlight/resources/upload_controller_spec.rb CHANGED

@@ -71,12 +71,11 @@ describe Spotlight::Resources::UploadController, type: :controller do
       let(:blacklight_solr) { double }
       before do
-        allow(blacklight_solr).to receive(:commit)
-        allow_any_instance_of(Spotlight::Resource).to receive(:reindex)
+        allow_any_instance_of(Spotlight::Resource).to receive(:reindex).and_return(true)
         allow_any_instance_of(Spotlight::Resource).to receive(:blacklight_solr).and_return blacklight_solr
       end
       it 'create a Spotlight::Resources::Upload resource' do
-        expect(blacklight_solr).to receive(:commit)
+        expect_any_instance_of(Spotlight::Resource).to receive(:reindex_later)
         post :create, exhibit_id: exhibit, resources_upload: { url: 'url-data' }
         expect(assigns[:resource]).to be_persisted
         expect(assigns[:resource]).to be_a(Spotlight::Resources::Upload)

data/spec/controllers/spotlight/resources_controller_spec.rb CHANGED

@@ -53,8 +53,7 @@ describe Spotlight::ResourcesController, type: :controller do
     describe 'POST create' do
       let(:blacklight_solr) { double }
       it 'create a resource' do
-        allow_any_instance_of(Spotlight::Resource).to receive(:reindex)
-        expect(blacklight_solr).to receive(:commit)
+        expect_any_instance_of(Spotlight::Resource).to receive(:reindex_later)
         allow_any_instance_of(Spotlight::Resource).to receive(:blacklight_solr).and_return blacklight_solr
         post :create, exhibit_id: exhibit, resource: { url: 'info:uri' }
         expect(assigns[:resource]).to be_persisted

data/spec/features/add_item_bookmarklet_spec.rb CHANGED

@@ -23,7 +23,7 @@ describe 'adding an item using the provided bookmarklet', type: :feature do
   end
   it 'submits the form to create a new item' do
-    allow_any_instance_of(Spotlight::Resource).to receive(:reindex)
+    allow_any_instance_of(Spotlight::Resource).to receive(:reindex_later)
     visit spotlight.new_exhibit_resource_path(exhibit, popup: true, resource: { url: 'info:url' })
     expect(page).to have_content 'Add Resource'
     expect(first('#resource_url', visible: false).value).to eq 'info:url'

data/spec/models/spotlight/resource_provider_spec.rb ADDED

@@ -0,0 +1,27 @@
+require 'spec_helper'
+describe Spotlight::ResourceProvider do
+  describe '.for_resource' do
+    let(:thing) { double }
+    let(:type_a) { double('TypeA', weight: 10) }
+    let(:type_b) { double('TypeB', weight: 5) }
+    let(:providers) { [type_a, type_b] }
+    subject { described_class.for_resource(thing) }
+    before do
+      allow(described_class).to receive_messages(providers: providers)
+    end
+    it 'returns a class that can provide indexing for the resource' do
+      expect(type_a).to receive(:can_provide?).with(thing).and_return(true)
+      expect(type_b).to receive(:can_provide?).with(thing).and_return(false)
+      expect(subject).to eq type_a
+    end
+    it 'returns the lowest weighted class that can provide indexing for the resource' do
+      expect(type_a).to receive(:can_provide?).with(thing).and_return(true)
+      expect(type_b).to receive(:can_provide?).with(thing).and_return(true)
+      expect(subject).to eq type_b
+    end
+  end
+end

data/spec/models/spotlight/resource_spec.rb CHANGED

@@ -6,30 +6,6 @@ describe Spotlight::Resource, type: :model do
   end
   let(:exhibit) { FactoryGirl.create(:exhibit) }
-  describe '.class_for_resource' do
-    let(:thing) { double }
-    let(:type_a) { double('TypeA', weight: 10) }
-    let(:type_b) { double('TypeB', weight: 5) }
-    let(:providers) { [type_a, type_b] }
-    subject { described_class.class_for_resource(thing) }
-    before do
-      allow(described_class).to receive_messages(providers: providers)
-    end
-    it 'returns a class that can provide indexing for the resource' do
-      expect(type_a).to receive(:can_provide?).with(thing).and_return(true)
-      expect(type_b).to receive(:can_provide?).with(thing).and_return(false)
-      expect(subject).to eq type_a
-    end
-    it 'returns the lowest weighted class that can provide indexing for the resource' do
-      expect(type_a).to receive(:can_provide?).with(thing).and_return(true)
-      expect(type_b).to receive(:can_provide?).with(thing).and_return(true)
-      expect(subject).to eq type_b
-    end
-  end
   describe '#to_solr' do
     before do
       allow(subject).to receive(:exhibit).and_return(exhibit)
@@ -48,20 +24,20 @@ describe Spotlight::Resource, type: :model do
   describe '#reindex' do
     context 'with a provider that generates ids' do
       subject do
-        Class.new(described_class) do
-          def to_solr
-            super.merge(id: 123)
-          end
-        end.new(exhibit: exhibit)
+        Class.new(described_class).new(exhibit: exhibit)
       end
+      let(:solr_response) { { id: 123 } }
       before do
         SolrDocument.new(id: 123).sidecars.create!(exhibit: exhibit, data: { document_data: true })
-        allow(subject).to receive_messages(to_global_id: '', update_index_time!: nil)
+        allow(subject).to receive_messages(to_global_id: '')
+        allow(subject).to receive(:to_solr).and_return(solr_response)
       end
       it 'includes exhibit document-specific data' do
-        expect(subject.send(:blacklight_solr)).to receive(:update) do |options|
+        allow(subject.send(:blacklight_solr)).to receive(:update) do |options|
           data = JSON.parse(options[:data], symbolize_names: true)
           expect(data.length).to eq 1
@@ -72,22 +48,69 @@ describe Spotlight::Resource, type: :model do
         subject.reindex
       end
+      context 'with a resource that creates multiple solr documents' do
+        let(:solr_response) { [{ id: 1 }, { id: 2 }] }
+        before do
+          allow(subject.send(:blacklight_solr)).to receive(:update)
+        end
+        it 'returns the number of indexed objects' do
+          expect(subject.reindex).to eq 2
+        end
+        it 'triggers a solr commit' do
+          expect(subject.send(:blacklight_solr)).to receive(:commit).once
+          subject.reindex
+        end
+        it 'records indexing metadata as document attributes' do
+          subject.reindex
+          expect(subject.indexed_at).to be > Time.zone.now - 5.seconds
+          expect(subject.last_indexed_estimate).to eq 2
+          expect(subject.last_indexed_count).to eq 2
+          expect(subject.last_index_elapsed_time).to be < 1
+        end
+      end
     end
   end
   describe '#becomes_provider' do
     it 'converts the resource to a provider-specific resource' do
       SomeClass = Class.new(described_class)
-      allow(described_class).to receive_messages(class_for_resource: SomeClass)
+      allow(Spotlight::ResourceProvider).to receive_messages(for_resource: SomeClass)
       expect(subject.becomes_provider).to be_a_kind_of(SomeClass)
       expect(subject.becomes_provider.type).to eq 'SomeClass'
     end
   end
-  it 'reindexs after save' do
-    expect(subject).to receive(:reindex)
-    subject.data_will_change!
-    subject.save!
+  describe '#save_and_index' do
+    before do
+      allow(subject.send(:blacklight_solr)).to receive(:update)
+      allow(subject).to receive(:reindex_later)
+    end
+    it 'saves the object' do
+      expect(subject).to receive(:save).and_return(true)
+      subject.save_and_index
+    end
+    it 'reindexes after save' do
+      expect(subject).to receive(:save).and_return(true)
+      expect(subject).to receive(:reindex_later)
+      subject.save_and_index
+    end
+    context 'if the save fails' do
+      it 'does not reindex' do
+        expect(subject).to receive(:save).and_return(false)
+        expect(subject).not_to receive(:reindex_later)
+        subject.save_and_index
+      end
+    end
   end
   it 'stores arbitrary data' do
@@ -97,19 +120,4 @@ describe Spotlight::Resource, type: :model do
     expect(subject.data[:a]).to eq 1
     expect(subject.data[:b]).to eq 2
   end
-  describe '#update_index_time!' do
-    it 'updates the index_time column' do
-      expect(subject).to receive(:update_columns).with(hash_including(:indexed_at))
-      subject.update_index_time!
-    end
-  end
-  describe '#save_and_commit' do
-    it 'saves the object and commit to solr' do
-      expect(subject).to receive(:save)
-      expect(subject.send(:blacklight_solr)).to receive(:commit)
-      subject.save_and_commit
-    end
-  end
 end

metadata CHANGED

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: blacklight-spotlight
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Chris Beer
@@ -11,7 +11,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-09 00:00:00.000000000 Z
+date: 2015-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -845,6 +845,7 @@ files:
 - app/models/spotlight/masthead.rb
 - app/models/spotlight/page.rb
 - app/models/spotlight/resource.rb
+- app/models/spotlight/resource_provider.rb
 - app/models/spotlight/resources/csv_upload.rb
 - app/models/spotlight/resources/upload.rb
 - app/models/spotlight/role.rb
@@ -1036,6 +1037,7 @@ files:
 - db/migrate/20150410180016_change_collation_for_tag_names.acts_as_taggable_on_engine.rb
 - db/migrate/20150713160101_change_spotlight_searches_to_published.rb
 - db/migrate/20151016092343_remove_searchable_from_exhibit.rb
+- db/migrate/20151110082345_add_metadata_to_spotlight_resource.rb
 - lib/blacklight/spotlight.rb
 - lib/generators/spotlight/install_generator.rb
 - lib/generators/spotlight/templates/catalog_controller.rb
@@ -1174,6 +1176,7 @@ files:
 - spec/models/spotlight/main_navigation_spec.rb
 - spec/models/spotlight/masthead_spec.rb
 - spec/models/spotlight/page_spec.rb
+- spec/models/spotlight/resource_provider_spec.rb
 - spec/models/spotlight/resource_spec.rb
 - spec/models/spotlight/resources/open_graph_spec.rb
 - spec/models/spotlight/resources/upload_spec.rb
@@ -1270,7 +1273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.6
+rubygems_version: 2.4.5.1
 signing_key:
 specification_version: 4
 summary: Enable librarians, curators, and others who are responsible for digital collections
@@ -1394,6 +1397,7 @@ test_files:
 - spec/models/spotlight/main_navigation_spec.rb
 - spec/models/spotlight/masthead_spec.rb
 - spec/models/spotlight/page_spec.rb
+- spec/models/spotlight/resource_provider_spec.rb
 - spec/models/spotlight/resource_spec.rb
 - spec/models/spotlight/resources/open_graph_spec.rb
 - spec/models/spotlight/resources/upload_spec.rb