spotlight-dor-resources 0.0.6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c130b41c0a1a72df4461aeb827bc66f7959244f6
-  data.tar.gz: 2c91800eb334e092f90e5cb290fa69ace4163b0e
+  metadata.gz: 8df55f329be83de6c7ad4eded72dab44cb457888
+  data.tar.gz: d38a0f5e00576a446e71ce1d6ea77cd3276aefc3
 SHA512:
-  metadata.gz: e8ab91d1840f91f56a32a6eefe3d249fb7727fa985aad7324db746d805a3e3c7c2b02ca06e8ac1f49d5fad5ca97645ea107663c9a4cb984e14b51dc03d525050
-  data.tar.gz: 812b4522e38b815d846ce4a240a0792c64cd2b3139895a990d195eb938882828970c6c533ee61f35dc196e591798b8c06dd80466f7fe9a3ff6a34b382f7d73ed
+  metadata.gz: 010cb19a2fe3c6c2d16753685675e54de632acbf16c40babc1cce108a6aea2d15ef85220345c3e056eeeb22ae6ace6496f22e556d082408b9de8aef6718db12c
+  data.tar.gz: 7060e3540e2e950b013d76f70263748eb461d0207b68e681fdcd46fba366c284d94d3b3c384b6bebc3415b550dd154248e1644a772a44fa8c4c14b08fdccc9c2

data/.travis.yml

@@ -2,7 +2,7 @@ notifications:
   email: false
 
 rvm:
-  - 2.2.2
+  - 2.2.3
 
 env:
   global:

data/app/models/spotlight/resources/dor_resource.rb

@@ -1,19 +1,36 @@
 module Spotlight::Resources
   # Base Resource indexer for objects in DOR
   class DorResource < Spotlight::Resource
+    include ActiveSupport::Benchmarkable
+
     ##
-    # Generate solr documents for the DOR resources identified by this object
+    # Generate solr documents for the DOR resources identified by this object.
     #
     # @return [Enumerator] an enumerator of solr document hashes for indexing
-    def to_solr
-      return to_enum :to_solr unless block_given?
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def to_solr(&block)
+      return to_enum(:to_solr) { indexable_resources.size } unless block_given?
+
+      # We use the Parallel gem to support parallel processing of the collection,
+      # but need to jump through some hoops to make it yield an enumerable in the end.
+      #
+      # Here, we create a hook that simply yields the result to the enumerable. We configure
+      # this as a 'finish' hook, which Parallel will run on the main process.
+      yield_to_enum = ->(_item, _i, result) { block.call(result) }
 
-      base_doc = super
+      size = indexable_resources.size
 
-      indexable_resources.each do |res|
-        yield base_doc.merge(to_solr_document(res))
+      benchmark "Indexing resource #{inspect} (est. #{size} items)" do
+        base_doc = super
+
+        Parallel.each_with_index(indexable_resources, parallel_options.merge(finish: yield_to_enum)) do |res, idx|
+          benchmark "Indexing item #{res.druid} in resource #{id} (#{idx} / #{size})" do
+            base_doc.merge(to_solr_document(res))
+          end
+        end
       end
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     def resource
       @resource ||= Spotlight::Dor::Resources.indexer.resource doc_id
@@ -27,7 +44,7 @@ module Spotlight::Resources
     #
     # @return [Enumerator] an enumerator of resources to index
     def indexable_resources
-      return to_enum(:indexable_resources) unless block_given?
+      return to_enum(:indexable_resources) { 1 + resource.items.size } unless block_given?
 
       yield resource
 
@@ -44,5 +61,15 @@ module Spotlight::Resources
     def to_solr_document(resource)
       Spotlight::Dor::Resources.indexer.solr_document(resource)
     end
+
+    ##
+    # Write any logs (or benchmarking information) from this class to the gdor logs
+    def logger
+      Spotlight::Dor::Resources.indexer.logger
+    end
+
+    def parallel_options
+      Spotlight::Dor::Resources::Engine.config.parallel_options
+    end
   end
 end

data/lib/spotlight/dor/resources/version.rb

@@ -2,7 +2,7 @@ module Spotlight
   module Dor
     # :nodoc:
     module Resources
-      VERSION = '0.0.6'
+      VERSION = '0.1.0'
     end
   end
 end

data/spec/models/spotlight/resources/purl_spec.rb

@@ -91,7 +91,7 @@ describe Spotlight::Resources::Purl do
       end
 
       it 'provides a solr document for the items too' do
-        item = double
+        item = double(druid: 'xyz')
         allow(subject.resource).to receive(:items).and_return([item])
         expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return(collection: true)
         expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(item).and_return(item: true)

data/spec/models/spotlight/resources/searchworks_spec.rb

@@ -70,7 +70,7 @@ describe Spotlight::Resources::Searchworks do
       end
 
       it 'provides a solr document for the items too' do
-        item = double
+        item = double(druid: 'xyz')
         allow(subject.resource).to receive(:items).and_return([item])
         expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return(collection: true)
         expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(item).and_return(item: true)

data/spec/test_app_templates/lib/generators/test_app_generator.rb

@@ -1,7 +1,7 @@
 require 'rails/generators'
 
 class TestAppGenerator < Rails::Generators::Base
-  source_root "../spec/test_app_templates"
+  source_root File.expand_path("../../../../spec/test_app_templates", __FILE__)
 
   def add_gems
     gem 'blacklight', '~> 5.1'

data/spec/vcr_cassettes/gdor_indexing_integration_test/has_a_doc_id.yml

@@ -19,7 +19,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:05:58 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -33,9 +33,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.009352'
+      - '0.007425'
       X-Request-Id:
-      - 85356f50-b67e-4b69-8b5f-6eafa322f5fd
+      - d5753bb0-f4b7-4126-a356-19986744b9e2
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -129,7 +129,7 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:05:58 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
 - request:
     method: get
     uri: https://purl.stanford.edu/xf680rd3068.xml
@@ -149,7 +149,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:05:58 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -163,9 +163,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.003438'
+      - '0.006559'
       X-Request-Id:
-      - de993014-8374-4a85-a6bf-659e4c66996a
+      - 2ec12915-245b-4b41-a7ab-27e18fc8d8ea
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -259,7 +259,7 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:05:58 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
 - request:
     method: get
     uri: https://purl.stanford.edu/xf680rd3068.xml
@@ -279,7 +279,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:05:58 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -293,9 +293,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.024469'
+      - '0.007618'
       X-Request-Id:
-      - 55587aa0-8808-4d88-8401-5a23f146ab8b
+      - 5de78377-6a49-4f6d-9fe5-267bf228b679
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -389,7 +389,7 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:05:59 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
 - request:
     method: get
     uri: https://purl.stanford.edu/xf680rd3068.xml
@@ -409,7 +409,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:05:59 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -423,9 +423,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.007237'
+      - '0.006278'
       X-Request-Id:
-      - 39e44cd3-2775-4625-854e-20f5ff392066
+      - 89b2603c-b221-47e7-9572-94efcc11e757
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -519,7 +519,7 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:05:59 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
 - request:
     method: get
     uri: https://purl.stanford.edu/xf680rd3068.xml
@@ -539,7 +539,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:05:59 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -553,9 +553,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.003560'
+      - '0.006848'
       X-Request-Id:
-      - e88b1ad9-b3fe-48a2-9cde-4acabf72a28a
+      - 9e8f1c01-8b08-479e-86f7-3b5802bca16c
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -649,7 +649,7 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:05:59 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
 - request:
     method: get
     uri: https://purl.stanford.edu/xf680rd3068.xml
@@ -669,7 +669,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:05:59 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -683,9 +683,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.009015'
+      - '0.008278'
       X-Request-Id:
-      - d68d239f-6660-49e5-a019-0e06dc70cde4
+      - 2444d031-edff-4bd9-856a-46764cf01c77
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -779,10 +779,10 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:05:59 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
 - request:
     method: get
-    uri: https://purl.stanford.edu/xf680rd3068.mods
+    uri: https://purl.stanford.edu/xf680rd3068.xml
     body:
       encoding: US-ASCII
       string: ''
@@ -799,7 +799,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:05:59 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -813,9 +813,139 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.021499'
+      - '0.005658'
+      X-Request-Id:
+      - c65ed0f4-7efb-4a65-91e5-d9a219e912d9
+      X-Powered-By:
+      - Phusion Passenger 5.0.16
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubDomains
+      Last-Modified:
+      - Fri, 24 Apr 2015 18:45:21 GMT
+      Status:
+      - 200 OK
+      Transfer-Encoding:
+      - chunked
+      Content-Type:
+      - application/xml; charset=utf-8
+    body:
+      encoding: UTF-8
+      string: |
+        <?xml version="1.0" encoding="UTF-8"?>
+        <publicObject id="druid:xf680rd3068" published="2015-04-24T11:45:16-07:00">
+          <identityMetadata>
+            <sourceId source="sul">MISC_1855</sourceId>
+            <objectId>druid:xf680rd3068</objectId>
+            <objectCreator>DOR</objectCreator>
+            <objectLabel>Latin glossary : small manuscript fragment on vellum.</objectLabel>
+            <objectType>item</objectType>
+            <adminPolicy>druid:yg077np2529</adminPolicy>
+            <otherId name="catkey">10157160</otherId>
+            <otherId name="uuid">48b412d0-516c-11e3-bb66-0050569b3c3c</otherId>
+            <tag>Process : Content Type : Image</tag>
+            <tag>Project : Special Collections - Exhibitions</tag>
+            <tag>JIRA : PROJQUEUE-85</tag>
+            <tag>DPG : Medieval Manuscript Exhibition 2014</tag>
+            <tag>DPG : Curator Request : Jordan</tag>
+            <tag>Registered By : astrids</tag>
+            <tag>Remediated By : 4.14.4</tag>
+          </identityMetadata>
+          <contentMetadata type="image" objectId="xf680rd3068">
+            <resource type="image" sequence="1" id="xf680rd3068_1">
+              <label>Image 1</label>
+              <file id="xf680rd3068_00_0001.jp2" mimetype="image/jp2" size="887079">
+                <imageData width="1794" height="2627"/>
+              </file>
+            </resource>
+            <resource type="image" sequence="2" id="xf680rd3068_2">
+              <label>Image 2</label>
+              <file id="xf680rd3068_00_0002.jp2" mimetype="image/jp2" size="887064">
+                <imageData width="1794" height="2627"/>
+              </file>
+            </resource>
+          </contentMetadata>
+          <rightsMetadata>
+            <access type="discover">
+              <machine>
+                <world/>
+              </machine>
+            </access>
+            <access type="read">
+              <machine>
+                <world/>
+              </machine>
+            </access>
+            <use>
+              <human type="useAndReproduction">Property rights reside with the repository. Literary rights reside with the creators of the documents or their heirs. To obtain permission to publish or reproduce, please contact the Special Collections Public Services Librarian at speccollref@stanford.edu.</human>
+            </use>
+            <use>
+              <human type="creativeCommons"/>
+              <machine type="creativeCommons"/>
+            </use>
+          </rightsMetadata>
+          <rdf:RDF xmlns:fedora="info:fedora/fedora-system:def/relations-external#" xmlns:fedora-model="info:fedora/fedora-system:def/model#" xmlns:hydra="http://projecthydra.org/ns/relations#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+            <rdf:Description rdf:about="info:fedora/druid:xf680rd3068">
+              <fedora:isMemberOf rdf:resource="info:fedora/druid:sk373nx0013"/>
+              <fedora:isMemberOfCollection rdf:resource="info:fedora/druid:sk373nx0013"/>
+            </rdf:Description>
+          </rdf:RDF>
+          <oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:srw_dc="info:srw/schema/1/dc-schema" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+            <dc:title>Latin glossary : small manuscript fragment on vellum</dc:title>
+            <dc:date>1200-1299</dc:date>
+            <dc:date>13th century</dc:date>
+            <dc:language>lat</dc:language>
+            <dc:format>1 fragment, 96 x 151 mm.</dc:format>
+            <dc:description>Unidentified Latin vocabulary, this portion part of the letter M, with a total of 35 lemmata (a word or phrase that is glossed; headword).</dc:description>
+            <dc:rights>Open for research; material must be requested at least 24 hours in advance of intended use.</dc:rights>
+            <dc:description>Lower portion of vellum leaf, removed from a binding.</dc:description>
+            <dc:description>Script: professional gotica textualis, 2 columns, with a fair amount of abbreviations.</dc:description>
+            <dc:description type="acquisition">Purchased from Bonhams, 2013. Accession 2013-080.</dc:description>
+            <dc:description type="ownership">Previously in the Bernard M. Rosenthal collection; with Rosenthal's dealer description included with the manuscript, together with a note re previous ownership of Bernard Bischoff.</dc:description>
+            <dc:subject>Manuscripts, Latin (Medieval and modern)</dc:subject>
+            <dc:coverage>France</dc:coverage>
+            <dc:subject>Manuscripts, Latin (Medieval and modern)--France</dc:subject>
+            <dc:relation type="collection">Stanford University Libraries Special Collections, Manuscripts Division</dc:relation>
+          </oai_dc:dc>
+          <ReleaseData/>
+        </publicObject>
+    http_version: 
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
+- request:
+    method: get
+    uri: https://purl.stanford.edu/xf680rd3068.mods
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Tue, 10 Nov 2015 20:02:54 GMT
+      Server:
+      - Apache/2.2.15 (CentOS)
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Xss-Protection:
+      - 1; mode=block
       X-Request-Id:
-      - 3a61c05b-7d28-411e-94bf-c7a493d2bb15
+      - 2a50567b-490b-4840-a030-aca2b0082b36
+      Etag:
+      - '"37e08cb364489ba8b40e25f7e60ef051"'
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Runtime:
+      - '0.007788'
+      X-Content-Type-Options:
+      - nosniff
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -885,7 +1015,7 @@ http_interactions:
           <accessCondition type="useAndReproduction">Property rights reside with the repository. Literary rights reside with the creators of the documents or their heirs. To obtain permission to publish or reproduce, please contact the Special Collections Public Services Librarian at speccollref@stanford.edu.</accessCondition>
         </mods>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:06:00 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
 - request:
     method: get
     uri: https://purl.stanford.edu/xf680rd3068.xml
@@ -905,7 +1035,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:06:00 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -919,9 +1049,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.013537'
+      - '0.006246'
       X-Request-Id:
-      - da760230-9983-4cfa-ba90-59e3b85f715e
+      - 71915f12-b921-45f7-9599-98efef27a72c
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -1015,7 +1145,7 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:06:00 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:53 GMT
 - request:
     method: get
     uri: https://purl.stanford.edu/sk373nx0013.xml
@@ -1035,7 +1165,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:06:00 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -1049,9 +1179,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.007240'
+      - '0.007076'
       X-Request-Id:
-      - 6e1be167-0868-48a2-8128-fb908e2a74f0
+      - 496e6a92-9df7-4c20-a8ea-6c1cea8b9337
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -1116,7 +1246,7 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:06:00 GMT
+  recorded_at: Tue, 10 Nov 2015 20:02:54 GMT
 - request:
     method: get
     uri: https://purl.stanford.edu/sk373nx0013.xml
@@ -1136,7 +1266,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Fri, 30 Oct 2015 14:06:00 GMT
+      - Tue, 10 Nov 2015 20:02:54 GMT
       Server:
       - Apache/2.2.15 (CentOS)
       Cache-Control:
@@ -1150,9 +1280,9 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       X-Runtime:
-      - '0.005477'
+      - '0.007446'
       X-Request-Id:
-      - ebc9a403-3191-46c9-8c5f-cf545dd29e79
+      - 3dd8585e-37a0-4c3b-8167-6820244a14ce
       X-Powered-By:
       - Phusion Passenger 5.0.16
       Strict-Transport-Security:
@@ -1217,5 +1347,5 @@ http_interactions:
           <ReleaseData/>
         </publicObject>
     http_version: 
-  recorded_at: Fri, 30 Oct 2015 14:06:00 GMT
-recorded_with: VCR 2.9.3
+  recorded_at: Tue, 10 Nov 2015 20:02:54 GMT
+recorded_with: VCR 3.0.0