spotlight-dor-resources 0.0.3 → 0.0.4

data/lib/spotlight/dor/resources.rb

@@ -1,15 +1,27 @@
-require "harvestdor-indexer"
-require "spotlight/dor/resources/version"
+require 'harvestdor-indexer'
+require 'spotlight/dor/resources/version'
 
 module Spotlight
   module Dor
+    # Spotlight::Dor::Resources provides a Rails engine
+    # that is capable of harvesting and indexing resources
+    # from Searchworks and PURL endpoints
     module Resources
+      require 'spotlight/dor/indexer'
+      require 'spotlight/dor/resources/engine'
 
-      require "spotlight/dor/indexer"
-      require "spotlight/dor/resources/engine"
-      
-      def self.indexer
-        @indexer ||= Spotlight::Dor::Indexer.new File.join(Rails.root, "config", "gdor.yml"), solr: Blacklight.solr_config
+      class <<self
+        def indexer
+          @indexer ||= Spotlight::Dor::Indexer.new gdor_config_path, solr: solr_config
+        end
+
+        def gdor_config_path
+          File.join(Rails.root, 'config', 'gdor.yml')
+        end
+
+        def solr_config
+          Blacklight.solr_config
+        end
       end
     end
   end

data/lib/spotlight/dor/resources/engine.rb

@@ -2,12 +2,13 @@ require 'spotlight/engine'
 require 'spotlight/dor/resources'
 
 module Spotlight::Dor::Resources
+  # :nodoc:
   class Engine < ::Rails::Engine
-
-    initializer "spotlight.dor.initialize" do
+    initializer 'spotlight.dor.initialize' do
       Spotlight::Engine.config.resource_providers << Spotlight::Resources::Searchworks
       Spotlight::Engine.config.resource_providers << Spotlight::Resources::Purl
       Spotlight::Dor::Resources::Engine.config.parallel_options = { in_threads: 1 }
+      Spotlight::Dor::Resources::Engine.config.stacks_iiif_url = 'https://stacks.stanford.edu/image/iiif'
     end
   end
 end

data/lib/spotlight/dor/resources/version.rb

@@ -1,7 +1,8 @@
 module Spotlight
   module Dor
+    # :nodoc:
     module Resources
-      VERSION = "0.0.3"
+      VERSION = '0.0.4'
     end
   end
 end

data/spec/integration/gdor_integration_spec.rb

@@ -1,30 +1,30 @@
 require 'spec_helper'
 
-describe "gdor indexing integration test", :vcr do
+describe 'gdor indexing integration test', :vcr do
   let :exhibit do
-    double(solr_data: { }, blacklight_config: Blacklight::Configuration.new)
+    double(solr_data: {}, blacklight_config: Blacklight::Configuration.new)
   end
 
   subject do
-    r = Spotlight::Resources::Purl.new(url: "https://purl.stanford.edu/xf680rd3068")
+    r = Spotlight::Resources::Purl.new(url: 'https://purl.stanford.edu/xf680rd3068')
     allow(r).to receive(:to_global_id).and_return('x')
     allow(r).to receive(:exhibit).and_return(exhibit)
     r.to_solr.first
   end
 
-  it "should have a doc id" do
-    expect(subject[:id]).to eq "xf680rd3068"
+  it 'has a doc id' do
+    expect(subject[:id]).to eq 'xf680rd3068'
   end
 
-  it "should have the gdor data" do
+  it 'has the gdor data' do
     expect(subject).to include :collection, :modsxml, :url_fulltext
   end
 
-  it "should have spotlight data" do
+  it 'has spotlight data' do
     expect(subject).to include :spotlight_resource_id_ssim
   end
 
-  it "should have exhibit-specific indexing" do
-    expect(subject).to include "full_image_url_ssm"
+  it 'has exhibit-specific indexing' do
+    expect(subject).to include 'full_image_url_ssm'
   end
 end

data/spec/{unit → lib}/spotlight/dor/indexer_spec.rb

@@ -40,13 +40,151 @@ describe Spotlight::Dor::Indexer do
   end
 
   before do
-    # ignore noisy logs
+    # reduce log noise
     allow(r).to receive(:harvestdor_client)
     i = Harvestdor::Indexer.new
     i.logger.level = Logger::WARN
     allow(r).to receive(:indexer).and_return i
   end
 
+  describe '#add_content_metadata_fields' do
+    before do
+      allow(r).to receive(:public_xml).and_return(public_xml)
+
+      # stacks url calculations require the druid
+      solr_doc[:id] = fake_druid
+
+      subject.send(:add_content_metadata_fields, sdb, solr_doc)
+    end
+
+    context 'with a record without contentMetadata' do
+      let(:public_xml) do
+        Nokogiri::XML <<-EOF
+          <publicObject></publicObject>
+          EOF
+      end
+
+      it 'is blank, except for the document id' do
+        expect(solr_doc.except(:id)).to be_blank
+      end
+    end
+
+    context 'with a record with contentMetadata' do
+      let(:public_xml) do
+        Nokogiri::XML <<-EOF
+          <publicObject>
+            <contentMetadata type="image">
+              <resource id="bj356mh7176_1" sequence="1" type="image">
+                <label>Image 1</label>
+                <file id="bj356mh7176_00_0001.jp2" mimetype="image/jp2" size="56108727">
+                  <imageData width="12967" height="22970"/>
+                </file>
+              </resource>
+            </contentMetadata>
+          </publicObject>
+          EOF
+      end
+
+      it 'indexes the declared content metadata type' do
+        expect(solr_doc['content_metadata_type_ssim']).to contain_exactly 'image'
+      end
+
+      it 'indexes the thumbnail information' do
+        expect(solr_doc['content_metadata_first_image_file_name_ssm']).to contain_exactly 'bj356mh7176_00_0001'
+        expect(solr_doc['content_metadata_first_image_width_ssm']).to contain_exactly '12967'
+        expect(solr_doc['content_metadata_first_image_height_ssm']).to contain_exactly '22970'
+      end
+
+      it 'indexes the images' do
+        stacks_base_url = 'https://stacks.stanford.edu/image/iiif/oo000oo0000%2Fbj356mh7176_00_0001'
+        expect(solr_doc['content_metadata_image_iiif_info_ssm']).to include "#{stacks_base_url}/info.json"
+        expect(solr_doc['thumbnail_square_url_ssm']).to include "#{stacks_base_url}/square/100,100/0/default.jpg"
+        expect(solr_doc['thumbnail_url_ssm']).to include "#{stacks_base_url}/full/!400,400/0/default.jpg"
+        expect(solr_doc['large_image_url_ssm']).to include "#{stacks_base_url}/full/pct:25/0/default.jpg"
+        expect(solr_doc['full_image_url_ssm']).to include "#{stacks_base_url}/full/full/0/default.jpg"
+      end
+    end
+  end
+
+  describe '#add_donor_tags' do
+    before do
+      allow(r).to receive(:mods).and_return(mods)
+      subject.send(:add_donor_tags, sdb, solr_doc)
+    end
+
+    context 'with a record without donor tags' do
+      let(:mods) do
+        Nokogiri::XML <<-EOF
+          <mods xmlns="#{Mods::MODS_NS}">
+            <note displayLabel="preferred citation">(not a donor tag)</note>
+          </mods>
+          EOF
+      end
+
+      it 'is blank' do
+        expect(solr_doc['donor_tags_ssim']).to be_blank
+      end
+    end
+
+    context 'with a record with donor tags' do
+      let(:mods) do
+        # e.g. from https://purl.stanford.edu/vw282gv1740
+        Nokogiri::XML <<-EOF
+          <mods xmlns="#{Mods::MODS_NS}">
+            <note displayLabel="Donor tags">Knowledge Systems Laboratory</note>
+            <note displayLabel="Donor tags">medical applications</note>
+            <note displayLabel="Donor tags">Publishing</note>
+            <note displayLabel="Donor tags">Stanford</note>
+            <note displayLabel="Donor tags">Stanford Computer Science Department</note>
+          </mods>
+          EOF
+      end
+
+      it 'extracts the donor tags' do
+        expect(solr_doc['donor_tags_ssim']).to contain_exactly 'Knowledge Systems Laboratory',
+                                                               'medical applications',
+                                                               'Publishing',
+                                                               'Stanford',
+                                                               'Stanford Computer Science Department'
+      end
+    end
+  end
+
+  describe '#add_genre' do
+    before do
+      allow(r).to receive(:mods).and_return(mods)
+      subject.send(:add_genre, sdb, solr_doc)
+    end
+
+    context 'with a record without a genre' do
+      let(:mods) do
+        Nokogiri::XML <<-EOF
+          <mods xmlns="#{Mods::MODS_NS}">
+          </mods>
+          EOF
+      end
+
+      it 'is blank' do
+        expect(solr_doc['genre_ssim']).to be_blank
+      end
+    end
+
+    context 'with a record with a genre' do
+      let(:mods) do
+        # e.g. from https://purl.stanford.edu/vw282gv1740
+        Nokogiri::XML <<-EOF
+          <mods xmlns="#{Mods::MODS_NS}">
+            <genre authority="aat" valueURI="http://vocab.getty.edu/aat/300028579">manuscripts for publication</genre>
+          </mods>
+          EOF
+      end
+
+      it 'extracts the genre' do
+        expect(solr_doc['genre_ssim']).to contain_exactly 'manuscripts for publication'
+      end
+    end
+  end
+
   describe '#add_series' do
     # example string as key, expected series name as value
     {
@@ -110,7 +248,7 @@ describe Spotlight::Dor::Indexer do
     end # each
   end # add_series
 
-  describe "#add_box" do
+  describe '#add_box' do
     # example string as key, expected box name as value
     {
       # feigenbaum
@@ -147,7 +285,7 @@ describe Spotlight::Dor::Indexer do
             allow(r).to receive(:mods).and_return(mods_loc_phys_loc)
             subject.send(:add_box, sdb, solr_doc)
           end
-          it "has the expected box name '#{expected}'" do
+          it "has the expected box label '#{expected}'" do
             expect(solr_doc['box_ssi']).to eq expected
           end
         end
@@ -156,7 +294,7 @@ describe Spotlight::Dor::Indexer do
             allow(r).to receive(:mods).and_return(mods_rel_item_loc_phys_loc)
             subject.send(:add_box, sdb, solr_doc)
           end
-          it "has the expected box name '#{expected}'" do
+          it "has the expected box label '#{expected}'" do
             expect(solr_doc['box_ssi']).to eq expected
           end
         end
@@ -166,7 +304,7 @@ describe Spotlight::Dor::Indexer do
             allow(r).to receive(:mods).and_return(mods_loc_multiple_phys_loc)
             subject.send(:add_box, sdb, solr_doc)
           end
-          it "has the expected series name '#{expected}'" do
+          it "has the expected box label '#{expected}'" do
             expect(solr_doc['box_ssi']).to eq expected
           end
         end
@@ -213,7 +351,7 @@ describe Spotlight::Dor::Indexer do
             allow(r).to receive(:mods).and_return(mods_loc_phys_loc)
             subject.send(:add_folder, sdb, solr_doc)
           end
-          it "has the expected folder name '#{expected}'" do
+          it "has the expected folder label '#{expected}'" do
             expect(solr_doc['folder_ssi']).to eq expected
           end
         end
@@ -222,7 +360,7 @@ describe Spotlight::Dor::Indexer do
             allow(r).to receive(:mods).and_return(mods_rel_item_loc_phys_loc)
             subject.send(:add_folder, sdb, solr_doc)
           end
-          it "has the expected folder name '#{expected}'" do
+          it "has the expected folder label '#{expected}'" do
             expect(solr_doc['folder_ssi']).to eq expected
           end
         end
@@ -232,7 +370,7 @@ describe Spotlight::Dor::Indexer do
             allow(r).to receive(:mods).and_return(mods_loc_multiple_phys_loc)
             subject.send(:add_folder, sdb, solr_doc)
           end
-          it "has the expected series name '#{expected}'" do
+          it "has the expected folder label '#{expected}'" do
             expect(solr_doc['folder_ssi']).to eq expected
           end
         end
@@ -240,6 +378,64 @@ describe Spotlight::Dor::Indexer do
     end # each
   end # add_folder
 
+  # rubocop:disable Metrics/LineLength
+  describe '#add_location' do
+    # example string as key, expected box name as value
+    {
+      # feigenbaum
+      'Call Number: SC0340, Accession 2005-101, Box : 1, Folder: 1': 'Call Number: SC0340, Accession 2005-101, Box : 1, Folder: 1',
+      'Call Number: SC0340, Accession 2005-101': 'Call Number: SC0340, Accession 2005-101',
+      'SC0340, 1986-052, Box 18': 'SC0340, 1986-052, Box 18',
+      'SC0340, Accession 2005-101, Box 18': 'SC0340, Accession 2005-101, Box 18',
+      'SC0340': nil,
+      'SC0340, Accession 1986-052': 'SC0340, Accession 1986-052',
+      'Stanford University. Libraries. Department of Special Collections and University Archives': nil,
+      # shpc (actually in <relatedItem><location><physicalLocation>)
+      'Series Biographical Photographs | Box 42 | Folder Abbot, Nathan': 'Series Biographical Photographs | Box 42 | Folder Abbot, Nathan',
+      'Series General Photographs | Box 42 | Folder Administration building--Outer Quad': 'Series General Photographs | Box 42 | Folder Administration building--Outer Quad',
+      # menuez
+      'MSS Photo 451, Series 1, Box 32, Folder 11, Sleeve 32-11-2, Frame B32-F11-S2-6': 'MSS Photo 451, Series 1, Box 32, Folder 11, Sleeve 32-11-2, Frame B32-F11-S2-6',
+      'Series 1, Box 10, Folder 8': 'Series 1, Box 10, Folder 8',
+      # fuller
+      'Collection: M1090 , Series: 1 , Box: 5 , Folder: 42': 'Collection: M1090 , Series: 1 , Box: 5 , Folder: 42',
+      # hummel (actually in <relatedItem><location><physicalLocation>)
+      'Box 42 | Folder 3': 'Box 42 | Folder 3',
+      'Flat-box 228 | Volume 1': 'Flat-box 228 | Volume 1'
+    }.each do |example, expected|
+      describe "for example '#{example}'" do
+        let(:example) { example }
+        context 'in /location/physicalLocation' do
+          before do
+            allow(r).to receive(:mods).and_return(mods_loc_phys_loc)
+            subject.send(:add_location, sdb, solr_doc)
+          end
+          it "has the expected location '#{expected}'" do
+            expect(solr_doc['location_ssi']).to eq expected
+          end
+        end
+        context 'in /relatedItem/location/physicalLocation' do
+          before do
+            allow(r).to receive(:mods).and_return(mods_rel_item_loc_phys_loc)
+            subject.send(:add_location, sdb, solr_doc)
+          end
+          it "has the expected location '#{expected}'" do
+            expect(solr_doc['location_ssi']).to eq expected
+          end
+        end
+        context 'with multiple physicalLocation elements' do
+          before do
+            allow(r).to receive(:mods).and_return(mods_loc_multiple_phys_loc)
+            subject.send(:add_location, sdb, solr_doc)
+          end
+          it "has the expected location '#{expected}'" do
+            expect(solr_doc['location_ssi']).to eq expected
+          end
+        end
+      end # for example
+    end # each
+  end # add_location
+  # rubocop:enable Metrics/LineLength
+
   let(:mods_note_plain) do
     Nokogiri::XML <<-EOF
       <mods xmlns="#{Mods::MODS_NS}">
@@ -254,9 +450,11 @@ describe Spotlight::Dor::Indexer do
       </mods>
     EOF
   end
-  describe "#add_folder_name" do
+
+  # rubocop:disable Metrics/LineLength
+  describe '#add_folder_name' do
     # example string as key, expected folder name as value
-    # all from feigenbaum (or based on feigenbaum), as that is only coll
+    # all from feigenbaum (or based on feigenbaum), as that is only coll with this data
     {
       'Call Number: SC0340, Accession: 1986-052, Box: 20, Folder: 40, Title: S': 'S',
       'Call Number: SC0340, Accession: 1986-052, Box: 54, Folder: 25, Title: Balzer': 'Balzer',
@@ -299,4 +497,5 @@ describe Spotlight::Dor::Indexer do
       end # for example
     end # each
   end # add_folder_name
+  # rubocop:enable Metrics/LineLength
 end

data/spec/models/spotlight/resources/purl_spec.rb

@@ -2,13 +2,13 @@ require 'spec_helper'
 
 describe Spotlight::Resources::Purl do
   let :exhibit do
-    double(solr_data: { }, blacklight_config: Blacklight::Configuration.new)
+    double(solr_data: {}, blacklight_config: Blacklight::Configuration.new)
   end
   let :blacklight_solr do
     double
   end
 
-  subject { Spotlight::Resources::Purl.new url: "http://purl.stanford.edu/xf680rd3068" }
+  subject { described_class.new url: 'http://purl.stanford.edu/xf680rd3068' }
 
   before do
     allow(subject).to receive(:exhibit).and_return(exhibit)
@@ -16,98 +16,104 @@ describe Spotlight::Resources::Purl do
     allow(subject).to receive(:to_global_id).and_return('x')
   end
 
-  describe ".can_provide?" do
-    subject { Spotlight::Resources::Purl }
-    it "should be able to provide any purl URL" do
-      expect(subject.can_provide?(double(url: "https://purl.stanford.edu/xyz"))).to eq true
-      expect(subject.can_provide?(double(url: "http://purl.stanford.edu/xyz"))).to eq true
+  describe '.can_provide?' do
+    subject { described_class }
+    it 'is true for a PURL URL' do
+      expect(subject.can_provide?(double(url: 'https://purl.stanford.edu/xyz'))).to eq true
+      expect(subject.can_provide?(double(url: 'http://purl.stanford.edu/xyz'))).to eq true
+    end
+
+    it 'is false other URLs' do
+      expect(subject.can_provide?(double(url: 'https://example.com/xyz'))).to eq false
     end
   end
 
-  describe "#doc_id" do
-    it "should be able to extract DRUIDs from a PURL url" do
-      subject.url = "http://purl.stanford.edu/xyz"
-      expect(subject.doc_id).to eq "xyz"
+  describe '#doc_id' do
+    it 'extracts DRUIDs from a PURL url' do
+      subject.url = 'http://purl.stanford.edu/xyz'
+      expect(subject.doc_id).to eq 'xyz'
     end
 
-    it "should be able to extract DRUIDs from a PURL format url" do
-      subject.url = "http://purl.stanford.edu/xf680rd3068.xml"
-      expect(subject.doc_id).to eq "xf680rd3068"
+    it 'extracts DRUIDs from a PURL format url' do
+      subject.url = 'http://purl.stanford.edu/xf680rd3068.xml'
+      expect(subject.doc_id).to eq 'xf680rd3068'
     end
 
-    it "should be able to extract DRUIDs from a PURL's viewer url" do
-      subject.url = "http://purl.stanford.edu/xf680rd3068#image/1/small"
-      expect(subject.doc_id).to eq "xf680rd3068"
+    it "extracts DRUIDs from a PURL's viewer url" do
+      subject.url = 'http://purl.stanford.edu/xf680rd3068#image/1/small'
+      expect(subject.doc_id).to eq 'xf680rd3068'
     end
   end
 
-  describe "#resource" do
-    it "should be a Harvestdor::Indexer resource" do
+  describe '#resource' do
+    it 'is a Harvestdor::Indexer resource' do
       expect(subject.resource).to be_a_kind_of Harvestdor::Indexer::Resource
     end
 
-    it "should have the correct druid" do
-      expect(subject.resource.druid).to eq "xf680rd3068"
+    it 'has the correct druid' do
+      expect(subject.resource.druid).to eq 'xf680rd3068'
     end
 
-    it "should have the correct indexer" do
+    it 'has the correct indexer' do
       expect(subject.resource.indexer).to eq Spotlight::Dor::Resources.indexer.harvestdor
     end
   end
 
-  describe "#reindex" do
+  describe '#reindex' do
     before do
-      allow(Spotlight::Dor::Resources.indexer).to receive(:solr_document).and_return({upstream: true})
+      allow(Spotlight::Dor::Resources.indexer).to receive(:solr_document).and_return(upstream: true)
       allow(subject.resource).to receive(:collection?).and_return(false)
     end
 
-    it "should add a document to solr" do
-      solr_data = [{spotlight_resource_id_ssim: nil, spotlight_resource_type_ssim: "spotlight/resources/purls", upstream: true}]
-      expect(blacklight_solr).to receive(:update).with({params: {commitWithin: 500}, data: solr_data.to_json, headers: {"Content-Type" => "application/json"}})
+    it 'adds a document to solr' do
+      solr_data = [{ spotlight_resource_id_ssim: nil, spotlight_resource_type_ssim: 'spotlight/resources/purls', upstream: true }]
+      expect(blacklight_solr).to receive(:update).with(params: { commitWithin: 500 },
+                                                       data: solr_data.to_json,
+                                                       headers: { 'Content-Type' => 'application/json' })
       expect(subject).to receive(:update_index_time!)
       subject.reindex
     end
   end
 
-  describe "#to_solr" do
+  describe '#to_solr' do
     before do
       allow(Spotlight::Dor::Resources.indexer).to receive(:solr_document)
     end
-    context "with a collection" do
+    context 'with a collection' do
       before do
         allow(subject.resource).to receive(:collection?).and_return(true)
       end
 
-      it "should provide a solr document for the collection" do
+      it 'provides a solr document for the collection' do
         allow(subject.resource).to receive(:items).and_return([])
-        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({upstream: true})
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return(upstream: true)
         expect(subject.to_solr.first).to include :upstream, :spotlight_resource_id_ssim, :spotlight_resource_type_ssim
       end
 
-      it "should provide a solr document for the items too" do
+      it 'provides a solr document for the items too' do
         item = double
         allow(subject.resource).to receive(:items).and_return([item])
-        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({collection: true})
-        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(item).and_return({item: true})
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return(collection: true)
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(item).and_return(item: true)
         solr_doc = subject.to_solr.to_a
         expect(solr_doc.first).to include :collection
         expect(solr_doc.last).to include :item
       end
     end
 
-    context "with a single item" do
+    context 'with a single item' do
       before do
         allow(subject.resource).to receive(:collection?).and_return(false)
       end
 
-      it "should provide a solr document for the resource" do
-        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({upstream: true})
+      it 'provides a solr document for the resource' do
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return(upstream: true)
         expect(subject.to_solr.first).to include :upstream, :spotlight_resource_id_ssim, :spotlight_resource_type_ssim
       end
 
-      it "should index outside the context of an exhibit" do
+      it 'indexs outside the context of an exhibit' do
         allow(subject).to receive(:exhibit).and_return(nil)
-        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({upstream: true})
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return(upstream: true)
         expect(subject.to_solr.first).to include :upstream, :spotlight_resource_id_ssim, :spotlight_resource_type_ssim
       end
     end