spotlight-dor-resources 0.0.1

data/solr_conf/conf/solrconfig.xml

@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<config>
+  <!-- NOTE: various comments and unused configuration possibilities have been purged
+     from this file.  Please refer to http://wiki.apache.org/solr/SolrConfigXml,
+     as well as the default solrconfig file included with Solr -->
+  
+  <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
+  
+  <luceneMatchVersion>LUCENE_40</luceneMatchVersion>
+
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
+
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <updateLog>
+      <str name="dir">${solr.core0.data.dir:}</str>
+    </updateLog>
+  </updateHandler>
+
+  <!-- solr lib dirs -->
+  <lib dir="../lib/contrib/analysis-extras/lib" />
+  <lib dir="../lib/contrib/analysis-extras/lucene-libs" />
+  
+  <dataDir>${solr.data.dir:}</dataDir>
+
+  <requestHandler name="search" class="solr.SearchHandler" default="true">
+    <!-- default values for query parameters can be specified, these
+         will be overridden by parameters in the request
+      -->
+     <lst name="defaults">
+       <str name="defType">edismax</str>
+       <str name="echoParams">explicit</str>
+       <str name="q.alt">*:*</str>
+       <str name="mm">2&lt;-1 5&lt;-2 6&lt;90%</str>
+       <int name="qs">1</int>
+       <int name="ps">2</int>
+       <float name="tie">0.01</float>
+       <!-- this qf and pf are used by default, if not otherwise specified by
+            client. The default blacklight_config will use these for the
+            "keywords" search. See the author_qf/author_pf, title_qf, etc 
+            below, which the default blacklight_config will specify for
+            those searches. You may also be interested in:
+            http://wiki.apache.org/solr/LocalParams
+       -->
+        <str name="qf">
+          id
+          full_title_tesim
+          short_title_tesim
+          alternative_title_tesim
+          active_fedora_model_ssi
+          title_tesim
+          author_tesim
+          subject_tesim
+          all_text_timv
+        </str>
+        <str name="pf">
+          all_text_timv^10
+        </str>
+
+       <str name="author_qf">
+          author_tesim
+       </str>
+       <str name="author_pf">
+       </str>
+       <str name="title_qf">
+          title_tesim
+          full_title_tesim
+          short_title_tesim
+          alternative_title_tesim
+       </str>
+       <str name="title_pf">
+       </str>
+       <str name="subject_qf">
+          subject_tesim
+       </str>
+       <str name="subject_pf">
+       </str>
+       
+       <str name="fl">
+         *, 
+         score
+       </str>
+
+       <str name="facet">true</str>
+       <str name="facet.mincount">1</str>
+       <str name="facet.limit">10</str>
+       <str name="facet.field">active_fedora_model_ssi</str>
+       <str name="facet.field">subject_ssim</str>
+       
+       <str name="spellcheck">true</str>
+       <str name="spellcheck.dictionary">default</str>
+       <str name="spellcheck.onlyMorePopular">true</str>
+       <str name="spellcheck.extendedResults">true</str>
+       <str name="spellcheck.collate">false</str>
+       <str name="spellcheck.count">5</str>
+
+     </lst>
+    <arr name="last-components">
+      <str>spellcheck</str>
+    </arr>
+  </requestHandler>
+
+  <requestHandler name="permissions" class="solr.SearchHandler" >
+    <lst name="defaults">
+      <str name="facet">off</str>
+      <str name="echoParams">all</str>
+      <str name="rows">1</str>
+      <str name="q">{!raw f=id v=$id}</str> <!-- use id=666 instead of q=id:666 -->
+      <str name="fl">
+        id,
+        access_ssim,
+        discover_access_group_ssim,discover_access_person_ssim,
+        read_access_group_ssim,read_access_person_ssim,
+        edit_access_group_ssim,edit_access_person_ssim,
+        depositor_ti,
+        embargo_release_date_dtsi
+        inheritable_access_ssim,
+        inheritable_discover_access_group_ssim,inheritable_discover_access_person_ssim,
+        inheritable_read_access_group_ssim,inheritable_read_access_person_ssim,
+        inheritable_edit_access_group_ssim,inheritable_edit_access_person_ssim,
+        inheritable_embargo_release_date_dtsi
+      </str>
+    </lst>
+  </requestHandler>
+  
+  <requestHandler name="standard" class="solr.SearchHandler">
+     <lst name="defaults">
+       <str name="echoParams">explicit</str>
+       <str name="defType">lucene</str>
+     </lst>
+  </requestHandler>
+
+  <!-- for requests to get a single document; use id=666 instead of q=id:666 -->
+  <requestHandler name="document" class="solr.SearchHandler" >
+    <lst name="defaults">
+      <str name="echoParams">all</str>
+      <str name="fl">*</str>
+      <str name="rows">1</str>
+      <str name="q">{!raw f=id v=$id}</str> <!-- use id=666 instead of q=id:666 -->
+    </lst>
+  </requestHandler>
+
+
+  <searchComponent name="spellcheck" class="solr.SpellCheckComponent">
+    <str name="queryAnalyzerFieldType">textSpell</str>
+    <!-- Multiple "Spell Checkers" can be declared and used by this component
+      (e.g. for title_spell field)
+      -->
+    <lst name="spellchecker">
+      <str name="name">default</str>
+      <str name="field">spell</str>
+      <str name="spellcheckIndexDir">./spell</str>
+      <str name="buildOnOptimize">true</str>
+    </lst>
+  </searchComponent>
+  
+  <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" /> 
+  
+  <requestDispatcher handleSelect="true" >
+    <requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048" />
+  </requestDispatcher>
+  
+  <requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
+  <requestHandler name="/update" class="solr.UpdateRequestHandler"  />
+  <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
+  
+  <requestHandler name="/admin/ping" class="solr.PingRequestHandler">
+    <lst name="invariants">
+      <str name="q">solrpingquery</str>
+    </lst>
+    <lst name="defaults">
+      <str name="echoParams">all</str>
+    </lst>
+  </requestHandler>
+  
+  <!-- config for the admin interface --> 
+  <admin>
+    <defaultQuery>search</defaultQuery>
+  </admin>
+  
+</config>

data/spec/integration/gdor_integration_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe "gdor indexing integration test", :vcr do
+  let :exhibit do
+    double(solr_data: { }, blacklight_config: Blacklight::Configuration.new)
+  end
+
+  subject do
+    r = Spotlight::Resources::Purl.new(url: "https://purl.stanford.edu/xf680rd3068")
+    allow(r).to receive(:to_global_id).and_return('x')
+    allow(r).to receive(:exhibit).and_return(exhibit)
+    r.to_solr.first
+  end
+
+  it "should have a doc id" do
+    expect(subject[:id]).to eq "xf680rd3068"
+  end
+
+  it "should have the gdor data" do
+    expect(subject).to include :collection, :modsxml, :url_fulltext
+  end
+
+  it "should have spotlight data" do
+    expect(subject).to include :spotlight_resource_id_ssim
+  end
+
+  it "should have exhibit-specific indexing" do
+    expect(subject).to include "full_image_url_ssm"
+  end
+end

data/spec/integration/indexer_integration_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe 'indexer integration tests', :vcr do
+  describe 'donor tags' do
+    it 'solr_doc has donor_tags_ssim field when <note displayLabel="Donor tags"> is in MODS' do
+      r = Spotlight::Resources::Purl.new(url: 'https://purl.stanford.edu/vw282gv1740') # Feigenbaum PURL with donor tags
+      solr_doc = r.to_solr.first
+      expect(solr_doc['donor_tags_ssim']).to eq ['Knowledge Systems Laboratory', 'medical applications', 'Publishing', 'Stanford', 'Stanford Computer Science Department']
+    end
+    it 'no donor_tags_ssim field in solr doc when <note displayLabel="Donor tags"> not in MODS' do
+      r = Spotlight::Resources::Purl.new(url: 'https://purl.stanford.edu/bd955gr0721') # Revs PURL without donor tags
+      solr_doc = r.to_solr.first
+      expect(solr_doc['donor_tags_ssim']).to be_nil
+    end
+  end
+  describe 'genre' do
+    it 'solr_doc has genre_ssim field when <genre> in MODS' do
+      r = Spotlight::Resources::Purl.new(url: 'https://purl.stanford.edu/vw282gv1740') # Feigenbaum PURL with genre
+      solr_doc = r.to_solr.first
+      expect(solr_doc['genre_ssim']).to eq ['manuscripts for publication']
+    end
+    it 'no genre_ssim field when <genre> not in MODS' do
+      r = Spotlight::Resources::Purl.new(url: 'https://purl.stanford.edu/pz816zm7931') # Road & Track PURL without genre
+      solr_doc = r.to_solr.first
+      expect(solr_doc['genre_ssim']).to be_nil
+    end
+  end
+end

data/spec/models/spotlight/resources/purl_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper'
+
+describe Spotlight::Resources::Purl do
+  let :exhibit do
+    double(solr_data: { }, blacklight_config: Blacklight::Configuration.new)
+  end
+  let :blacklight_solr do
+    double
+  end
+
+  subject { Spotlight::Resources::Purl.new url: "http://purl.stanford.edu/xf680rd3068" }
+
+  before do
+    allow(subject).to receive(:exhibit).and_return(exhibit)
+    allow(subject).to receive(:blacklight_solr).and_return(blacklight_solr)
+    allow(subject).to receive(:to_global_id).and_return('x')
+  end
+
+  describe ".can_provide?" do
+    subject { Spotlight::Resources::Purl }
+    it "should be able to provide any purl URL" do
+      expect(subject.can_provide?(double(url: "https://purl.stanford.edu/xyz"))).to eq true
+      expect(subject.can_provide?(double(url: "http://purl.stanford.edu/xyz"))).to eq true
+    end
+  end
+
+  describe "#doc_id" do
+    it "should be able to extract DRUIDs from a PURL url" do
+      subject.url = "http://purl.stanford.edu/xyz"
+      expect(subject.doc_id).to eq "xyz"
+    end
+
+    it "should be able to extract DRUIDs from a PURL format url" do
+      subject.url = "http://purl.stanford.edu/xf680rd3068.xml"
+      expect(subject.doc_id).to eq "xf680rd3068"
+    end
+
+    it "should be able to extract DRUIDs from a PURL's viewer url" do
+      subject.url = "http://purl.stanford.edu/xf680rd3068#image/1/small"
+      expect(subject.doc_id).to eq "xf680rd3068"
+    end
+  end
+
+  describe "#resource" do
+    it "should be a Harvestdor::Indexer resource" do
+      expect(subject.resource).to be_a_kind_of Harvestdor::Indexer::Resource
+    end
+
+    it "should have the correct druid" do
+      expect(subject.resource.druid).to eq "xf680rd3068"
+    end
+
+    it "should have the correct indexer" do
+      expect(subject.resource.indexer).to eq Spotlight::Dor::Resources.indexer.harvestdor
+    end
+  end
+
+  describe "#reindex" do
+    before do
+      allow(Spotlight::Dor::Resources.indexer).to receive(:solr_document).and_return({upstream: true})
+      allow(subject.resource).to receive(:collection?).and_return(false)
+    end
+
+    it "should add a document to solr" do
+      solr_data = [{spotlight_resource_id_ssim: nil, spotlight_resource_type_ssim: "spotlight/resources/purls", upstream: true}]
+      expect(blacklight_solr).to receive(:update).with({params: {commitWithin: 500}, data: solr_data.to_json, headers: {"Content-Type" => "application/json"}})
+      expect(subject).to receive(:update_index_time!)
+      subject.reindex
+    end
+  end
+
+  describe "#to_solr" do
+    before do
+      allow(Spotlight::Dor::Resources.indexer).to receive(:solr_document)
+    end
+    context "with a collection" do
+      before do
+        allow(subject.resource).to receive(:collection?).and_return(true)
+      end
+
+      it "should provide a solr document for the collection" do
+        allow(subject.resource).to receive(:items).and_return([])
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({upstream: true})
+        expect(subject.to_solr.first).to include :upstream, :spotlight_resource_id_ssim, :spotlight_resource_type_ssim
+      end
+
+      it "should provide a solr document for the items too" do
+        item = double
+        allow(subject.resource).to receive(:items).and_return([item])
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({collection: true})
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(item).and_return({item: true})
+        solr_doc = subject.to_solr.to_a
+        expect(solr_doc.first).to include :collection
+        expect(solr_doc.last).to include :item
+      end
+    end
+
+    context "with a single item" do
+      before do
+        allow(subject.resource).to receive(:collection?).and_return(false)
+      end
+
+      it "should provide a solr document for the resource" do
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({upstream: true})
+        expect(subject.to_solr.first).to include :upstream, :spotlight_resource_id_ssim, :spotlight_resource_type_ssim
+      end
+
+      it "should index outside the context of an exhibit" do
+        allow(subject).to receive(:exhibit).and_return(nil)
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({upstream: true})
+        expect(subject.to_solr.first).to include :upstream, :spotlight_resource_id_ssim, :spotlight_resource_type_ssim
+      end
+    end
+  end
+end

data/spec/models/spotlight/resources/searchworks_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+
+describe Spotlight::Resources::Searchworks do
+  
+  let :exhibit do
+    double(solr_data: { }, blacklight_config: Blacklight::Configuration.new)
+  end
+
+  subject { Spotlight::Resources::Searchworks.new url: "http://searchworks.stanford.edu/view/xf680rd3068" }
+
+  before do
+    allow(subject).to receive(:exhibit).and_return(exhibit)
+    allow(subject).to receive(:to_global_id).and_return('x')
+  end
+
+  describe ".can_provide?" do
+    subject { Spotlight::Resources::Searchworks }
+    it "should be able to provide any searchworks URL" do
+      expect(subject.can_provide?(double(url: "https://searchworks.stanford.edu/xyz"))).to eq true
+      expect(subject.can_provide?(double(url: "http://searchworks.stanford.edu/xyz"))).to eq true
+    end
+    it "should also work with searchworks-test URLs" do
+      expect(subject.can_provide?(double(url: "https://searchworks-test.stanford.edu/xyz"))).to eq true
+      expect(subject.can_provide?(double(url: "http://searchworks-test.stanford.edu/xyz"))).to eq true
+    end
+  end
+  
+  describe "#doc_id" do
+    it "should be able to extract DRUIDs from a searchworks url" do
+      subject.url = "http://searchworks.stanford.edu/view/xyz"
+      expect(subject.doc_id).to eq "xyz"
+    end
+    
+    it "should be able to extract DRUIDs from a searchworks format url" do
+      subject.url = "http://searchworks.stanford.edu/view/xf680rd3068.xml"
+      expect(subject.doc_id).to eq "xf680rd3068"
+    end
+  end
+  
+  describe "#resource" do
+    it "should be a Harvestdor::Indexer resource" do
+      expect(subject.resource).to be_a_kind_of Harvestdor::Indexer::Resource
+    end
+    
+    it "should have the correct druid" do
+      expect(subject.resource.druid).to eq "xf680rd3068"
+    end
+    
+    it "should have the correct indexer" do
+      expect(subject.resource.indexer).to eq Spotlight::Dor::Resources.indexer.harvestdor
+    end
+  end
+  
+  describe "#to_solr" do
+    before do
+      allow(Spotlight::Dor::Resources.indexer).to receive(:solr_document)
+    end
+    context "with a collection" do
+      before do
+        allow(subject.resource).to receive(:collection?).and_return(true)
+      end
+      
+      it "should provide a solr document for the collection" do
+        allow(subject.resource).to receive(:items).and_return([])
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({upstream: true})
+        expect(subject.to_solr.first).to include :upstream, :spotlight_resource_id_ssim, :spotlight_resource_type_ssim
+      end
+      
+      it "should provide a solr document for the items too" do
+        item = double
+        allow(subject.resource).to receive(:items).and_return([item])
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({collection: true})
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(item).and_return({item: true})
+        solr_doc = subject.to_solr.to_a
+        expect(solr_doc.first).to include :collection
+        expect(solr_doc.last).to include :item
+      end
+    end
+    
+    context "with a single item" do
+      before do
+        allow(subject.resource).to receive(:collection?).and_return(false)
+      end
+      
+      it "should provide a solr document for the resource" do
+        expect(Spotlight::Dor::Resources.indexer).to receive(:solr_document).with(subject.resource).and_return({upstream: true})
+        expect(subject.to_solr.first).to include :upstream, :spotlight_resource_id_ssim, :spotlight_resource_type_ssim
+      end
+    end
+  end
+end