harvestdor-indexer 1.0.4 → 2.0.0

data/spec/fixtures/vcr_cassettes/single_rsolr_connection_call.yml

@@ -2,47 +2,100 @@
 http_interactions:
 - request:
     method: get
-    uri: http://127.0.0.1:3000/
+    uri: http://purl.stanford.edu/yg867hg1375.xml
     body:
       encoding: US-ASCII
       string: ''
     headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
-      - '*/*'
+      - "*/*"
       User-Agent:
       - Ruby
   response:
     status:
       code: 200
-      message: OK
+      message: ''
     headers:
-      X-Frame-Options:
-      - SAMEORIGIN
-      X-Xss-Protection:
-      - 1; mode=block
-      X-Content-Type-Options:
-      - nosniff
-      Content-Type:
-      - text/html; charset=utf-8
+      Date:
+      - Wed, 17 Dec 2014 19:39:37 GMT
+      Server:
+      - Apache/2.2.15 (Red Hat)
+      X-Powered-By:
+      - Phusion Passenger (mod_rails/mod_rack) 3.0.19
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
       Etag:
-      - '"444bcb3a3fcf8389296c49467f27e1d6"'
+      - '"67aa6d1481ba1537ae63af5aaf493f84"'
       Cache-Control:
       - max-age=0, private, must-revalidate
-      X-Meta-Request-Version:
-      - 0.3.4
       X-Request-Id:
-      - 4fef0b48-9ee9-4ef8-811e-78b8c0f643f0
+      - f2e753d56bf896cde6e941be0f51d05a
       X-Runtime:
-      - '0.005477'
-      Connection:
-      - close
-      Server:
-      - thin 1.6.2 codename Doc Brown
+      - '0.007983'
+      X-Rack-Cache:
+      - miss
+      Status:
+      - '200'
+      Content-Length:
+      - '2180'
+      Content-Type:
+      - application/xml; charset=utf-8
     body:
-      encoding: US-ASCII
-      string: ok
-    http_version:
-  recorded_at: Wed, 12 Nov 2014 19:34:02 GMT
+      encoding: UTF-8
+      string: |
+        <publicObject id="druid:yg867hg1375" published="2013-11-11T15:34:32-08:00">
+          <identityMetadata>
+            <objectId>druid:yg867hg1375</objectId>
+            <objectCreator>DOR</objectCreator>
+            <objectLabel>Francis E. Stafford photographs, 1909-1933</objectLabel>
+            <objectType>collection</objectType>
+            <adminPolicy>druid:vb546ms7107</adminPolicy>
+            <otherId name="catkey">9615156</otherId>
+            <otherId name="uuid">8f1feb20-4b29-11e3-8e31-0050569b3c3c</otherId>
+            <tag>Remediated By : 3.25.3</tag>
+          </identityMetadata>
+          <xml/>
+          <rightsMetadata>
+            <access type="discover">
+              <machine>
+                <world/>
+              </machine>
+            </access>
+            <access type="read">
+              <machine>
+                <world/>
+              </machine>
+            </access>
+            <use>
+              <human type="useAndReproduction"/>
+              <human type="creativeCommons"/>
+              <machine type="creativeCommons"/>
+            </use>
+            <copyright>
+              <human/>
+            </copyright>
+          </rightsMetadata>
+          <rdf:RDF xmlns:fedora-model="info:fedora/fedora-system:def/model#" xmlns:hydra="http://projecthydra.org/ns/relations#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+            <rdf:Description rdf:about="info:fedora/druid:yg867hg1375">
+            </rdf:Description>
+          </rdf:RDF>
+          <oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:srw_dc="info:srw/schema/1/dc-schema" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+            <dc:title>Francis E. Stafford photographs, 1909-1933</dc:title>
+            <dc:contributor>Stafford, Francis E., 1884-1938</dc:contributor>
+            <dc:type>Collection</dc:type>
+            <dc:date>1909-1933</dc:date>
+            <dc:language>und</dc:language>
+            <dc:format>3 oversize boxes.</dc:format>
+            <dc:description>Photographs of scenes in China, mainly between 1909 and 1915.</dc:description>
+            <dc:rights>Closed. Digital use copies available.</dc:rights>
+            <dc:description type="biographical/historical">American missionary in China, 1909-1915 and 1932-1933.</dc:description>
+            <dc:coverage>China</dc:coverage>
+          </oai_dc:dc>
+        </publicObject>
+    http_version: 
+  recorded_at: Wed, 17 Dec 2014 19:39:38 GMT
 - request:
     method: get
     uri: http://127.0.0.1:3000/collections/yg867hg1375
@@ -74,9 +127,9 @@ http_interactions:
       X-Meta-Request-Version:
       - 0.3.4
       X-Request-Id:
-      - 26936654-94cb-4135-b867-53f9ba72d1b5
+      - 125a9964-6326-4114-9f59-fb533551d554
       X-Runtime:
-      - '0.011706'
+      - '0.011086'
       Connection:
       - close
       Server:
@@ -95,5 +148,5 @@ http_interactions:
         B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
         social customs, and people."}],"counts":{"collections":1,"items":5,"total_count":6}}'
     http_version:
-  recorded_at: Wed, 12 Nov 2014 19:34:02 GMT
+  recorded_at: Wed, 12 Nov 2014 19:34:03 GMT
 recorded_with: VCR 2.9.3

data/spec/spec_helper.rb

@@ -4,7 +4,7 @@ Coveralls.wear!
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
-require 'harvestdor-indexer'
+require 'harvestdor/indexer'
 
 require 'vcr'
 

data/spec/unit/harvestdor-indexer-resource_spec.rb

@@ -0,0 +1,174 @@
+require 'spec_helper'
+
+describe Harvestdor::Indexer::Resource do
+  
+  before(:all) do
+    VCR.use_cassette('before_all_call') do
+      @config_yml_path = File.join(File.dirname(__FILE__), "..", "config", "ap.yml")
+      require 'yaml'
+      @config = YAML.load_file(@config_yml_path)
+      @fake_druid = 'oo000oo0000'
+      
+      @indexer = Harvestdor::Indexer.new(@config)
+      @hdor_client = @indexer.send(:harvestdor_client)
+      @whitelist_path = File.join(File.dirname(__FILE__), "../config/ap_whitelist.txt")
+    end
+  end
+
+  let :resource do
+    Harvestdor::Indexer::Resource.new(@indexer, @fake_druid)
+  end
+  
+  context "smods_rec method" do
+    before(:all) do
+      @ns_decl = "xmlns='#{Mods::MODS_NS}'"
+      @mods_xml = "<mods #{@ns_decl}><note>hi</note></mods>"
+      @ng_mods_xml = Nokogiri::XML(@mods_xml)      
+    end
+    it "should call mods method on harvestdor_client" do
+      expect(@hdor_client).to receive(:mods).with(@fake_druid).and_return(@ng_mods_xml)
+      resource.smods_rec
+    end
+    it "should return Stanford::Mods::Record object" do
+      expect(@hdor_client).to receive(:mods).with(@fake_druid).and_return(@ng_mods_xml)
+      expect(resource.smods_rec).to be_an_instance_of(Stanford::Mods::Record)
+    end
+    it "should raise exception if MODS xml for the druid is empty" do
+      allow(@hdor_client).to receive(:mods).with(@fake_druid).and_return(Nokogiri::XML("<mods #{@ns_decl}/>"))
+      expect { resource.smods_rec }.to raise_error(RuntimeError, Regexp.new("^Empty MODS metadata for #{@fake_druid}: <"))
+    end
+    it "should raise exception if there is no MODS xml for the druid" do
+      VCR.use_cassette('exception_no_MODS_call') do
+        expect { resource.smods_rec }.to raise_error(Harvestdor::Errors::MissingMods)
+      end
+    end
+  end
+  
+  context "public_xml related methods" do
+    before(:all) do
+      @id_md_xml = "<identityMetadata><objectId>druid:#{@fake_druid}</objectId></identityMetadata>"
+      @cntnt_md_xml = "<contentMetadata type='image' objectId='#{@fake_druid}'>foo</contentMetadata>"
+      @rights_md_xml = "<rightsMetadata><access type=\"discover\"><machine><world>bar</world></machine></access></rightsMetadata>"
+      @rdf_xml = "<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#'><rdf:Description rdf:about=\"info:fedora/druid:#{@fake_druid}\">relationship!</rdf:Description></rdf:RDF>"
+      @pub_xml = "<publicObject id='druid:#{@fake_druid}'>#{@id_md_xml}#{@cntnt_md_xml}#{@rights_md_xml}#{@rdf_xml}</publicObject>"
+      @ng_pub_xml = Nokogiri::XML(@pub_xml)
+    end
+    context "#public_xml" do
+      it "should call public_xml method on harvestdor_client" do
+        expect(@hdor_client).to receive(:public_xml).with(@fake_druid).and_return(@ng_pub_xml)
+        resource.public_xml
+      end
+      it "retrieves entire public xml as a Nokogiri::XML::Document" do
+        expect(@hdor_client).to receive(:public_xml).with(@fake_druid).and_return(@ng_pub_xml)
+        px = resource.public_xml
+        expect(px).to be_kind_of(Nokogiri::XML::Document)
+        expect(px.root.name).to eq('publicObject')
+        expect(px.root.attributes['id'].text).to eq("druid:#{@fake_druid}")
+      end
+      it "raises exception if public xml for the druid is empty" do
+        expect(@hdor_client).to receive(:public_xml).with(@fake_druid).and_return(Nokogiri::XML("<publicObject/>"))
+        expect { resource.public_xml }.to raise_error(RuntimeError, Regexp.new("^Empty public xml for #{@fake_druid}: <"))
+      end
+      it "raises error if there is no public_xml page for the druid" do
+        expect(@hdor_client).to receive(:public_xml).with(@fake_druid).and_return(nil)
+        expect { resource.public_xml }.to raise_error(RuntimeError, "No public xml for #{@fake_druid}")
+      end
+    end
+    context "#content_metadata" do
+      it "returns a Nokogiri::XML::Document derived from the public xml if a druid is passed" do
+        allow(Harvestdor).to receive(:public_xml).with(@fake_druid, @indexer.config.harvestdor.purl).and_return(@ng_pub_xml)
+        cm = resource.content_metadata
+        expect(cm).to be_kind_of(Nokogiri::XML::Document)
+        expect(cm.root).not_to eq(nil)
+        expect(cm.root.name).to eq('contentMetadata')
+        expect(cm.root.attributes['objectId'].text).to eq(@fake_druid)
+        expect(cm.root.text.strip).to eq('foo')
+      end
+      it "raises RuntimeError if nil is returned by Harvestdor::Client.contentMetadata for the druid" do
+        expect(@hdor_client).to receive(:content_metadata).with(@fake_druid).and_return(nil)
+        expect { resource.content_metadata }.to raise_error(RuntimeError, "No contentMetadata for \"#{@fake_druid}\"")
+      end
+    end
+    context "#identity_metadata" do
+      it "returns a Nokogiri::XML::Document derived from the public xml if a druid is passed" do
+        allow(Harvestdor).to receive(:public_xml).with(@fake_druid, @indexer.config.harvestdor.purl).and_return(@ng_pub_xml)
+        im = resource.identity_metadata
+        expect(im).to be_kind_of(Nokogiri::XML::Document)
+        expect(im.root).not_to eq(nil)
+        expect(im.root.name).to eq('identityMetadata')
+        expect(im.root.text.strip).to eq("druid:#{@fake_druid}")
+      end
+      it "raises RuntimeError if nil is returned by Harvestdor::Client.identityMetadata for the druid" do
+        expect(@hdor_client).to receive(:identity_metadata).with(@fake_druid).and_return(nil)
+        expect { resource.identity_metadata }.to raise_error(RuntimeError, "No identityMetadata for \"#{@fake_druid}\"")
+      end
+    end
+    context "#rights_metadata" do
+      it "returns a Nokogiri::XML::Document derived from the public xml if a druid is passed" do
+        allow(Harvestdor).to receive(:public_xml).with(@fake_druid, @indexer.config.harvestdor.purl).and_return(@ng_pub_xml)
+        im = resource.rights_metadata
+        expect(im).to be_kind_of(Nokogiri::XML::Document)
+        expect(im.root).not_to eq(nil)
+        expect(im.root.name).to eq('rightsMetadata')
+        expect(im.root.text.strip).to eq("bar")
+      end
+      it "raises RuntimeError if nil is returned by Harvestdor::Client.rightsMetadata for the druid" do
+        expect(@hdor_client).to receive(:rights_metadata).with(@fake_druid).and_return(nil)
+        expect { resource.rights_metadata }.to raise_error(RuntimeError, "No rightsMetadata for \"#{@fake_druid}\"")
+      end
+    end
+    context "#rdf" do
+      it "returns a Nokogiri::XML::Document derived from the public xml if a druid is passed" do
+        allow(Harvestdor).to receive(:public_xml).with(@fake_druid, @indexer.config.harvestdor.purl).and_return(@ng_pub_xml)
+        im = resource.rdf
+        expect(im).to be_kind_of(Nokogiri::XML::Document)
+        expect(im.root).not_to eq(nil)
+        expect(im.root.name).to eq('RDF')
+        expect(im.root.text.strip).to eq("relationship!")
+      end
+      it "raises RuntimeError if nil is returned by Harvestdor::Client.rdf for the druid" do
+        expect(@hdor_client).to receive(:rdf).with(@fake_druid).and_return(nil)
+        expect { resource.rdf }.to raise_error(RuntimeError, "No RDF for \"#{@fake_druid}\"")
+      end
+    end
+    
+    describe "#public_xml_or_druid" do
+      it "should return the public_xml, if the public_xml has been loaded" do
+        allow(resource).to receive(:public_xml?).and_return(true)
+        allow(resource).to receive(:public_xml).and_return(double)
+        expect(resource.public_xml_or_druid).to eq resource.public_xml
+      end
+      it "should return the druid, if the public_xml has not been loaded" do
+        allow(resource).to receive(:public_xml?).and_return(false)
+        expect(resource.public_xml_or_druid).to eq @fake_druid
+      end
+    end
+
+    describe "#identity_md_obj_label" do
+      it "should extract the objectLabel from the identity metadata" do
+        allow(resource).to receive(:identity_metadata).and_return(Nokogiri::XML("<identityMetadata><objectLabel>label</objectLabel></identityMetadata>"))
+        expect(resource.identity_md_obj_label).to eq "label"
+      end
+    end
+
+    describe "#collections" do
+      it "should extract the collection this resource is a member of and return Resource objects for those collections" do
+        allow(resource).to receive(:public_xml).and_return(Nokogiri::XML <<-EOF
+<publicObject>
+  <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:fedora="info:fedora/fedora-system:def/relations-external#">
+    <rdf:Description>
+      <fedora:isMemberOfCollection rdf:resource="some:druid" />
+    </rdf:Description>
+  </rdf:RDF>
+</publicObject>
+EOF
+        )
+
+        expect(resource.collections.length).to eq 1
+        expect(resource.collections.first.druid).to eq "some:druid"
+        expect(resource.collections.first.indexer).to eq resource.indexer
+      end
+    end
+  end
+  
+end

data/spec/unit/harvestdor-indexer-solr_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Harvestdor::Indexer::Solr do
+  let :indexer do
+    double(logger: Logger.new("/dev/null"))
+  end
+
+  let :solr do
+    Harvestdor::Indexer::Solr.new indexer
+  end
+
+  # The method that sends the solr document to solr
+  describe "#add" do
+    let(:doc_hash) do
+      {
+        :id => "whatever",
+        :modsxml => 'whatever',
+        :title_display => 'title',
+        :pub_year_tisim => 'some year',
+        :author_person_display => 'author',
+        :format => 'Image',
+        :language => 'English'
+      }
+    end
+
+    it "sends an add request to the solr_client" do
+      expect(solr.client).to receive(:add).with(doc_hash)
+      solr.add(doc_hash)
+    end
+  end
+
+end

data/spec/unit/harvestdor-indexer_spec.rb

@@ -5,61 +5,37 @@ describe Harvestdor::Indexer do
   before(:all) do
     VCR.use_cassette('before_all_call') do
       @config_yml_path = File.join(File.dirname(__FILE__), "..", "config", "ap.yml")
-      @client_config_path = File.join(File.dirname(__FILE__), "../..", "config", "dor-fetcher-client.yml")
-      @indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
       require 'yaml'
-      @yaml = YAML.load_file(@config_yml_path)
+      @config = YAML.load_file(@config_yml_path)
+
+      @indexer = Harvestdor::Indexer.new(@config) do |config|
+        config.whitelist = ["druid:yg867hg1375"]
+      end
       @hdor_client = @indexer.send(:harvestdor_client)
-      @fake_druid = 'oo000oo0000'
-      @blacklist_path = File.join(File.dirname(__FILE__), "../config/ap_blacklist.txt")
+      @fake_druid = 'druid:oo000oo0000'
       @whitelist_path = File.join(File.dirname(__FILE__), "../config/ap_whitelist.txt")
     end
   end
-  
-  # The method that sends the solr document to solr
-  describe "#solr_add" do
-    before(:each) do
-      doc_hash = {
-        :modsxml => 'whatever',
-        :title_display => 'title',
-        :pub_year_tisim => 'some year',
-        :author_person_display => 'author',
-        :format => 'Image',
-        :language => 'English'
-      }
-    end
-    it "sends an add request to the solr_client" do
-      expect(@indexer.solr_client).to receive(:add)
-      @indexer.solr_add(@doc_hash, "abc123")
-    end
-  end
-  
+
   describe "access methods" do
     it "initializes success count" do
-      @indexer.success_count.should == 0
+      expect(@indexer.metrics.success_count).to eq(0)
     end
     it "initializes error count" do
-      @indexer.error_count.should == 0
-    end
-    it "initializes max_retries" do
-      expect(@indexer.max_retries).to eql(10)
-    end
-    it "allows overriding of max_retries" do
-      @indexer.max_retries=6
-      @indexer.max_retries.should == 6
+      expect(@indexer.metrics.error_count).to eq(0)
     end
   end
   
   describe "logging" do
     it "should write the log file to the directory indicated by log_dir" do
       @indexer.logger.info("indexer_spec logging test message")
-      File.exists?(File.join(@yaml['log_dir'], @yaml['log_name'])).should == true
+      expect(File.exists?(File.join(@config['harvestdor']['log_dir'], @config['harvestdor']['log_name']))).to eq(true)
     end
   end
   
   it "should initialize the harvestdor_client from the config" do
     expect(@hdor_client).to be_an_instance_of(Harvestdor::Client)
-    expect(@hdor_client.config.default_set).to eq(@yaml['default_set'])
+    expect(@hdor_client.config.default_set).to eq(@config['harvestdor']['default_set'])
   end
   
   context "harvest_and_index" do
@@ -69,63 +45,34 @@ describe Harvestdor::Indexer do
       }
     end
     it "should call dor_fetcher_client.druid_array and then call :add on rsolr connection" do
-      @indexer.should_receive(:druids).and_return([@fake_druid])
-      @indexer.solr_client.should_receive(:add).with(@doc_hash)
-      @indexer.solr_client.should_receive(:commit)
+      allow_any_instance_of(Harvestdor::Indexer::Resource).to receive(:collection?).and_return(false)
+      expect(@indexer).to receive(:druids).and_return([@fake_druid])
+      expect(@indexer.solr).to receive(:add).with(@doc_hash)
+      expect(@indexer.solr).to receive(:commit!)
       @indexer.harvest_and_index
     end
 
     it "should only call :commit on rsolr connection once" do
       VCR.use_cassette('single_rsolr_connection_call') do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
-        hdor_client = indexer.send(:harvestdor_client)
-        indexer.dor_fetcher_client.should_receive(:druid_array).and_return(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
-        indexer.solr_client.should_receive(:add).exactly(6).times
-        indexer.solr_client.should_receive(:commit).once
-        indexer.harvest_and_index
+        hdor_client = @indexer.send(:harvestdor_client)
+        expect(@indexer.dor_fetcher_client).to receive(:druid_array).and_return(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
+        expect(@indexer.solr).to receive(:add).exactly(6).times
+        expect(@indexer.solr).to receive(:commit!).once
+        @indexer.harvest_and_index
       end
     end
 
-    it "should not process druids in blacklist" do
-      VCR.use_cassette('ignore_druids_in_blacklist_call') do
-        lambda{
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
-          hdor_client = indexer.send(:harvestdor_client)
-          indexer.dor_fetcher_client.should_receive(:druid_array).and_return(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
-          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:nz353cp1092'}))
-          indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
-          indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:tc552kq0798'}))
-          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:th998nk0722'}))
-          indexer.solr_client.should_receive(:commit)
-          indexer.harvest_and_index
-        }
-      end
-    end
-    it "should not process druid if it is in both blacklist and whitelist" do
-      VCR.use_cassette('ignore_druids_in_blacklist_and_whitelist_call') do
-        lambda{
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path, :whitelist => @whitelist_path})
-          hdor_client = indexer.send(:harvestdor_client)
-          indexer.dor_fetcher_client.should_not_receive(:druid_array)
-          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:yg867hg1375'}))
-          indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
-          indexer.solr_client.should_receive(:commit)
-          indexer.harvest_and_index
-        }
-      end
-    end
     it "should only process druids in whitelist if it exists" do
       VCR.use_cassette('process_druids_whitelist_call') do
-        lambda{
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
-          hdor_client = indexer.send(:harvestdor_client)
-          indexer.dor_fetcher_client.should_not_receive(:druid_array)
-          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:yg867hg1375'}))
-          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
-          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:nz353cp1092'}))
-          indexer.solr_client.should_receive(:commit)
-          indexer.harvest_and_index
+        indexer = Harvestdor::Indexer.new(@config.merge(:whitelist => @whitelist_path))
+        hdor_client = indexer.send(:harvestdor_client)
+        added = []
+        allow(indexer.solr).to receive(:add) { |hash|
+          added << hash[:id]
         }
+        expect(indexer.solr).to receive(:commit!)
+        indexer.harvest_and_index
+        expect(added).to match_array ["druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534", "druid:yg867hg1375", 'druid:jf275fd6276', 'druid:nz353cp1092']
       end
     end
 
@@ -137,227 +84,36 @@ describe Harvestdor::Indexer do
         expect(@indexer.dor_fetcher_client).to be_an_instance_of(DorFetcher::Client)
       end 
 
-      it "should strip off is_member_of_collection_ and is_governed_by_ and return only the druid" do
-        expect(@indexer.strip_default_set_string()).to eq("yg867hg1375")
-      end
-
       it "druids method should call druid_array and get_collection methods on fetcher_client" do
         VCR.use_cassette('get_collection_druids_call') do
-          expect(@indexer.druids).to eq(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
+          expect(@indexer.resources.map(&:druid)).to match_array ["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"]
         end
       end
 
       it "should get the configuration of the dor-fetcher client from included yml file" do
-        expect(@indexer.dor_fetcher_client.service_url).to eq(@indexer.client_config["dor_fetcher_service_url"])
+        expect(@indexer.dor_fetcher_client.service_url).to eq("http://127.0.0.1:3000")
       end
 
   end # ending replacing OAI context
-
-  context "smods_rec method" do
-    before(:all) do
-      @fake_druid = 'oo000oo0000'
-      @ns_decl = "xmlns='#{Mods::MODS_NS}'"
-      @mods_xml = "<mods #{@ns_decl}><note>hi</note></mods>"
-      @ng_mods_xml = Nokogiri::XML(@mods_xml)      
-    end
-    it "should call mods method on harvestdor_client" do
-      @hdor_client.should_receive(:mods).with(@fake_druid).and_return(@ng_mods_xml)
-      @indexer.smods_rec(@fake_druid)
-    end
-    it "should return Stanford::Mods::Record object" do
-      @hdor_client.should_receive(:mods).with(@fake_druid).and_return(@ng_mods_xml)
-      @indexer.smods_rec(@fake_druid).should be_an_instance_of(Stanford::Mods::Record)
-    end
-    it "should raise exception if MODS xml for the druid is empty" do
-      @hdor_client.stub(:mods).with(@fake_druid).and_return(Nokogiri::XML("<mods #{@ns_decl}/>"))
-      expect { @indexer.smods_rec(@fake_druid) }.to raise_error(RuntimeError, Regexp.new("^Empty MODS metadata for #{@fake_druid}: <"))
-    end
-    it "should raise exception if there is no MODS xml for the druid" do
-      VCR.use_cassette('exception_no_MODS_call') do
-        expect { @indexer.smods_rec(@fake_druid) }.to raise_error(Harvestdor::Errors::MissingMods)
-      end
-    end
-  end
-  
-  context "public_xml related methods" do
-    before(:all) do
-      @id_md_xml = "<identityMetadata><objectId>druid:#{@fake_druid}</objectId></identityMetadata>"
-      @cntnt_md_xml = "<contentMetadata type='image' objectId='#{@fake_druid}'>foo</contentMetadata>"
-      @rights_md_xml = "<rightsMetadata><access type=\"discover\"><machine><world>bar</world></machine></access></rightsMetadata>"
-      @rdf_xml = "<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#'><rdf:Description rdf:about=\"info:fedora/druid:#{@fake_druid}\">relationship!</rdf:Description></rdf:RDF>"
-      @pub_xml = "<publicObject id='druid:#{@fake_druid}'>#{@id_md_xml}#{@cntnt_md_xml}#{@rights_md_xml}#{@rdf_xml}</publicObject>"
-      @ng_pub_xml = Nokogiri::XML(@pub_xml)
-    end
-    context "#public_xml" do
-      it "should call public_xml method on harvestdor_client" do
-        @hdor_client.should_receive(:public_xml).with(@fake_druid).and_return(@ng_pub_xml)
-        @indexer.public_xml @fake_druid
-      end
-      it "retrieves entire public xml as a Nokogiri::XML::Document" do
-        @hdor_client.should_receive(:public_xml).with(@fake_druid).and_return(@ng_pub_xml)
-        px = @indexer.public_xml @fake_druid
-        px.should be_kind_of(Nokogiri::XML::Document)
-        px.root.name.should == 'publicObject'
-        px.root.attributes['id'].text.should == "druid:#{@fake_druid}"
-      end
-      it "raises exception if public xml for the druid is empty" do
-        @hdor_client.should_receive(:public_xml).with(@fake_druid).and_return(Nokogiri::XML("<publicObject/>"))
-        expect { @indexer.public_xml(@fake_druid) }.to raise_error(RuntimeError, Regexp.new("^Empty public xml for #{@fake_druid}: <"))
-      end
-      it "raises error if there is no public_xml page for the druid" do
-        @hdor_client.should_receive(:public_xml).with(@fake_druid).and_return(nil)
-        expect { @indexer.public_xml(@fake_druid) }.to raise_error(RuntimeError, "No public xml for #{@fake_druid}")
-      end
-    end
-    context "#content_metadata" do
-      it "returns a Nokogiri::XML::Document derived from the public xml if a druid is passed" do
-        Harvestdor.stub(:public_xml).with(@fake_druid, @indexer.config.purl).and_return(@ng_pub_xml)
-        cm = @indexer.content_metadata(@fake_druid)
-        cm.should be_kind_of(Nokogiri::XML::Document)
-        cm.root.should_not == nil
-        cm.root.name.should == 'contentMetadata'
-        cm.root.attributes['objectId'].text.should == @fake_druid
-        cm.root.text.strip.should == 'foo'
-      end
-      it "if passed a Nokogiri::XML::Document of the public xml, it does no fetch" do
-        URI::HTTP.any_instance.should_not_receive(:open)
-        @hdor_client.should_receive(:content_metadata).and_call_original
-        cm = @indexer.content_metadata(@ng_pub_xml)
-        cm.should be_kind_of(Nokogiri::XML::Document)
-        cm.root.should_not == nil
-        cm.root.name.should == 'contentMetadata'
-        cm.root.attributes['objectId'].text.should == @fake_druid
-        cm.root.text.strip.should == 'foo'
-      end
-      it "raises RuntimeError if nil is returned by Harvestdor::Client.contentMetadata for the druid" do
-        @hdor_client.should_receive(:content_metadata).with(@fake_druid).and_return(nil)
-        expect { @indexer.content_metadata(@fake_druid) }.to raise_error(RuntimeError, "No contentMetadata for \"#{@fake_druid}\"")
-      end
-    end
-    context "#identity_metadata" do
-      it "returns a Nokogiri::XML::Document derived from the public xml if a druid is passed" do
-        Harvestdor.stub(:public_xml).with(@fake_druid, @indexer.config.purl).and_return(@ng_pub_xml)
-        im = @indexer.identity_metadata(@fake_druid)
-        im.should be_kind_of(Nokogiri::XML::Document)
-        im.root.should_not == nil
-        im.root.name.should == 'identityMetadata'
-        im.root.text.strip.should == "druid:#{@fake_druid}"
-      end
-      it "if passed a Nokogiri::XML::Document of the public xml, it does no fetch" do
-        URI::HTTP.any_instance.should_not_receive(:open)
-        @hdor_client.should_receive(:identity_metadata).and_call_original
-        im = @indexer.identity_metadata(@ng_pub_xml)
-        im.should be_kind_of(Nokogiri::XML::Document)
-        im.root.should_not == nil
-        im.root.name.should == 'identityMetadata'
-        im.root.text.strip.should == "druid:#{@fake_druid}"
-      end
-      it "raises RuntimeError if nil is returned by Harvestdor::Client.identityMetadata for the druid" do
-        @hdor_client.should_receive(:identity_metadata).with(@fake_druid).and_return(nil)
-        expect { @indexer.identity_metadata(@fake_druid) }.to raise_error(RuntimeError, "No identityMetadata for \"#{@fake_druid}\"")
-      end
-    end
-    context "#rights_metadata" do
-      it "returns a Nokogiri::XML::Document derived from the public xml if a druid is passed" do
-        Harvestdor.stub(:public_xml).with(@fake_druid, @indexer.config.purl).and_return(@ng_pub_xml)
-        im = @indexer.rights_metadata(@fake_druid)
-        im.should be_kind_of(Nokogiri::XML::Document)
-        im.root.should_not == nil
-        im.root.name.should == 'rightsMetadata'
-        im.root.text.strip.should == "bar"
-      end
-      it "raises RuntimeError if nil is returned by Harvestdor::Client.rightsMetadata for the druid" do
-        @hdor_client.should_receive(:rights_metadata).with(@fake_druid).and_return(nil)
-        expect { @indexer.rights_metadata(@fake_druid) }.to raise_error(RuntimeError, "No rightsMetadata for \"#{@fake_druid}\"")
-      end
-    end
-    context "#rdf" do
-      it "returns a Nokogiri::XML::Document derived from the public xml if a druid is passed" do
-        Harvestdor.stub(:public_xml).with(@fake_druid, @indexer.config.purl).and_return(@ng_pub_xml)
-        im = @indexer.rdf(@fake_druid)
-        im.should be_kind_of(Nokogiri::XML::Document)
-        im.root.should_not == nil
-        im.root.name.should == 'RDF'
-        im.root.text.strip.should == "relationship!"
-      end
-      it "raises RuntimeError if nil is returned by Harvestdor::Client.rdf for the druid" do
-        @hdor_client.should_receive(:rdf).with(@fake_druid).and_return(nil)
-        expect { @indexer.rdf(@fake_druid) }.to raise_error(RuntimeError, "No RDF for \"#{@fake_druid}\"")
-      end
-    end    
-  end
-  
-  context "blacklist" do
-    it "should be an Array with an entry for each non-empty line in the file" do
-      @indexer.send(:load_blacklist, @blacklist_path)
-      @indexer.send(:blacklist).should be_an_instance_of(Array)
-      @indexer.send(:blacklist).size.should == 2
-    end
-    it "should be empty Array if there was no blacklist config setting" do
-      VCR.use_cassette('empty_array_no_blacklist_config_call') do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
-        expect(indexer.blacklist).to eq([])
-      end
-    end
-    context "load_blacklist" do
-      it "knows what is in the blacklist" do
-        VCR.use_cassette('know_what_is_in_blacklist_call') do
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
-          expect(indexer.blacklist).to eq(["druid:jf275fd6276", "druid:tc552kq0798"])
-        end
-      end
-      it "should not be called if there was no blacklist config setting" do
-        VCR.use_cassette('no_blacklist_config_call') do
-          lambda{
-            indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
-
-            indexer.should_not_receive(:load_blacklist)
-
-            hdor_client = indexer.send(:harvestdor_client)
-            indexer.dor_fetcher_client.should_receive(:druid_array).and_return([@fake_druid])
-            indexer.solr_client.should_receive(:add)
-            indexer.solr_client.should_receive(:commit)
-            indexer.harvest_and_index
-          }
-        end
-      end
-      it "should only try to load a blacklist once" do
-        VCR.use_cassette('load_blacklist_once_call') do
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
-          indexer.send(:blacklist)
-          File.any_instance.should_not_receive(:open)
-          indexer.send(:blacklist)
-        end
-      end
-      it "should log an error message and throw RuntimeError if it can't find the indicated blacklist file" do
-        VCR.use_cassette('no_blacklist_found_call') do
-          exp_msg = 'Unable to find list of druids at bad_path'
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => 'bad_path'})
-          indexer.logger.should_receive(:fatal).with(exp_msg)
-          expect { indexer.send(:load_blacklist, 'bad_path') }.to raise_error(exp_msg)
-        end
-      end   
-    end
-  end # blacklist
   
   context "whitelist" do
     it "knows what is in the whitelist" do
       VCR.use_cassette('know_what_is_in_whitelist_call') do
         lambda{
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
+          indexer = Harvestdor::Indexer.new({:whitelist => @whitelist_path})
           expect(indexer.whitelist).to eq(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092"])
         }     
       end
     end
     it "should be an Array with an entry for each non-empty line in the file" do
       @indexer.send(:load_whitelist, @whitelist_path)
-      @indexer.send(:whitelist).should be_an_instance_of(Array)
-      @indexer.send(:whitelist).size.should == 3
+      expect(@indexer.send(:whitelist)).to be_an_instance_of(Array)
+      expect(@indexer.send(:whitelist).size).to eq(3)
     end
     it "should be empty Array if there was no whitelist config setting" do
       VCR.use_cassette('empty_array_no_whitelist_config_call') do
         lambda{
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
+          indexer = Harvestdor::Indexer.new()
           expect(indexer.whitelist).to eq([])
         }
       end
@@ -366,31 +122,31 @@ describe Harvestdor::Indexer do
       it "should not be called if there was no whitelist config setting" do
         VCR.use_cassette('no_whitelist_config_call') do
           lambda{
-            indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
+            indexer = Harvestdor::Indexer.new()
 
-            indexer.should_not_receive(:load_whitelist)
+            expect(indexer).not_to receive(:load_whitelist)
 
             hdor_client = indexer.send(:harvestdor_client)
-            indexer.dor_fetcher_client.should_receive(:druid_array).and_return([@fake_druid])
-            indexer.solr_client.should_receive(:add)
-            indexer.solr_client.should_receive(:commit)
+            expect(indexer.dor_fetcher_client).to receive(:druid_array).and_return([@fake_druid])
+            expect(indexer.solr_client).to receive(:add)
+            expect(indexer.solr_client).to receive(:commit)
             indexer.harvest_and_index
           }
         end
       end
       it "should only try to load a whitelist once" do
         VCR.use_cassette('load_whitelist_once_call') do
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
+          indexer = Harvestdor::Indexer.new({:whitelist => @whitelist_path})
           indexer.send(:whitelist)
-          File.any_instance.should_not_receive(:open)
+          expect_any_instance_of(File).not_to receive(:open)
           indexer.send(:whitelist)
         end
       end
       it "should log an error message and throw RuntimeError if it can't find the indicated whitelist file" do
         VCR.use_cassette('cant_find_whitelist_call') do
           exp_msg = 'Unable to find list of druids at bad_path'
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => 'bad_path'})
-          indexer.logger.should_receive(:fatal).with(exp_msg)
+          indexer = Harvestdor::Indexer.new(@config.merge(:whitelist => 'bad_path'))
+          expect(indexer.logger).to receive(:fatal).with(exp_msg)
           expect { indexer.send(:load_whitelist, 'bad_path') }.to raise_error(exp_msg)
         end
       end   
@@ -399,15 +155,15 @@ describe Harvestdor::Indexer do
   
   it "solr_client should initialize the rsolr client using the options from the config" do
     VCR.use_cassette('rsolr_client_config_call') do
-      indexer = Harvestdor::Indexer.new(nil, @client_config_path, Confstruct::Configuration.new(:solr => { :url => 'http://localhost:2345', :a => 1 }) )
-      RSolr.should_receive(:connect).with(hash_including(:a => 1, :url => 'http://localhost:2345')).and_return('foo')
-      indexer.solr_client
+      indexer = Harvestdor::Indexer.new(Confstruct::Configuration.new(:solr => { :url => 'http://localhost:2345', :a => 1 }) )
+      expect(RSolr).to receive(:connect).with(hash_including(:a => 1, :url => 'http://localhost:2345')).and_return('foo')
+      indexer.solr
     end
   end
 
   context "skip heartbeat" do
     it "allows me to use a fake url for dor-fetcher-client" do
-      expect {Harvestdor::Indexer.new(@config_yml_path, @client_config_path)}.not_to raise_error
+      expect {Harvestdor::Indexer.new()}.not_to raise_error
     end
   end
 end