harvestdor-indexer 1.0.0 → 1.0.3

data/spec/fixtures/vcr_cassettes/process_druids_whitelist_call.yml

@@ -0,0 +1,48 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*; q=0.5, application/xml'
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/xml; charset=utf-8
+      Etag:
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - 3f57f369-8d39-43dc-b15e-dc7a7f37d25e
+      X-Runtime:
+      - '0.003776'
+      Connection:
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 18:46:07 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/rsolr_client_config_call.yml

@@ -0,0 +1,48 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*; q=0.5, application/xml'
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/xml; charset=utf-8
+      Etag:
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - f3fe5cb0-9a23-4906-bf4b-2e7f236a53a9
+      X-Runtime:
+      - '0.003822'
+      Connection:
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 18:50:01 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/single_rsolr_connection_call.yml

@@ -2,7 +2,7 @@
 http_interactions:
 - request:
     method: get
-    uri: http://127.0.0.1:3000/collection/yg867hg1375
+    uri: http://127.0.0.1:3000/
     body:
       encoding: US-ASCII
       string: ''
@@ -14,7 +14,7 @@ http_interactions:
   response:
     status:
       code: 200
-      message: 'OK '
+      message: OK
     headers:
       X-Frame-Options:
       - SAMEORIGIN
@@ -23,36 +23,77 @@ http_interactions:
       X-Content-Type-Options:
       - nosniff
       Content-Type:
-      - application/json; charset=utf-8
+      - text/html; charset=utf-8
       Etag:
-      - '"682afec57f678e4d153a5841b21395dd"'
+      - '"444bcb3a3fcf8389296c49467f27e1d6"'
       Cache-Control:
       - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
       X-Request-Id:
-      - 37413f0c-a104-4df1-8a80-1873389200f4
+      - 4fef0b48-9ee9-4ef8-811e-78b8c0f643f0
       X-Runtime:
-      - '0.006754'
+      - '0.005477'
+      Connection:
+      - close
       Server:
-      - WEBrick/1.3.1 (Ruby/2.1.2/2014-05-08)
-      Date:
-      - Wed, 22 Oct 2014 18:42:32 GMT
-      Content-Length:
-      - '1121'
+      - thin 1.6.2 codename Doc Brown
+    body:
+      encoding: US-ASCII
+      string: ok
+    http_version:
+  recorded_at: Wed, 12 Nov 2014 19:34:02 GMT
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/collections/yg867hg1375
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/json; charset=utf-8
+      Etag:
+      - '"c3a375a17d64792857f21d1af227ed7b"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Meta-Request-Version:
+      - 0.3.4
+      X-Request-Id:
+      - 26936654-94cb-4135-b867-53f9ba72d1b5
+      X-Runtime:
+      - '0.011706'
       Connection:
-      - Keep-Alive
+      - close
+      Server:
+      - thin 1.6.2 codename Doc Brown
     body:
       encoding: US-ASCII
-      string: '{"collection":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":["Francis
-        E. Stafford photographs, 1909-1933"]}],"item":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":["Album
+      string: '{"collections":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":"Francis
+        E. Stafford photographs, 1909-1933"}],"items":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":"Album
         A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":["Album
-        E: Photographs of the Seventh Day Adventist Church missionaries in China"]},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        social customs, and people."},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":"Album
+        E: Photographs of the Seventh Day Adventist Church missionaries in China"},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":"Album
         D: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        social customs, and people."},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":"Album
         C: Photographs of the Chinese Revolution of 1911 and the Shanghai Commercial
-        Press"]},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        Press"},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":"Album
         B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
-        social customs, and people."]}],"counts":[{"collection":1},{"item":5},{"total_count":6}]}'
+        social customs, and people."}],"counts":{"collections":1,"items":5,"total_count":6}}'
     http_version:
-  recorded_at: Wed, 22 Oct 2014 18:42:32 GMT
+  recorded_at: Wed, 12 Nov 2014 19:34:02 GMT
 recorded_with: VCR 2.9.3

data/spec/unit/harvestdor-indexer_spec.rb

@@ -3,15 +3,17 @@ require 'spec_helper'
 describe Harvestdor::Indexer do
   
   before(:all) do
-    @config_yml_path = File.join(File.dirname(__FILE__), "..", "config", "ap.yml")
-    @client_config_path = File.join(File.dirname(__FILE__), "../..", "config", "dor-fetcher-client.yml")
-    @indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
-    require 'yaml'
-    @yaml = YAML.load_file(@config_yml_path)
-    @hdor_client = @indexer.send(:harvestdor_client)
-    @fake_druid = 'oo000oo0000'
-    @blacklist_path = File.join(File.dirname(__FILE__), "../config/ap_blacklist.txt")
-    @whitelist_path = File.join(File.dirname(__FILE__), "../config/ap_whitelist.txt")
+    VCR.use_cassette('before_all_call') do
+      @config_yml_path = File.join(File.dirname(__FILE__), "..", "config", "ap.yml")
+      @client_config_path = File.join(File.dirname(__FILE__), "../..", "config", "dor-fetcher-client.yml")
+      @indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
+      require 'yaml'
+      @yaml = YAML.load_file(@config_yml_path)
+      @hdor_client = @indexer.send(:harvestdor_client)
+      @fake_druid = 'oo000oo0000'
+      @blacklist_path = File.join(File.dirname(__FILE__), "../config/ap_blacklist.txt")
+      @whitelist_path = File.join(File.dirname(__FILE__), "../config/ap_whitelist.txt")
+    end
   end
   
   # The method that sends the solr document to solr
@@ -56,8 +58,8 @@ describe Harvestdor::Indexer do
   end
   
   it "should initialize the harvestdor_client from the config" do
-    @hdor_client.should be_an_instance_of(Harvestdor::Client)
-    @hdor_client.config.default_set.should == @yaml['default_set']
+    expect(@hdor_client).to be_an_instance_of(Harvestdor::Client)
+    expect(@hdor_client.config.default_set).to eq(@yaml['default_set'])
   end
   
   context "harvest_and_index" do
@@ -86,38 +88,44 @@ describe Harvestdor::Indexer do
 
     it "should not process druids in blacklist" do
       VCR.use_cassette('ignore_druids_in_blacklist_call') do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
-        hdor_client = indexer.send(:harvestdor_client)
-        indexer.dor_fetcher_client.should_receive(:druid_array).and_return(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
-        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:nz353cp1092'}))
-        indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
-        indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:tc552kq0798'}))
-        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:th998nk0722'}))
-        indexer.solr_client.should_receive(:commit)
-        indexer.harvest_and_index
+        lambda{
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
+          hdor_client = indexer.send(:harvestdor_client)
+          indexer.dor_fetcher_client.should_receive(:druid_array).and_return(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
+          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:nz353cp1092'}))
+          indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
+          indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:tc552kq0798'}))
+          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:th998nk0722'}))
+          indexer.solr_client.should_receive(:commit)
+          indexer.harvest_and_index
+        }
       end
     end
     it "should not process druid if it is in both blacklist and whitelist" do
       VCR.use_cassette('ignore_druids_in_blacklist_and_whitelist_call') do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path, :whitelist => @whitelist_path})
-        hdor_client = indexer.send(:harvestdor_client)
-        indexer.dor_fetcher_client.should_not_receive(:druid_array)
-        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:yg867hg1375'}))
-        indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
-        indexer.solr_client.should_receive(:commit)
-        indexer.harvest_and_index
+        lambda{
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path, :whitelist => @whitelist_path})
+          hdor_client = indexer.send(:harvestdor_client)
+          indexer.dor_fetcher_client.should_not_receive(:druid_array)
+          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:yg867hg1375'}))
+          indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
+          indexer.solr_client.should_receive(:commit)
+          indexer.harvest_and_index
+        }
       end
     end
     it "should only process druids in whitelist if it exists" do
       VCR.use_cassette('process_druids_whitelist_call') do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
-        hdor_client = indexer.send(:harvestdor_client)
-        indexer.dor_fetcher_client.should_not_receive(:druid_array)
-        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:yg867hg1375'}))
-        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
-        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:nz353cp1092'}))
-        indexer.solr_client.should_receive(:commit)
-        indexer.harvest_and_index
+        lambda{
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
+          hdor_client = indexer.send(:harvestdor_client)
+          indexer.dor_fetcher_client.should_not_receive(:druid_array)
+          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:yg867hg1375'}))
+          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
+          indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:nz353cp1092'}))
+          indexer.solr_client.should_receive(:commit)
+          indexer.harvest_and_index
+        }
       end
     end
 
@@ -286,81 +294,115 @@ describe Harvestdor::Indexer do
       @indexer.send(:blacklist).size.should == 2
     end
     it "should be empty Array if there was no blacklist config setting" do
-      indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
-      indexer.send(:blacklist).should == []
+      VCR.use_cassette('empty_array_no_blacklist_config_call') do
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
+        expect(indexer.blacklist).to eq([])
+      end
     end
     context "load_blacklist" do
+      it "knows what is in the blacklist" do
+        VCR.use_cassette('know_what_is_in_blacklist_call') do
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
+          expect(indexer.blacklist).to eq(["druid:jf275fd6276", "druid:tc552kq0798"])
+        end
+      end
       it "should not be called if there was no blacklist config setting" do
         VCR.use_cassette('no_blacklist_config_call') do
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
+          lambda{
+            indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
 
-          indexer.should_not_receive(:load_blacklist)
+            indexer.should_not_receive(:load_blacklist)
 
-          hdor_client = indexer.send(:harvestdor_client)
-          indexer.dor_fetcher_client.should_receive(:druid_array).and_return([@fake_druid])
-          indexer.solr_client.should_receive(:add)
-          indexer.solr_client.should_receive(:commit)
-          indexer.harvest_and_index
+            hdor_client = indexer.send(:harvestdor_client)
+            indexer.dor_fetcher_client.should_receive(:druid_array).and_return([@fake_druid])
+            indexer.solr_client.should_receive(:add)
+            indexer.solr_client.should_receive(:commit)
+            indexer.harvest_and_index
+          }
         end
       end
       it "should only try to load a blacklist once" do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
-        indexer.send(:blacklist)
-        File.any_instance.should_not_receive(:open)
-        indexer.send(:blacklist)
+        VCR.use_cassette('load_blacklist_once_call') do
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
+          indexer.send(:blacklist)
+          File.any_instance.should_not_receive(:open)
+          indexer.send(:blacklist)
+        end
       end
       it "should log an error message and throw RuntimeError if it can't find the indicated blacklist file" do
-        exp_msg = 'Unable to find list of druids at bad_path'
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => 'bad_path'})
-        indexer.logger.should_receive(:fatal).with(exp_msg)
-        expect { indexer.send(:load_blacklist, 'bad_path') }.to raise_error(exp_msg)
+        VCR.use_cassette('no_blacklist_found_call') do
+          exp_msg = 'Unable to find list of druids at bad_path'
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => 'bad_path'})
+          indexer.logger.should_receive(:fatal).with(exp_msg)
+          expect { indexer.send(:load_blacklist, 'bad_path') }.to raise_error(exp_msg)
+        end
       end   
     end
   end # blacklist
   
   context "whitelist" do
+    it "knows what is in the whitelist" do
+      VCR.use_cassette('know_what_is_in_whitelist_call') do
+        lambda{
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
+          expect(indexer.whitelist).to eq(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092"])
+        }     
+      end
+    end
     it "should be an Array with an entry for each non-empty line in the file" do
       @indexer.send(:load_whitelist, @whitelist_path)
       @indexer.send(:whitelist).should be_an_instance_of(Array)
       @indexer.send(:whitelist).size.should == 3
     end
     it "should be empty Array if there was no whitelist config setting" do
-      indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
-      indexer.send(:whitelist).should == []
+      VCR.use_cassette('empty_array_no_whitelist_config_call') do
+        lambda{
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
+          expect(indexer.whitelist).to eq([])
+        }
+      end
     end
     context "load_whitelist" do
       it "should not be called if there was no whitelist config setting" do
         VCR.use_cassette('no_whitelist_config_call') do
-          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
+          lambda{
+            indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
 
-          indexer.should_not_receive(:load_whitelist)
+            indexer.should_not_receive(:load_whitelist)
 
-          hdor_client = indexer.send(:harvestdor_client)
-          indexer.dor_fetcher_client.should_receive(:druid_array).and_return([@fake_druid])
-          indexer.solr_client.should_receive(:add)
-          indexer.solr_client.should_receive(:commit)
-          indexer.harvest_and_index
+            hdor_client = indexer.send(:harvestdor_client)
+            indexer.dor_fetcher_client.should_receive(:druid_array).and_return([@fake_druid])
+            indexer.solr_client.should_receive(:add)
+            indexer.solr_client.should_receive(:commit)
+            indexer.harvest_and_index
+          }
         end
       end
       it "should only try to load a whitelist once" do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
-        indexer.send(:whitelist)
-        File.any_instance.should_not_receive(:open)
-        indexer.send(:whitelist)
+        VCR.use_cassette('load_whitelist_once_call') do
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
+          indexer.send(:whitelist)
+          File.any_instance.should_not_receive(:open)
+          indexer.send(:whitelist)
+        end
       end
       it "should log an error message and throw RuntimeError if it can't find the indicated whitelist file" do
-        exp_msg = 'Unable to find list of druids at bad_path'
-        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => 'bad_path'})
-        indexer.logger.should_receive(:fatal).with(exp_msg)
-        expect { indexer.send(:load_whitelist, 'bad_path') }.to raise_error(exp_msg)
+        VCR.use_cassette('cant_find_whitelist_call') do
+          exp_msg = 'Unable to find list of druids at bad_path'
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => 'bad_path'})
+          indexer.logger.should_receive(:fatal).with(exp_msg)
+          expect { indexer.send(:load_whitelist, 'bad_path') }.to raise_error(exp_msg)
+        end
       end   
     end
   end # whitelist
   
   it "solr_client should initialize the rsolr client using the options from the config" do
-    indexer = Harvestdor::Indexer.new(nil, @client_config_path, Confstruct::Configuration.new(:solr => { :url => 'http://localhost:2345', :a => 1 }) )
-    RSolr.should_receive(:connect).with(hash_including(:a => 1, :url => 'http://localhost:2345')).and_return('foo')
-    indexer.solr_client
+    VCR.use_cassette('rsolr_client_config_call') do
+      indexer = Harvestdor::Indexer.new(nil, @client_config_path, Confstruct::Configuration.new(:solr => { :url => 'http://localhost:2345', :a => 1 }) )
+      RSolr.should_receive(:connect).with(hash_including(:a => 1, :url => 'http://localhost:2345')).and_return('foo')
+      indexer.solr_client
+    end
   end
     
 end