harvestdor-indexer 0.0.13 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a14d39d89afa8e167b50facf06cf7cf0a6c5fa76
-  data.tar.gz: 63478e959742b0640189aaf90b2f185315b4b9c3
+  metadata.gz: 45d033dffd56d8c3abd90ccafd0fa4666b4379da
+  data.tar.gz: f579c955fb390f2d7ed497aa237dbe763361d60b
 SHA512:
-  metadata.gz: eb6f6830232b5d7609a29779c8ca58139cbad499d5c51daa9bfeb0252fe52fc633296a360ccd5e2e9c7b2b68313402c7fb9b6e4532a414a2d130c3f0fe867a69
-  data.tar.gz: f97897beffaebca548732810ed0d3281b169feead5203aaa9ec1a63167cdbf4813a5619052b9ab1cdf8f2fc54bb5068d827487ec9e668b27a1f8b85acd375552
+  metadata.gz: a7e9213e27145df273eaee8384b690e9ae720aeffd7f781f64dec9bc13948c6716b942499305037c7210c59ae9b5f07747f5ff67e4fe315577a78bce7d530a81
+  data.tar.gz: 896d1c26f157087e2bd6f935e0106024e9a7840cdb5c5db05e6a348ff1fe0579410bc70895b26b3db60a6461883bfd5bfe4a8ee43483d0f8821bf73cad4c2b1c

data/README.rdoc

@@ -1,9 +1,8 @@
 = Harvestdor::Indexer
 {<img src="https://travis-ci.org/sul-dlss/harvestdor-indexer.svg" alt="Build Status" />}[https://travis-ci.org/sul-dlss/harvestdor-indexer]
-{<img src="https://coveralls.io/repos/sul-dlss/harvestdor-indexer/badge.png" alt="Coverage Status" />}[https://coveralls.io/r/sul-dlss/harvestdor-indexer]{<img src="https://gemnasium.com/sul-dlss/harvestdor-indexer.svg" alt="Dependency Status" />}[https://gemnasium.com/sul-dlss/harvestdor-indexer]{<img src="https://badge.fury.io/rb/harvestdor-indexer.svg" alt="Gem Version" />}[http://badge.fury.io/rb/harvestdor-indexer]
-
-
-
+{<img src="https://coveralls.io/repos/sul-dlss/harvestdor-indexer/badge.png" alt="Coverage Status" />}[https://coveralls.io/r/sul-dlss/harvestdor-indexer]
+{<img src="https://gemnasium.com/sul-dlss/harvestdor-indexer.svg" alt="Dependency Status" />}[https://gemnasium.com/sul-dlss/harvestdor-indexer]
+{<img src="https://badge.fury.io/rb/harvestdor-indexer.svg" alt="Gem Version" />}[http://badge.fury.io/rb/harvestdor-indexer]
 
 A Gem to harvest meta/data from DOR and the skeleton code to index it and write to Solr.
 
@@ -42,13 +41,10 @@ Note: Because of an update to underlying HTTP libraries, versions of this gem >
 See  spec/config/ap.yml for an example.
 You will want to copy that file and change the following settings:
 1. log_name
-2. default_set (in OAI harvesting params section)
-3. other OAI harvesting params
-4. blacklist or whitelist if you are using them
+2. default_set
+3. blacklist or whitelist if you are using them
 
-You can also pass in non-default configurations as a hash
-
-  indexer = Harvestdor::Indexer.new({:oai_repository_url => 'http://my_oai.org, :default_from_date => '2012-12-01'})
+Update the dor-fetcher-client.yml file in the config directory with the location of the URL of the dor-fetcher-service provider.  The defaulted value is the 3000 port for a localhost - dor_fetcher_service_url: http://127.0.0.1:3000
 
 === Override the Harvestdor::Indexer.index method
 
@@ -94,17 +90,21 @@ I suggest you write a script to run the code.  Your script might look like this:
     end
   config_yml_path = ARGV.pop
   if config_yml_path.nil?
-    puts "** You must provide the full path to a config yml file **"
+    puts "** You must provide the full path to a collection config yml file **"
+    exit
+  end
+  if client_config_path.nil?
+    puts "** You must provide the full path to dor-fetcher-client config yml file **"
     exit
   end
-  indexer = Harvestdor::Indexer.new(config_yml_path, opts)
+  indexer = Harvestdor::Indexer.new(config_yml_path, client_config_path, opts)
   indexer.harvest_and_index
 
 Then you run the script like so:
 
 	 ./bin/indexer config/(your coll).yml
 
-I suggest you run your code on harvestdor-dev, as it is already set up to be able to harvest from the DOR OAI provider
+I suggest you run your code on harvestdor-dev, as it is already set up to be able to harvest from the DorFetcher
 
 
 == Contributing
@@ -118,6 +118,7 @@ I suggest you run your code on harvestdor-dev, as it is already set up to be abl
 
 == Releases
 
+* <b>1.0.0</b> Replaced OAI harvesting mechanism with dor-fetcher
 * <b>0.0.13</b> Upgrade to latest faraday HTTP client syntax; Use retries gem (https://github.com/ooyala/retries) to make retrying of index process more robust
 * <b>0.0.12</b> fix total_object nil error
 * <b>0.0.11</b> fix error_count and success_count, allow setting of max-tries (retry solr add if error)

data/config/dor-fetcher-client.yml

@@ -0,0 +1,4 @@
+# ---------- DorFetcher Client parameters -----------
+
+# dor_fetcher_service_url:  URL of the dor-fetcher-service provider
+dor_fetcher_service_url: http://127.0.0.1:3000

data/harvestdor-indexer.gemspec

@@ -6,8 +6,8 @@ require 'harvestdor-indexer/version'
 Gem::Specification.new do |gem|
   gem.name          = "harvestdor-indexer"
   gem.version       = Harvestdor::Indexer::VERSION
-  gem.authors       = ["Naomi Dushay"]
-  gem.email         = ["ndushay@stanford.edu"]
+  gem.authors       = ["Naomi Dushay", "Bess Sadler", "Laney McGlohon"]
+  gem.email         = ["ndushay@stanford.edu", "bess@stanford.edu", "laneymcg@stanford.edu"]
   gem.description   = %q{Harvest DOR object metadata via a relationship (e.g. hydra:isGovernedBy rdf:resource="info:fedora/druid:hy787xj5878") and dates, plus code framework to write Solr docs to index}
   gem.summary       = %q{Harvest DOR object metadata and index it to Solr}
   gem.homepage      = "https://consul.stanford.edu/display/chimera/Chimera+project"
@@ -21,6 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'retries'
   gem.add_dependency 'harvestdor', '>=0.0.14'
   gem.add_dependency 'stanford-mods'
+  gem.add_dependency 'dor-fetcher', '>=1.0.0'
   
   # Runtime dependencies
   gem.add_runtime_dependency 'confstruct'
@@ -36,5 +37,7 @@ Gem::Specification.new do |gem|
 	gem.add_development_dependency 'rspec'
 	gem.add_development_dependency 'coveralls'
 	# gem.add_development_dependency 'ruby-debug19'
+  gem.add_development_dependency 'vcr'
+  gem.add_development_dependency 'webmock'
   
 end

data/lib/harvestdor-indexer/version.rb

@@ -1,6 +1,6 @@
 module Harvestdor
   class Indexer
     # this is the Ruby Gem version
-    VERSION = "0.0.13"
+    VERSION = "1.0.0"
   end
 end

data/lib/harvestdor-indexer.rb

@@ -2,10 +2,12 @@
 require 'confstruct'
 require 'rsolr'
 require 'retries'
+require 'json'
 
 # sul-dlss gems
 require 'harvestdor'
 require 'stanford-mods'
+require 'dor-fetcher'
 
 # stdlib
 require 'logger'
@@ -18,8 +20,9 @@ module Harvestdor
 
     attr_accessor :error_count, :success_count, :max_retries
     attr_accessor :total_time_to_parse,:total_time_to_solr
+    attr_accessor :dor_fetcher_client, :client_config
 
-    def initialize yml_path, options = {}
+    def initialize yml_path, client_config_path, options = {}
       @success_count=0    # the number of objects successfully indexed
       @error_count=0      # the number of objects that failed
       @max_retries=10      # the number of times to retry an object 
@@ -27,8 +30,10 @@ module Harvestdor
       @total_time_to_parse=0
       @yml_path = yml_path
       config.configure(YAML.load_file(yml_path)) if yml_path    
-      config.configure options 
+      config.configure options
       yield(config) if block_given?
+      @client_config = YAML.load_file(client_config_path) if client_config_path && File.exists?(client_config_path)
+      @dor_fetcher_client=DorFetcher::Client.new({:service_url => client_config["dor_fetcher_service_url"]})
     end
 
     def config
@@ -40,7 +45,7 @@ module Harvestdor
     end
 
     # per this Indexer's config options 
-    #  harvest the druids via OAI
+    #  harvest the druids via DorFetcher
     #   create a Solr profiling document for each druid
     #   write the result to the Solr index
     def harvest_and_index
@@ -67,14 +72,14 @@ module Harvestdor
       logger.info("Total records processed: #{total_objects}")
     end
 
-    # return Array of druids contained in the OAI harvest indicated by OAI params in yml configuration file
+    # return Array of druids contained in the DorFetcher pulling indicated by DorFetcher params
     # @return [Array<String>] or enumeration over it, if block is given.  (strings are druids, e.g. ab123cd1234)
     def druids
       if @druids.nil?
         start_time=Time.now
-        logger.info("Starting OAI harvest of druids at #{start_time}.")  
-        @druids = harvestdor_client.druids_via_oai
-        logger.info("Completed OAI harves of druids at #{Time.now}.  Found #{@druids.size} druids.  Total elapsed time for OAI harvest = #{elapsed_time(start_time,:minutes)} minutes")  
+        logger.info("Starting DorFetcher pulling of druids at #{start_time}.")  
+        @druids = @dor_fetcher_client.druid_array(@dor_fetcher_client.get_collection(strip_default_set_string(), {}))
+        logger.info("Completed DorFetcher pulling of druids at #{Time.now}.  Found #{@druids.size} druids.  Total elapsed time for DorFetcher pulling = #{elapsed_time(start_time,:minutes)} minutes")  
       end
       return @druids
     end
@@ -224,6 +229,12 @@ module Harvestdor
       @whitelist ||= []
     end
 
+    # Get only the druid from the end of the default_set string
+    # from the yml file
+    def strip_default_set_string()
+      @config.default_set.split('_').last
+    end
+
     protected #---------------------------------------------------------------------
 
     def harvestdor_client

data/spec/config/ap.yml

@@ -1,7 +1,6 @@
 # You will want to copy this file and change the following settings:
 # 1. log_dir, log_name
-# 2. default_set (in OAI harvesting params section)
-#  2a. other OAI harvesting params
+# 2. default_set
 # 3. blacklist or whitelist if you are using them
 # 4. Solr baseurl
 
@@ -16,8 +15,9 @@ purl: http://purl.stanford.edu
 
 # ---------- White and Black list parameters -----
 
-# name of file containing druids that will NOT be processed even if they are harvested via OAI
-#  either give absolute path or path relative to where the command will be executed
+# name of file containing druids that will NOT be processed even if they are harvested 
+#  via DorFetcher either give absolute path or path relative to where the command will 
+#  be executed
 #blacklist: config/ap_blacklist.txt
 
 # name of file containing druids that WILL be processed (all others will be ignored)
@@ -32,14 +32,9 @@ solr:
   read_timeout: 60
   open_timeout: 60
 
-# ---------- OAI harvesting parameters -----------
-
-# oai_repository_url:  URL of the OAI data provider
-oai_repository_url: https://dor-oaiprovider-prod.stanford.edu/oai
-
 # default_set:  default set for harvest  (default: nil)
 #   can be overridden on calls to harvest_ids and harvest_records
-default_set: is_governed_by_hy787xj5878
+default_set: is_governed_by_yg867hg1375
 
 # default_metadata_prefix:  default metadata prefix to be used for harvesting  (default: mods)
 #   can be overridden on calls to harvest_ids and harvest_records
@@ -50,8 +45,6 @@ default_set: is_governed_by_hy787xj5878
 # default_until_date:  default until date for harvest  (default: nil)
 #   can be overridden on calls to harvest_ids and harvest_records
 
-# oai_client_debug:  true for OAI::Client debug mode  (default: false)
-
 # Additional options to pass to Faraday http client (https://github.com/technoweenie/faraday)
 http_options:
   ssl:

data/spec/config/ap_blacklist.txt

@@ -1,5 +1,5 @@
 # blacklist containing druids that should NOT be processed.
 # druids should be in the form aa111bb2222
 
-oo111oo1111
-oo222oo2222
+druid:jf275fd6276
+druid:tc552kq0798

data/spec/config/ap_whitelist.txt

@@ -1,5 +1,6 @@
 # whitelist containing the specific druids to be processed (all others will be ignored)
 # druids should be in the form aa111bb2222
 
-oo000oo0000
-oo222oo2222
+druid:yg867hg1375
+druid:jf275fd6276
+druid:nz353cp1092

data/spec/fixtures/vcr_cassettes/exception_no_MODS_call.yml

@@ -0,0 +1,114 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://purl.stanford.edu/oo000oo0000.mods
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 404
+      message: ''
+    headers:
+      Date:
+      - Wed, 22 Oct 2014 20:26:30 GMT
+      Server:
+      - Apache/2.2.15 (Red Hat)
+      X-Powered-By:
+      - Phusion Passenger (mod_rails/mod_rack) 3.0.19
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
+      Cache-Control:
+      - no-cache
+      X-Request-Id:
+      - eb7854ee5cc96cbf20bfdafb0e8ea1c2
+      X-Runtime:
+      - '0.011781'
+      X-Rack-Cache:
+      - miss
+      Status:
+      - '404'
+      Content-Length:
+      - '3015'
+      Content-Type:
+      - text/html; charset=utf-8
+    body:
+      encoding: US-ASCII
+      string: |
+        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns="http://www.w3.org/1999/xhtml">
+          <head>
+            <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+            <title>Stanford Digital Repository</title>
+            <link rel="shortcut icon" href="favicon.ico" />
+            <link href="/assets/site.css" media="screen" rel="stylesheet" type="text/css" />
+          </head>
+          <body>
+            <div id="container">
+              <div id="banner"><h1>Stanford Digital Repository</h1></div>
+              <div id="contents">
+                <div class="dialog">
+          <h2>The item you requested is not available.</h2>
+          <p>The item you requested is not yet available. It will be available at this URL when Library processing is completed.</p>
+        </div>
+
+              </div>
+            </div>
+
+            <div id="footer">
+          <div class="footer-contents">
+            <div class="footer-sul">
+              <div class="footer-logo">
+                <a href="http://library.stanford.edu" target="_blank"><img src="/images/footer-sul-logo.png"></a>
+              </div>
+              <div class="footer-links">
+                <a href="http://library.stanford.edu" target="_blank">Stanford University Libraries</a>
+                <a href="http://searchworks.stanford.edu" target="_blank">SearchWorks</a>
+                <a href="http://library.stanford.edu/ejournals" target="_blank">eJournals</a>
+                <a href="hhttp://library.stanford.edu/myawrap.html" target="_blank">My Account</a>
+                <a href="http://library.stanford.edu/ask" target="_blank">Ask Us</a>
+              </div>
+            </div>
+            <div class="footer-su">
+              <div class="footer-logo">
+                <a href="http://www.stanford.edu" target="_blank"><img src="/images/footer-stanford-logo.png"></a>
+              </div>
+              <div class="footer-links">
+                <a href="http://www.stanford.edu" target="_blank">SU Home</a>
+                <a href="http://visit.stanford.edu/plan/maps.html" target="_blank">Maps &amp; Directions</a>
+                <a href="http://www.stanford.edu/search/" target="_blank">Search Stanford</a>
+                <a href="http://www.stanford.edu/site/terms.html" target="_blank">Terms of Use</a>
+                <a href="http://www.stanford.edu/site/copyright.html" target="_blank">Copyright Complaints</a>
+                </br>&copy; Stanford University, Stanford, California 94305
+              </div>
+            </div>
+          </div>
+        </div>
+            <script src="/assets/jquery.js" type="text/javascript"></script>
+        <script src="/assets/jquery.truncator.js" type="text/javascript"></script>
+        <script src="/assets/application.js" type="text/javascript"></script>
+
+            <script type="text/javascript">
+
+          var _gaq = _gaq || [];
+          _gaq.push(['_setAccount', 'UA-7219229-11']);
+          _gaq.push(['_trackPageview']);
+
+          (function() {
+            var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+            ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+            var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+          })();
+
+        </script>
+          </body>
+        </html>
+    http_version:
+  recorded_at: Wed, 22 Oct 2014 20:26:30 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/get_collection_druids_call.yml

@@ -0,0 +1,58 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/collection/yg867hg1375
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: 'OK '
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/json; charset=utf-8
+      Etag:
+      - '"682afec57f678e4d153a5841b21395dd"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Request-Id:
+      - 0954c447-9cb9-4eeb-8020-d87f13098f07
+      X-Runtime:
+      - '0.006736'
+      Server:
+      - WEBrick/1.3.1 (Ruby/2.1.2/2014-05-08)
+      Date:
+      - Wed, 22 Oct 2014 18:42:32 GMT
+      Content-Length:
+      - '1121'
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: '{"collection":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":["Francis
+        E. Stafford photographs, 1909-1933"]}],"item":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        E: Photographs of the Seventh Day Adventist Church missionaries in China"]},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        D: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        C: Photographs of the Chinese Revolution of 1911 and the Shanghai Commercial
+        Press"]},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]}],"counts":[{"collection":1},{"item":5},{"total_count":6}]}'
+    http_version:
+  recorded_at: Wed, 22 Oct 2014 18:42:32 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/ignore_druids_in_blacklist_call.yml

@@ -0,0 +1,58 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/collection/yg867hg1375
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: 'OK '
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/json; charset=utf-8
+      Etag:
+      - '"682afec57f678e4d153a5841b21395dd"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Request-Id:
+      - 1e0232c6-fc39-49bf-b874-89567e225d00
+      X-Runtime:
+      - '0.006851'
+      Server:
+      - WEBrick/1.3.1 (Ruby/2.1.2/2014-05-08)
+      Date:
+      - Wed, 22 Oct 2014 18:53:15 GMT
+      Content-Length:
+      - '1121'
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: '{"collection":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":["Francis
+        E. Stafford photographs, 1909-1933"]}],"item":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        E: Photographs of the Seventh Day Adventist Church missionaries in China"]},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        D: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        C: Photographs of the Chinese Revolution of 1911 and the Shanghai Commercial
+        Press"]},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]}],"counts":[{"collection":1},{"item":5},{"total_count":6}]}'
+    http_version:
+  recorded_at: Wed, 22 Oct 2014 18:53:15 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/no_blacklist_config_call.yml

@@ -0,0 +1,58 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/collection/yg867hg1375
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: 'OK '
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/json; charset=utf-8
+      Etag:
+      - '"682afec57f678e4d153a5841b21395dd"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Request-Id:
+      - d35b0793-e841-496b-bce1-720bfbf2ad04
+      X-Runtime:
+      - '0.006751'
+      Server:
+      - WEBrick/1.3.1 (Ruby/2.1.2/2014-05-08)
+      Date:
+      - Wed, 22 Oct 2014 20:32:36 GMT
+      Content-Length:
+      - '1121'
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: '{"collection":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":["Francis
+        E. Stafford photographs, 1909-1933"]}],"item":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        E: Photographs of the Seventh Day Adventist Church missionaries in China"]},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        D: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        C: Photographs of the Chinese Revolution of 1911 and the Shanghai Commercial
+        Press"]},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]}],"counts":[{"collection":1},{"item":5},{"total_count":6}]}'
+    http_version:
+  recorded_at: Wed, 22 Oct 2014 20:32:36 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/no_whitelist_config_call.yml

@@ -0,0 +1,58 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/collection/yg867hg1375
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: 'OK '
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/json; charset=utf-8
+      Etag:
+      - '"682afec57f678e4d153a5841b21395dd"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Request-Id:
+      - a631e18e-8396-4699-b7a9-fd05fd115e02
+      X-Runtime:
+      - '0.006491'
+      Server:
+      - WEBrick/1.3.1 (Ruby/2.1.2/2014-05-08)
+      Date:
+      - Wed, 22 Oct 2014 20:34:01 GMT
+      Content-Length:
+      - '1121'
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: '{"collection":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":["Francis
+        E. Stafford photographs, 1909-1933"]}],"item":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        E: Photographs of the Seventh Day Adventist Church missionaries in China"]},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        D: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        C: Photographs of the Chinese Revolution of 1911 and the Shanghai Commercial
+        Press"]},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]}],"counts":[{"collection":1},{"item":5},{"total_count":6}]}'
+    http_version:
+  recorded_at: Wed, 22 Oct 2014 20:34:01 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/single_rsolr_connection_call.yml

@@ -0,0 +1,58 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://127.0.0.1:3000/collection/yg867hg1375
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: 'OK '
+    headers:
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Content-Type-Options:
+      - nosniff
+      Content-Type:
+      - application/json; charset=utf-8
+      Etag:
+      - '"682afec57f678e4d153a5841b21395dd"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Request-Id:
+      - 37413f0c-a104-4df1-8a80-1873389200f4
+      X-Runtime:
+      - '0.006754'
+      Server:
+      - WEBrick/1.3.1 (Ruby/2.1.2/2014-05-08)
+      Date:
+      - Wed, 22 Oct 2014 18:42:32 GMT
+      Content-Length:
+      - '1121'
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: '{"collection":[{"druid":"druid:yg867hg1375","latest_change":"2013-11-11T23:34:29Z","title":["Francis
+        E. Stafford photographs, 1909-1933"]}],"item":[{"druid":"druid:jf275fd6276","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        A: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:nz353cp1092","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        E: Photographs of the Seventh Day Adventist Church missionaries in China"]},{"druid":"druid:tc552kq0798","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        D: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]},{"druid":"druid:th998nk0722","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        C: Photographs of the Chinese Revolution of 1911 and the Shanghai Commercial
+        Press"]},{"druid":"druid:ww689vs6534","latest_change":"2013-11-11T23:34:29Z","title":["Album
+        B: Photographs of China''s natural landscapes, urban scenes, cultural landmarks,
+        social customs, and people."]}],"counts":[{"collection":1},{"item":5},{"total_count":6}]}'
+    http_version:
+  recorded_at: Wed, 22 Oct 2014 18:42:32 GMT
+recorded_with: VCR 2.9.3

data/spec/spec_helper.rb

@@ -6,5 +6,9 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 
 require 'harvestdor-indexer'
 
-#RSpec.configure do |config|
-#end
+require 'vcr'
+
+VCR.configure do |c|
+  c.cassette_library_dir = 'spec/fixtures/vcr_cassettes'
+  c.hook_into :webmock
+end

data/spec/unit/harvestdor-indexer_spec.rb

@@ -4,7 +4,8 @@ describe Harvestdor::Indexer do
   
   before(:all) do
     @config_yml_path = File.join(File.dirname(__FILE__), "..", "config", "ap.yml")
-    @indexer = Harvestdor::Indexer.new(@config_yml_path)
+    @client_config_path = File.join(File.dirname(__FILE__), "../..", "config", "dor-fetcher-client.yml")
+    @indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
     require 'yaml'
     @yaml = YAML.load_file(@config_yml_path)
     @hdor_client = @indexer.send(:harvestdor_client)
@@ -65,55 +66,85 @@ describe Harvestdor::Indexer do
         :id => @fake_druid
       }
     end
-    it "should call druids_via_oai and then call :add on rsolr connection" do
+    it "should call dor_fetcher_client.druid_array and then call :add on rsolr connection" do
       @indexer.should_receive(:druids).and_return([@fake_druid])
       @indexer.solr_client.should_receive(:add).with(@doc_hash)
       @indexer.solr_client.should_receive(:commit)
       @indexer.harvest_and_index
     end
-    it "should not process druids in blacklist" do
-      indexer = Harvestdor::Indexer.new(@config_yml_path, {:blacklist => @blacklist_path})
-      hdor_client = indexer.send(:harvestdor_client)
-      hdor_client.should_receive(:druids_via_oai).and_return(['oo000oo0000', 'oo111oo1111', 'oo222oo2222', 'oo333oo3333'])
-      indexer.solr_client.should_receive(:add).with(hash_including({:id => 'oo000oo0000'}))
-      indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'oo111oo1111'}))
-      indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'oo222oo2222'}))
-      indexer.solr_client.should_receive(:add).with(hash_including({:id => 'oo333oo3333'}))
-      indexer.solr_client.should_receive(:commit)
-      indexer.harvest_and_index
+
+    it "should only call :commit on rsolr connection once" do
+      VCR.use_cassette('single_rsolr_connection_call') do
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
+        hdor_client = indexer.send(:harvestdor_client)
+        indexer.dor_fetcher_client.should_receive(:druid_array).and_return(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
+        indexer.solr_client.should_receive(:add).exactly(6).times
+        indexer.solr_client.should_receive(:commit).once
+        indexer.harvest_and_index
+      end
     end
-    it "should only process druids in whitelist if it exists" do
-      indexer = Harvestdor::Indexer.new(@config_yml_path, {:whitelist => @whitelist_path})
-      hdor_client = indexer.send(:harvestdor_client)
-      hdor_client.should_not_receive(:druids_via_oai)
-      indexer.solr_client.should_receive(:add).with(hash_including({:id => 'oo000oo0000'}))
-      indexer.solr_client.should_receive(:add).with(hash_including({:id => 'oo222oo2222'}))
-      indexer.solr_client.should_receive(:commit)
-      indexer.harvest_and_index
+
+    it "should not process druids in blacklist" do
+      VCR.use_cassette('ignore_druids_in_blacklist_call') do
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
+        hdor_client = indexer.send(:harvestdor_client)
+        indexer.dor_fetcher_client.should_receive(:druid_array).and_return(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
+        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:nz353cp1092'}))
+        indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
+        indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:tc552kq0798'}))
+        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:th998nk0722'}))
+        indexer.solr_client.should_receive(:commit)
+        indexer.harvest_and_index
+      end
     end
     it "should not process druid if it is in both blacklist and whitelist" do
-      indexer = Harvestdor::Indexer.new(@config_yml_path, {:blacklist => @blacklist_path, :whitelist => @whitelist_path})
-      hdor_client = indexer.send(:harvestdor_client)
-      hdor_client.should_not_receive(:druids_via_oai)
-      indexer.solr_client.should_receive(:add).with(hash_including({:id => 'oo000oo0000'}))
-      indexer.solr_client.should_receive(:commit)
-      indexer.harvest_and_index
+      VCR.use_cassette('ignore_druids_in_blacklist_and_whitelist_call') do
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path, :whitelist => @whitelist_path})
+        hdor_client = indexer.send(:harvestdor_client)
+        indexer.dor_fetcher_client.should_not_receive(:druid_array)
+        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:yg867hg1375'}))
+        indexer.solr_client.should_not_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
+        indexer.solr_client.should_receive(:commit)
+        indexer.harvest_and_index
+      end
     end
-    it "should only call :commit on rsolr connection once" do
-      indexer = Harvestdor::Indexer.new(@config_yml_path)
-      hdor_client = indexer.send(:harvestdor_client)
-      hdor_client.should_receive(:druids_via_oai).and_return(['1', '2', '3'])
-      indexer.solr_client.should_receive(:add).exactly(3).times
-      indexer.solr_client.should_receive(:commit).once
-      indexer.harvest_and_index
+    it "should only process druids in whitelist if it exists" do
+      VCR.use_cassette('process_druids_whitelist_call') do
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
+        hdor_client = indexer.send(:harvestdor_client)
+        indexer.dor_fetcher_client.should_not_receive(:druid_array)
+        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:yg867hg1375'}))
+        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:jf275fd6276'}))
+        indexer.solr_client.should_receive(:add).with(hash_including({:id => 'druid:nz353cp1092'}))
+        indexer.solr_client.should_receive(:commit)
+        indexer.harvest_and_index
+      end
     end
+
   end
   
-  it "druids method should call druids_via_oai method on harvestdor_client" do
-    @hdor_client.should_receive(:druids_via_oai).and_return([@fake_druid])
-    @indexer.druids
-  end
-  
+  # Check for replacement of oai harvesting with dor-fetcher
+  context "replacing OAI harvesting with dor-fetcher" do
+      it "has a dor-fetcher client" do
+        expect(@indexer.dor_fetcher_client).to be_an_instance_of(DorFetcher::Client)
+      end 
+
+      it "should strip off is_member_of_collection_ and is_governed_by_ and return only the druid" do
+        expect(@indexer.strip_default_set_string()).to eq("yg867hg1375")
+      end
+
+      it "druids method should call druid_array and get_collection methods on fetcher_client" do
+        VCR.use_cassette('get_collection_druids_call') do
+          expect(@indexer.druids).to eq(["druid:yg867hg1375", "druid:jf275fd6276", "druid:nz353cp1092", "druid:tc552kq0798", "druid:th998nk0722", "druid:ww689vs6534"])
+        end
+      end
+
+      it "should get the configuration of the dor-fetcher client from included yml file" do
+        expect(@indexer.dor_fetcher_client.service_url).to eq(@indexer.client_config["dor_fetcher_service_url"])
+      end
+
+  end # ending replacing OAI context
+
   context "smods_rec method" do
     before(:all) do
       @fake_druid = 'oo000oo0000'
@@ -134,7 +165,9 @@ describe Harvestdor::Indexer do
       expect { @indexer.smods_rec(@fake_druid) }.to raise_error(RuntimeError, Regexp.new("^Empty MODS metadata for #{@fake_druid}: <"))
     end
     it "should raise exception if there is no MODS xml for the druid" do
-      expect { @indexer.smods_rec(@fake_druid) }.to raise_error(Harvestdor::Errors::MissingMods)
+      VCR.use_cassette('exception_no_MODS_call') do
+        expect { @indexer.smods_rec(@fake_druid) }.to raise_error(Harvestdor::Errors::MissingMods)
+      end
     end
   end
   
@@ -253,30 +286,32 @@ describe Harvestdor::Indexer do
       @indexer.send(:blacklist).size.should == 2
     end
     it "should be empty Array if there was no blacklist config setting" do
-      indexer = Harvestdor::Indexer.new(@config_yml_path)
+      indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
       indexer.send(:blacklist).should == []
     end
     context "load_blacklist" do
       it "should not be called if there was no blacklist config setting" do
-        indexer = Harvestdor::Indexer.new(@config_yml_path)
+        VCR.use_cassette('no_blacklist_config_call') do
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
 
-        indexer.should_not_receive(:load_blacklist)
+          indexer.should_not_receive(:load_blacklist)
 
-        hdor_client = indexer.send(:harvestdor_client)
-        hdor_client.should_receive(:druids_via_oai).and_return([@fake_druid])
-        indexer.solr_client.should_receive(:add)
-        indexer.solr_client.should_receive(:commit)
-        indexer.harvest_and_index
+          hdor_client = indexer.send(:harvestdor_client)
+          indexer.dor_fetcher_client.should_receive(:druid_array).and_return([@fake_druid])
+          indexer.solr_client.should_receive(:add)
+          indexer.solr_client.should_receive(:commit)
+          indexer.harvest_and_index
+        end
       end
       it "should only try to load a blacklist once" do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, {:blacklist => @blacklist_path})
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => @blacklist_path})
         indexer.send(:blacklist)
         File.any_instance.should_not_receive(:open)
         indexer.send(:blacklist)
       end
       it "should log an error message and throw RuntimeError if it can't find the indicated blacklist file" do
         exp_msg = 'Unable to find list of druids at bad_path'
-        indexer = Harvestdor::Indexer.new(@config_yml_path, {:blacklist => 'bad_path'})
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:blacklist => 'bad_path'})
         indexer.logger.should_receive(:fatal).with(exp_msg)
         expect { indexer.send(:load_blacklist, 'bad_path') }.to raise_error(exp_msg)
       end   
@@ -287,33 +322,35 @@ describe Harvestdor::Indexer do
     it "should be an Array with an entry for each non-empty line in the file" do
       @indexer.send(:load_whitelist, @whitelist_path)
       @indexer.send(:whitelist).should be_an_instance_of(Array)
-      @indexer.send(:whitelist).size.should == 2
+      @indexer.send(:whitelist).size.should == 3
     end
     it "should be empty Array if there was no whitelist config setting" do
-      indexer = Harvestdor::Indexer.new(@config_yml_path)
+      indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
       indexer.send(:whitelist).should == []
     end
     context "load_whitelist" do
       it "should not be called if there was no whitelist config setting" do
-        indexer = Harvestdor::Indexer.new(@config_yml_path)
+        VCR.use_cassette('no_whitelist_config_call') do
+          indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path)
 
-        indexer.should_not_receive(:load_whitelist)
+          indexer.should_not_receive(:load_whitelist)
 
-        hdor_client = indexer.send(:harvestdor_client)
-        hdor_client.should_receive(:druids_via_oai).and_return([@fake_druid])
-        indexer.solr_client.should_receive(:add)
-        indexer.solr_client.should_receive(:commit)
-        indexer.harvest_and_index
+          hdor_client = indexer.send(:harvestdor_client)
+          indexer.dor_fetcher_client.should_receive(:druid_array).and_return([@fake_druid])
+          indexer.solr_client.should_receive(:add)
+          indexer.solr_client.should_receive(:commit)
+          indexer.harvest_and_index
+        end
       end
       it "should only try to load a whitelist once" do
-        indexer = Harvestdor::Indexer.new(@config_yml_path, {:whitelist => @whitelist_path})
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => @whitelist_path})
         indexer.send(:whitelist)
         File.any_instance.should_not_receive(:open)
         indexer.send(:whitelist)
       end
       it "should log an error message and throw RuntimeError if it can't find the indicated whitelist file" do
         exp_msg = 'Unable to find list of druids at bad_path'
-        indexer = Harvestdor::Indexer.new(@config_yml_path, {:whitelist => 'bad_path'})
+        indexer = Harvestdor::Indexer.new(@config_yml_path, @client_config_path, {:whitelist => 'bad_path'})
         indexer.logger.should_receive(:fatal).with(exp_msg)
         expect { indexer.send(:load_whitelist, 'bad_path') }.to raise_error(exp_msg)
       end   
@@ -321,7 +358,7 @@ describe Harvestdor::Indexer do
   end # whitelist
   
   it "solr_client should initialize the rsolr client using the options from the config" do
-    indexer = Harvestdor::Indexer.new(nil, Confstruct::Configuration.new(:solr => { :url => 'http://localhost:2345', :a => 1 }) )
+    indexer = Harvestdor::Indexer.new(nil, @client_config_path, Confstruct::Configuration.new(:solr => { :url => 'http://localhost:2345', :a => 1 }) )
     RSolr.should_receive(:connect).with(hash_including(:a => 1, :url => 'http://localhost:2345')).and_return('foo')
     indexer.solr_client
   end

metadata

@@ -1,206 +1,263 @@
 --- !ruby/object:Gem::Specification
 name: harvestdor-indexer
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 1.0.0
 platform: ruby
 authors:
 - Naomi Dushay
-autorequire: 
+- Bess Sadler
+- Laney McGlohon
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-08-05 00:00:00.000000000 Z
+date: 2014-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rsolr
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: retries
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: retries
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: harvestdor
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: harvestdor
+        version: 0.0.14
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.0.14
-  type: :runtime
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stanford-mods
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.14
-- !ruby/object:Gem::Dependency
-  name: stanford-mods
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: dor-fetcher
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: confstruct
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rdoc
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: yard
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rspec
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: coveralls
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: vcr
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: webmock
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Harvest DOR object metadata via a relationship (e.g. hydra:isGovernedBy
-  rdf:resource="info:fedora/druid:hy787xj5878") and dates, plus code framework to
-  write Solr docs to index
+  prerelease: false
+  type: :development
+description: Harvest DOR object metadata via a relationship (e.g. hydra:isGovernedBy rdf:resource="info:fedora/druid:hy787xj5878") and dates, plus code framework to write Solr docs to index
 email:
 - ndushay@stanford.edu
+- bess@stanford.edu
+- laneymcg@stanford.edu
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
-- ".yardopts"
+- .gitignore
+- .travis.yml
+- .yardopts
 - Gemfile
 - LICENSE.txt
 - README.rdoc
 - Rakefile
+- config/dor-fetcher-client.yml
 - harvestdor-indexer.gemspec
 - lib/harvestdor-indexer.rb
 - lib/harvestdor-indexer/version.rb
 - spec/config/ap.yml
 - spec/config/ap_blacklist.txt
 - spec/config/ap_whitelist.txt
+- spec/fixtures/vcr_cassettes/exception_no_MODS_call.yml
+- spec/fixtures/vcr_cassettes/get_collection_druids_call.yml
+- spec/fixtures/vcr_cassettes/ignore_druids_in_blacklist_call.yml
+- spec/fixtures/vcr_cassettes/no_blacklist_config_call.yml
+- spec/fixtures/vcr_cassettes/no_whitelist_config_call.yml
+- spec/fixtures/vcr_cassettes/single_rsolr_connection_call.yml
 - spec/spec_helper.rb
 - spec/unit/harvestdor-indexer_spec.rb
 homepage: https://consul.stanford.edu/display/chimera/Chimera+project
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.2.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Harvest DOR object metadata and index it to Solr
 test_files:
 - spec/config/ap.yml
 - spec/config/ap_blacklist.txt
 - spec/config/ap_whitelist.txt
+- spec/fixtures/vcr_cassettes/exception_no_MODS_call.yml
+- spec/fixtures/vcr_cassettes/get_collection_druids_call.yml
+- spec/fixtures/vcr_cassettes/ignore_druids_in_blacklist_call.yml
+- spec/fixtures/vcr_cassettes/no_blacklist_config_call.yml
+- spec/fixtures/vcr_cassettes/no_whitelist_config_call.yml
+- spec/fixtures/vcr_cassettes/single_rsolr_connection_call.yml
 - spec/spec_helper.rb
 - spec/unit/harvestdor-indexer_spec.rb
-has_rdoc: 
+has_rdoc: