harvestdor 0.0.13

data/lib/harvestdor.rb

@@ -0,0 +1,121 @@
+require 'harvestdor/errors'
+require 'harvestdor/oai_harvest'
+require 'harvestdor/purl_xml'
+require 'harvestdor/version'
+# external gems
+require 'confstruct'
+require 'oai'
+# stdlib
+require 'logger'
+require 'open-uri'
+require 'yaml'
+
+module Harvestdor
+  
+  LOG_NAME_DEFAULT = "harvestdor.log"
+  LOG_DIR_DEFAULT = File.join(File.dirname(__FILE__), "..", "logs")
+  PURL_DEFAULT = 'http://purl.stanford.edu'
+  HTTP_OPTIONS_DEFAULT = { 'ssl' => { 
+                              'verify' => false 
+                            }, 
+                           'request' => {
+                              'timeout' => 60, # open/read timeout (seconds)
+                              'open_timeout' => 60 # connection open timeout (seconds)
+                            }
+                          }
+  OAI_CLIENT_DEBUG_DEFAULT = false
+  OAI_REPOSITORY_URL_DEFAULT = 'https://dor-oaiprovider-prod.stanford.edu/oai'
+  DEFAULT_METADATA_PREFIX = 'mods'
+  DEFAULT_FROM_DATE = nil
+  DEFAULT_UNTIL_DATE = nil
+  DEFAULT_SET = nil
+  
+  class Client
+    
+    # Set default values for the construction of Harvestdor::Client objects
+    def self.default_config
+      @class_config ||= Confstruct::Configuration.new({
+        :log_dir => LOG_DIR_DEFAULT,
+        :log_name => LOG_NAME_DEFAULT,
+        :purl => PURL_DEFAULT,
+        :http_options => HTTP_OPTIONS_DEFAULT,
+        :oai_repository_url => OAI_REPOSITORY_URL_DEFAULT,
+        :oai_client_debug => OAI_CLIENT_DEBUG_DEFAULT,
+        :default_metadata_prefix => DEFAULT_METADATA_PREFIX,
+        :default_from_date => DEFAULT_FROM_DATE,
+        :default_until_date => DEFAULT_UNTIL_DATE,
+        :default_set => DEFAULT_SET
+      })
+    end
+    
+    # Initialize a new instance of Harvestdor::Client
+    # @param Hash options
+    # @example
+    #   client = Harvestdor::Client.new({ # Example with all possible options
+    #      :log_dir => File.join(File.dirname(__FILE__), "..", "logs"),
+    #      :log_name => 'harvestdor.log',
+    #      :purl => 'http://purl.stanford.edu',
+    #      :http_options => { 'ssl' => { 
+    #                          'verify' => false 
+    #                          }, 
+    #                         'request' => {
+    #                            'timeout' => 30, # open/read timeout (seconds)
+    #                            'open_timeout' => 30 # connection open timeout (seconds)
+    #                          }
+    #                        },
+    #      :oai_repository_url => 'https://dor-oaiprovider-prod.stanford.edu/oai', # The OAI repository to connect to
+    #      :oai_client_debug => false,
+    #      :default_metadata_prefix => 'mods',
+    #      :default_from_date => '2012-12-01', 
+    #      :default_until_date => '2014-12-01',
+    #      :default_set => nil, 
+    #   })
+    def initialize options = {}
+      config.configure(YAML.load_file(options[:config_yml_path])) if options[:config_yml_path]
+      config.configure options
+      yield(config) if block_given?
+    end
+    
+    def config
+      @config ||= Confstruct::Configuration.new(self.class.default_config)
+    end
+    
+    # @return OAI::Client an instantiated OAI::Client object, based on config options
+    def oai_client
+      @oai_client ||= OAI::Client.new config.oai_repository_url, :debug => config.oai_client_debug, :http => oai_http_client
+    end
+
+    def logger
+      @logger ||= self.class.logger(config.log_dir, config.log_name)
+    end
+
+    protected #---------------------------------------------------------------------
+    
+    def oai_http_client
+      logger.info "Constructing OAI http client with faraday options #{config.http_options.to_hash.inspect}"
+      @oai_http_client ||= Faraday.new config.oai_repository_url, config.http_options.to_hash
+    end
+
+    # Global, memoized, lazy initialized instance of a logger
+    # @param [String] log_dir directory for to get log file
+    # @param [String] log_name name of log file
+    def self.logger(log_dir, log_name)
+      Dir.mkdir(log_dir) unless File.directory?(log_dir) 
+      @logger ||= Logger.new(File.join(log_dir, log_name), 'daily')
+    end
+
+  end # class Client
+  
+  # @param [Object] arg OAI::Header object or OAI::Record object or String (oai identifier)
+  # @return [String] the druid part of an OAI identifier in an OAI header, e.g.  bb134cc1324
+  def self.druid(arg)
+    oai_id = arg
+    if arg.is_a?(OAI::Header)
+      oai_id = arg.identifier
+    elsif arg.is_a?(OAI::Record)
+      oai_id = arg.header.identifier
+    end
+    oai_id.split('druid:').last
+  end
+
+end # module Harvestdor

data/spec/config/oai.yml

@@ -0,0 +1,37 @@
+# log_dir:  directory for log file  (default logs, relative to harvestdor gem path)
+log_dir: spec/test_logs  
+
+# log_name: name of log file  (default: harvestdor.log)
+
+# purl: url for the DOR purl server (used to get ContentMetadata, etc.)
+# purl: http://purl-test.stanford.edu
+
+# ---------- OAI harvesting parameters -----------
+
+# oai_client_debug:  true for OAI::Client debug mode  (default: false)
+
+# oai_repository_url:  URL of the OAI data provider
+oai_repository_url: https://dor-oaiprovider-test.stanford.edu/oai
+
+# default_metadata_prefix:  default metadata prefix to be used for harvesting  (default: mods)
+#   can be overridden on calls to harvest_ids and harvest_records
+default_metadata_prefix: mods 
+
+# default_from_date:  default from date for harvest  (default: nil)
+#   can be overridden on calls to harvest_ids and harvest_records
+default_from_date: '2012-11-01'
+
+# default_until_date:  default until date for harvest  (default: nil)
+#   can be overridden on calls to harvest_ids and harvest_records
+
+# default_set:  default set for harvest  (default: nil)
+#   can be overridden on calls to harvest_ids and harvest_records
+
+# Additional options to pass to Faraday http client (https://github.com/technoweenie/faraday)
+# timeouts are in seconds;  timeout -> open/read, open_timeout -> connection open
+http_options:
+  ssl:
+    verify: false
+  request:
+    timeout: 121
+    open_timeout: 122

data/spec/harvestdor_client_spec.rb

@@ -0,0 +1,135 @@
+require "spec_helper"
+
+describe Harvestdor::Client do
+
+  before(:all) do
+    @config_yml_path = File.join(File.dirname(__FILE__), "config", "oai.yml")
+    @client_via_yml_only = Harvestdor::Client.new({:config_yml_path => @config_yml_path})
+    require 'yaml'
+    @yaml = YAML.load_file(@config_yml_path)
+  end
+
+  describe "initialization" do
+    before(:all) do
+      @from_date = '2012-11-29'
+      @repo_url = 'http://my_oai_repo.org/oai'
+    end
+    context "attributes passed in hash argument" do
+      before(:all) do
+        @some_args = Harvestdor::Client.new({:default_from_date => @from_date, :oai_repository_url => @repo_url}).config
+      end
+      it "should set the attributes to the passed values" do
+        expect(@some_args.oai_repository_url).to eql(@repo_url)
+        expect(@some_args.default_from_date).to eql(@from_date)
+      end
+      it "should keep the defaults for attributes not in the hash argument" do
+        expect(@some_args.log_name).to eql(Harvestdor::LOG_NAME_DEFAULT)
+        expect(@some_args.log_dir).to eql(Harvestdor::LOG_DIR_DEFAULT)
+        expect(@some_args.purl).to eql(Harvestdor::PURL_DEFAULT)
+        expect(@some_args.http_options).to eql(Confstruct::Configuration.new(Harvestdor::HTTP_OPTIONS_DEFAULT))
+        expect(@some_args.oai_client_debug).to eql(Harvestdor::OAI_CLIENT_DEBUG_DEFAULT)
+        expect(@some_args.default_metadata_prefix).to eql(Harvestdor::DEFAULT_METADATA_PREFIX)
+        expect(@some_args.default_until_date).to eql(Harvestdor::DEFAULT_UNTIL_DATE)
+        expect(@some_args.default_set).to eql(Harvestdor::DEFAULT_SET)
+      end
+    end
+    
+    context "config_yml_path in hash argument" do
+      before(:all) do
+        @config_via_yml_only = @client_via_yml_only.config
+      end
+      it "should set attributes in yml file over defaults" do
+        expect(@config_via_yml_only.log_dir).to eql(@yaml['log_dir'])
+        expect(@config_via_yml_only.oai_repository_url).to eql(@yaml['oai_repository_url'])
+        expect(@config_via_yml_only.default_from_date).to eql(@yaml['default_from_date'])
+        expect(@config_via_yml_only.default_metadata_prefix).to eql(@yaml['default_metadata_prefix'])
+        expect(@config_via_yml_only.http_options.request.timeout).to eql(@yaml['http_options']['request']['timeout'])
+      end
+      it "should keep the defaults for attributes not present in yml file nor a config yml file" do
+        expect(@config_via_yml_only.log_name).to eql(Harvestdor::LOG_NAME_DEFAULT)
+        expect(@config_via_yml_only.purl).to eql(Harvestdor::PURL_DEFAULT)
+        expect(@config_via_yml_only.default_until_date).to eql(Harvestdor::DEFAULT_UNTIL_DATE)
+        expect(@config_via_yml_only.default_set).to eql(Harvestdor::DEFAULT_SET)
+      end
+      context "and some hash arguments" do
+        before(:all) do
+          @config_via_yml_plus = Harvestdor::Client.new({:config_yml_path => @config_yml_path, 
+            :default_from_date => @from_date, :oai_repository_url => @repo_url}).config
+        end
+        it "should favor hash arg attribute values over yml file values" do
+          expect(@config_via_yml_plus.oai_repository_url).to eql(@repo_url)
+          expect(@config_via_yml_plus.default_from_date).to eql(@from_date)
+        end
+        it "should favor yml file values over defaults" do
+          expect(@config_via_yml_plus.log_dir).to eql(@yaml['log_dir'])
+          expect(@config_via_yml_plus.default_metadata_prefix).to eql(@yaml['default_metadata_prefix'])
+          expect(@config_via_yml_plus.http_options.timeout).to eql(@yaml['http_options']['timeout'])
+        end
+        it "should keep the defaults for attributes not present in yml file" do
+          expect(@config_via_yml_plus.log_name).to eql(Harvestdor::LOG_NAME_DEFAULT)
+          expect(@config_via_yml_plus.default_until_date).to eql(Harvestdor::DEFAULT_UNTIL_DATE)
+          expect(@config_via_yml_plus.default_set).to eql(Harvestdor::DEFAULT_SET)
+        end
+      end
+    end
+    
+    context "without hash arguments" do
+      it "should keep the defaults for all attributes" do
+        no_args = Harvestdor::Client.new.config
+        expect(no_args.log_name).to eql(Harvestdor::LOG_NAME_DEFAULT)
+        expect(no_args.log_dir).to eql(Harvestdor::LOG_DIR_DEFAULT)
+        expect(no_args.purl).to eql(Harvestdor::PURL_DEFAULT)
+        expect(no_args.http_options).to eql(Confstruct::Configuration.new(Harvestdor::HTTP_OPTIONS_DEFAULT))
+        expect(no_args.oai_client_debug).to eql(Harvestdor::OAI_CLIENT_DEBUG_DEFAULT)
+        expect(no_args.oai_repository_url).to eql(Harvestdor::OAI_REPOSITORY_URL_DEFAULT)
+        expect(no_args.default_metadata_prefix).to eql(Harvestdor::DEFAULT_METADATA_PREFIX)
+        expect(no_args.default_from_date).to eql(Harvestdor::DEFAULT_FROM_DATE)
+        expect(no_args.default_until_date).to eql(Harvestdor::DEFAULT_UNTIL_DATE)
+        expect(no_args.default_set).to eql(Harvestdor::DEFAULT_SET)
+      end
+    end
+  end # initialize client
+  
+  it "should allow direct setting of configuration attributes" do
+    conf = Harvestdor::Client.new.config
+    expect(conf.log_dir).to eql(Harvestdor::LOG_DIR_DEFAULT)
+    conf['log_dir'] = 'my_log_dir'
+    expect(conf.log_dir).to eql('my_log_dir')
+  end
+
+  describe "logging" do
+    it "should write the log file to the directory indicated by log_dir" do
+      @client_via_yml_only.logger.info("harvestdor_client_spec logging test message")
+      expect(File.exists?(File.join(@yaml['log_dir'], Harvestdor::LOG_NAME_DEFAULT))).to eql(true)
+    end
+  end
+  
+  context "oai_client" do
+    before(:all) do
+      @client = Harvestdor::Client.new
+      @default_oai_client = Harvestdor::Client.new.oai_client
+    end
+    
+    it "oai_client should return an OAI::Client object based on config data" do
+      expect(@default_oai_client).to be_an_instance_of(OAI::Client)
+    end 
+    
+    it "oai_client should have an http_client" do
+      expect(@default_oai_client.instance_variable_get(:@http_client)).to be_an_instance_of(Faraday::Connection)
+    end
+
+    context "oai_http_client (protected method)" do
+	    before(:all) do
+	      @http_client = @client.send(:oai_http_client)
+	    end
+	    it "should be a Faraday object" do
+	      expect(@http_client).to be_an_instance_of(Faraday::Connection)
+	    end
+	    it "should have the oai_provider url from config" do
+	      uri_obj = @http_client.url_prefix
+	      expect(@client.config.oai_repository_url).to match(Regexp.new(uri_obj.host + uri_obj.path))
+	    end
+	  end
+  end # context oai_client
+  
+end

data/spec/harvestdor_spec.rb

@@ -0,0 +1,23 @@
+require "spec_helper"
+
+describe Harvestdor do
+
+  context "#druid" do
+    it "should return the druid part of an oai identifier" do
+      Harvestdor.druid('oai:searchworks.stanford.edu/druid:foo').should == 'foo'
+    end
+    it "should work with OAI::Header as argument" do
+      header = OAI::Header.new(nil)
+      header.identifier = 'oai:searchworks.stanford.edu/druid:foo'
+      Harvestdor.druid(header).should == 'foo'
+    end
+    it "should work with OAI::Record as argument" do
+      oai_rec = OAI::Record.new(nil)
+      header = OAI::Header.new(nil)
+      header.identifier = 'oai:searchworks.stanford.edu/druid:foo'
+      oai_rec.header = header
+      Harvestdor.druid(oai_rec).should == 'foo'
+    end
+  end
+
+end

data/spec/oai_harvest_spec.rb

@@ -0,0 +1,220 @@
+require 'spec_helper'
+
+describe 'Harvestdor::Client oai harvesting' do
+  before(:all) do
+    @harvestdor_client = Harvestdor::Client.new
+    @oai_arg_defaults = {:metadata_prefix => @harvestdor_client.config.default_metadata_prefix, 
+                :from => @harvestdor_client.config.default_from_date,
+                :until => @harvestdor_client.config.default_until_date,
+                :set => @harvestdor_client.config.default_set  }
+  end
+  
+  describe "druids_via_oai" do
+    before(:each) do
+      oai_response = double('oai_response')
+      oai_response.stub(:entries).and_return(['foo', 'bar'])
+      oai_response.stub(:resumption_token).and_return('')
+      @harvestdor_client.oai_client.stub(:list_identifiers).with(an_instance_of(Hash)) { 
+          oai_response
+      }
+    end
+    it "should return druids" do
+      header1 = OAI::Header.new(nil)
+      header1.identifier = 'oai:searchworks.stanford.edu/druid:foo'
+      header2 = OAI::Header.new(nil)
+      header2.identifier = 'oai:searchworks.stanford.edu/druid:bar'
+      oai_response = double('oai_response')
+      oai_response.stub(:entries).and_return([header1, header2])
+      @harvestdor_client.druids_via_oai.should == ['foo', 'bar']
+    end
+    it "should have results viewable as an array" do
+      @harvestdor_client.druids_via_oai.should be_an_instance_of(Array)
+    end
+    it "should have enumerable results" do
+      @harvestdor_client.druids_via_oai.should respond_to(:each, :count)
+    end
+    it "should yield to a passed block" do
+      expect { |b| @harvestdor_client.druids_via_oai(&b) }.to yield_successive_args('foo', 'bar')
+    end
+  end
+  
+  describe "oai_records" do
+    before(:each) do
+      @oai_response = double('oai_response')
+      @oai_response.stub(:entries).and_return([1, 2])
+      @oai_response.stub(:resumption_token).and_return('')
+      @harvestdor_client.oai_client.stub(:list_records).with(an_instance_of(Hash)) { 
+          @oai_response
+      }
+    end
+    it "should return OAI::Record objects" do
+      header1 = OAI::Header.new(nil)
+      header1.identifier = 'oai:searchworks.stanford.edu/druid:foo'
+      oai_rec1 = OAI::Record.new(nil)
+      oai_rec1.header = header1
+      header2 = OAI::Header.new(nil)
+      header2.identifier = 'oai:searchworks.stanford.edu/druid:bar'
+      oai_rec2 = OAI::Record.new(nil)
+      oai_rec2.header = header2
+      @oai_response.stub(:entries).and_return([oai_rec1, oai_rec2])
+      @harvestdor_client.oai_records.should == [oai_rec1, oai_rec2]
+    end
+    it "should have results viewable as an array" do
+      @harvestdor_client.oai_records.should be_an_instance_of(Array)
+    end
+    it "should have enumerable results" do
+      @harvestdor_client.oai_records.should respond_to(:each, :count)
+    end
+    it "should yield to a passed block" do
+      expect { |b| @harvestdor_client.oai_records(&b) }.to yield_successive_args(1, 2)
+    end
+  end  
+
+  describe "oai_headers" do
+    before(:each) do
+      @oai_response = double('oai_response')
+      @oai_response.stub(:entries).and_return([1, 2])
+      @oai_response.stub(:resumption_token).and_return('')
+      @harvestdor_client.oai_client.stub(:list_identifiers).with(an_instance_of(Hash)) { 
+          @oai_response
+      }
+    end
+    it "should return OAI::Header objects" do
+      header1 = OAI::Header.new(nil)
+      header1.identifier = 'oai:searchworks.stanford.edu/druid:foo'
+      header2 = OAI::Header.new(nil)
+      header2.identifier = 'oai:searchworks.stanford.edu/druid:bar'
+      @oai_response.stub(:entries).and_return([header1, header2])
+      @harvestdor_client.oai_headers.should == [header1, header2]
+    end
+    it "should have results viewable as an array" do
+      @harvestdor_client.oai_headers.should be_an_instance_of(Array)
+    end
+    it "should have enumerable results" do
+      @harvestdor_client.oai_headers.should respond_to(:each, :count)
+    end
+    it "should yield to a passed block" do
+      expect { |b| @harvestdor_client.oai_headers(&b) }.to yield_successive_args(1, 2)
+    end
+  end  
+    
+  describe "oai_record (single record request)" do
+    it "should return OAI::Record object" do
+      oai_rec = OAI::Record.new(nil)
+      oai_resp = double('oai_response')
+      oai_resp.stub(:record).and_return(oai_rec)
+      @harvestdor_client.oai_client.stub(:get_record) { 
+          oai_resp
+      }
+      @harvestdor_client.oai_record('druid').should == oai_rec
+      @harvestdor_client.oai_record('druid', 'mods').should == oai_rec
+    end
+  end  
+
+  describe "scrub_oai_args" do
+    before(:all) do
+      @expected_oai_args = @oai_arg_defaults.dup
+      @expected_oai_args.each { |k, v|  
+        @expected_oai_args.delete(k) if v.nil? || v.size == 0
+      }
+      
+    end
+    it "should use client's default values for OAI arguments if they are not present in the method param hash" do
+      @harvestdor_client.send(:scrub_oai_args).should == @expected_oai_args
+    end
+    it "should use OAI arguments from the method param hash if they are present" do
+      passed_options = {:metadata_prefix => 'mods', :from => '2012-11-30'}
+      @harvestdor_client.send(:scrub_oai_args, passed_options).should == @expected_oai_args.merge(passed_options)
+    end
+    it "should use nil value for option when it is passed in options hash" do
+      client = Harvestdor::Client.new({:default_from_date => '2012-01-01'})
+      client.config.default_from_date.should == '2012-01-01'
+      passed_options = {:from => nil}
+      client.send(:scrub_oai_args, passed_options)[:from].should == nil
+    end
+  end
+
+  describe "harvest" do
+    it "should perform a list_records OAI request when first arg is true" do
+      oai_response = double('oai_response')
+      oai_response.stub(:entries).and_return([])
+      @harvestdor_client.oai_client.stub(:list_records).with(an_instance_of(Hash)) { 
+          oai_response
+      }
+      @harvestdor_client.oai_client.should_receive(:list_records)
+      @harvestdor_client.send(:harvest, :list_records, {})
+    end
+    
+    it "should perform a list_identifiers OAI request when first arg is false" do
+      oai_response = double('oai_response')
+      oai_response.stub(:entries).and_return([])
+      @harvestdor_client.oai_client.stub(:list_identifiers).with(an_instance_of(Hash)) { 
+          oai_response
+      }
+      @harvestdor_client.oai_client.should_receive(:list_identifiers)
+      @harvestdor_client.send(:harvest, :list_identifiers, {})
+    end
+
+    it "should use passed OAI arguments" do
+      oai_response = double('oai_response')
+      oai_response.stub(:entries).and_return([])
+      @harvestdor_client.oai_client.stub(:list_identifiers).with(an_instance_of(Hash)) { 
+          oai_response
+      }
+      oai_options_hash = {:metadata_prefix => 'mods', :from => '2012-11-30'}
+      @harvestdor_client.oai_client.should_receive(:list_identifiers).with(oai_options_hash)
+      @harvestdor_client.send(:harvest, :list_identifiers, oai_options_hash)
+    end
+    
+    it "should yield to a passed block" do
+      oai_response = double('oai_response')
+      oai_response.stub(:entries).and_return([1, 2])
+      oai_response.stub(:resumption_token).and_return('')
+      @harvestdor_client.oai_client.stub(:list_records).with(an_instance_of(Hash)) { 
+          oai_response
+      }
+      expect { |b| @harvestdor_client.send(:harvest, :list_records, {}, &b) }.to yield_successive_args(1, 2)
+    end
+
+    context "resumption tokens" do
+      it "should stop processing when no records/headers are received" do
+        oai_response = double('oai_response')
+        oai_response.stub(:entries).and_return([])
+        @harvestdor_client.oai_client.stub(:list_records).with(an_instance_of(Hash)) { 
+            oai_response
+        }
+
+        i = 0
+        @harvestdor_client.send(:harvest, :list_records, {}) { |record| i += 1 }
+        i.should == 0
+      end
+
+      it "should stop processing when the resumption token is empty" do
+        oai_response_with_token = double('oai_response')
+        oai_response_with_token.stub(:entries).and_return([1,2,3,4,5])
+        oai_response_with_token.stub(:resumption_token).and_return('')
+        @harvestdor_client.oai_client.stub(:list_records).with(an_instance_of(Hash)) { 
+          oai_response_with_token
+        }
+
+        i = 0
+        @harvestdor_client.send(:harvest, :list_records, {}) { |record| i += 1 }
+        i.should == 5
+      end
+
+      it "should stop processing when there was no resumption token" do
+        oai_response_with_token = double('oai_response')
+        oai_response_with_token.stub(:entries).and_return([1,2,3,4,5])
+        oai_response_with_token.stub(:resumption_token).and_return(nil)
+        @harvestdor_client.oai_client.stub(:list_records).with(an_instance_of(Hash)) { 
+          oai_response_with_token
+        }
+
+        i = 0
+        @harvestdor_client.send(:harvest, :list_records, {}) { |record| i += 1 }
+        expect(i).to eql(5)
+      end      
+    end # resumption tokens
+  end
+        
+end

data/spec/oai_integration_spec.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe 'Harvestdor::Client OAI Harvesting Integration Tests', :integration => true do
+
+  before(:all) do
+    @config_yml_path = File.join(File.dirname(__FILE__), "config", "oai.yml")
+  end
+
+  context "test OAI server" do
+    before(:all) do
+      @test_hclient ||= Harvestdor::Client.new({:config_yml_path => @config_yml_path, :oai_client_debug => 'true', :oai_repository_url => 'https://dor-oaiprovider-test.stanford.edu/oai'})
+    end
+    context "withOUT resumption tokens" do
+      before(:all) do
+        @oai_args = {:metadata_prefix => 'mods', :from => nil, :until => nil, :set => 'is_governed_by_hy787xj5878'}
+      end
+      it "should be able to harvest headers" do
+        headers = @test_hclient.oai_headers(@oai_args)
+        headers.should be_an_instance_of(Array)
+        headers.size.should > 0
+        headers.size.should < 50  # no resumption token
+        headers.first.should be_an_instance_of(OAI::Header)
+      end
+      it "should be able to harvest records" do
+        records = @test_hclient.oai_records(@oai_args)
+        records.should be_an_instance_of(Array)
+        records.size.should > 0
+        records.size.should < 50  # no resumption token
+        records.first.should be_an_instance_of(OAI::Record)
+      end
+    end
+    context "with resumption tokens" do
+      before(:all) do
+        @oai_args = {:metadata_prefix => 'mods', :from => nil, :until => nil, :set => 'is_member_of_kh678dr8608'}
+      end
+      it "should be able to harvest headers" do
+        skip "need to find small set > 50 on test"
+        headers = @test_hclient.oai_headers(@oai_args)
+        headers.should be_an_instance_of(Array)
+        headers.size.should > 50
+        headers.first.should be_an_instance_of(OAI::Header)
+      end
+      it "should be able to harvest records" do
+        pending "need to find small set > 50 on test"
+        records = @test_hclient.harvest_records(@oai_args)
+        records.should be_an_instance_of(Array)
+        records.size.should > 50
+        records.first.should be_an_instance_of(OAI::Record)
+      end
+    end
+    context "oai_record (single record request)" do
+      before(:all) do
+        @rec = @test_hclient.oai_record('jt959wc5586')
+      end
+      it "should get a single OAI::Record object" do
+        @rec.should be_an_instance_of(OAI::Record)
+      end
+      it "should keep utf-8 encoded characters intact" do
+        xml = Nokogiri::XML(@rec.metadata.to_s)
+        xml.remove_namespaces!
+        xml.root.xpath('/metadata/mods/titleInfo/subTitle').text.should =~ /^recueil complet des débats législatifs & politiques des chambres françaises/
+      end
+    end
+  end
+  
+  context "production OAI server" do
+    before(:all) do
+      @prod_hclient ||= Harvestdor::Client.new({:config_yml_path => @config_yml_path, :oai_repository_url => 'https://dor-oaiprovider-prod.stanford.edu/oai'})
+    end
+    context "withOUT resumption tokens" do
+      before(:all) do
+        # Reid-Dennis: 47 objects
+        @oai_args = {:metadata_prefix => 'mods', :from => nil, :until => nil, :set => 'is_governed_by_sd064kn5856'}
+      end
+      it "should be able to harvest headers" do
+        headers = @prod_hclient.oai_headers(@oai_args)
+        headers.should be_an_instance_of(Array)
+        headers.size.should > 0
+        headers.size.should < 50  # no resumption token
+        headers.first.should be_an_instance_of(OAI::Header)
+      end
+      it "should be able to harvest records" do
+        records = @prod_hclient.oai_records(@oai_args)
+        records.should be_an_instance_of(Array)
+        records.size.should > 0
+        records.size.should < 50  # no resumption token
+        records.first.should be_an_instance_of(OAI::Record)
+      end
+    end
+    context "with resumption tokens" do
+      before(:all) do
+        # Archives Parlementaires - 8x objects
+        @oai_args = {:metadata_prefix => 'mods', :from => nil, :until => nil, :set => 'is_member_of_collection_jh957jy1101'}
+      end
+      it "should be able to harvest headers" do
+        headers = @prod_hclient.oai_headers(@oai_args)
+        headers.should be_an_instance_of(Array)
+        headers.size.should > 50
+        headers.first.should be_an_instance_of(OAI::Header)
+      end
+      it "should be able to harvest records" do
+        pending "the request always seems to time out"
+        records = @prod_hclient.oai_records(@oai_args)
+        records.should be_an_instance_of(Array)
+        records.size.should > 50
+        records.first.should be_an_instance_of(OAI::Record)
+      end
+    end
+    context "oai_record (single record request)" do
+      before(:all) do
+        @rec = @prod_hclient.oai_record('jt959wc5586')
+      end
+      it "should get a single OAI::Record object" do
+        @rec.should be_an_instance_of(OAI::Record)
+      end
+      it "should keep utf-8 encoded characters intact" do
+        xml = Nokogiri::XML(@rec.metadata.to_s)
+        xml.remove_namespaces!
+        xml.root.xpath('/metadata/mods/titleInfo/subTitle').text.should =~ /^recueil complet des débats législatifs & politiques des chambres françaises/
+      end
+    end
+  end
+
+end