oai 0.0.1 → 1.3.0

data/test/activerecord_provider/tc_simple_paging_provider.rb

@@ -0,0 +1,71 @@
+require 'test_helper_ar_provider'
+
+class SimpleResumptionProviderTest < TransactionalTestCase
+  include REXML
+
+  def test_full_harvest
+    doc = Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
+    assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
+    assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
+    token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
+
+    doc = Document.new(@provider.list_records(:resumption_token => token))
+    assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
+    token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
+    assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
+
+    doc = Document.new(@provider.list_records(:resumption_token => token))
+    assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
+    token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
+    assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
+
+    doc = Document.new(@provider.list_records(:resumption_token => token))
+    assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
+    assert_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
+    assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
+  end
+
+  def test_non_integer_identifiers_resumption
+    @provider = SimpleResumptionProviderWithNonIntegerID.new
+
+    doc = Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
+    assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
+    assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
+    token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
+
+    next_doc = Document.new(@provider.list_records(:resumption_token => token))
+    assert_not_nil next_doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
+    next_token = next_doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
+    assert_equal 26, next_doc.elements["/OAI-PMH/ListRecords"].to_a.size
+
+    assert_not_equal token, next_token
+  end
+
+  def test_from_and_until
+    first_id = DCField.order("id asc").first.id
+    DCField.where("dc_fields.id < ?", first_id + 25).update_all(updated_at: Time.parse("September 15 2005"))
+    DCField.where(["dc_fields.id <= ? and dc_fields.id > ?", first_id + 50, first_id + 25]).update_all(updated_at: Time.parse("November 1 2005"))
+
+    total = DCField.where(["dc_fields.updated_at >= ? AND dc_fields.updated_at <= ?", Time.parse("September 1 2005"), Time.parse("November 30 2005")]).count
+
+    # Should return 50 records broken into 2 groups of 25.
+    doc = Document.new(
+      @provider.list_records(
+        :metadata_prefix => 'oai_dc',
+        :from => Time.parse("September 1 2005"),
+        :until => Time.parse("November 30 2005"))
+      )
+    assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
+    assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
+    token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
+    doc = Document.new(@provider.list_records(:resumption_token => token))
+    assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
+    assert_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
+    assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
+  end
+
+  def setup
+    @provider = SimpleResumptionProvider.new
+  end
+
+end

data/test/activerecord_provider/test_helper_ar_provider.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+
+require 'test/unit'
+require File.dirname(__FILE__) + '/config/connection'
+require File.dirname(__FILE__) + '/helpers/providers'
+require File.dirname(__FILE__) + '/helpers/set_provider'
+require File.dirname(__FILE__) + '/helpers/transactional_test_case'

data/test/client/helpers/provider.rb

@@ -0,0 +1,65 @@
+require 'webrick'
+require File.dirname(__FILE__) + '/../../provider/models'
+
+
+class ProviderServer
+  class ComplexClientProvider < OAI::Provider::Base
+    repository_name 'Complex Provider'
+    repository_url 'http://localhost'
+    record_prefix 'oai:test'
+    source_model ComplexModel.new(100)
+  end
+
+  attr_reader :consumed, :server
+
+  def initialize(port, mount_point)
+    @consumed = []
+    @provider = ComplexClientProvider.new
+    @server = WEBrick::HTTPServer.new(
+      :BindAddress => '127.0.0.1',
+      :Logger => WEBrick::Log.new('/dev/null'),
+      :AccessLog => [],
+      :Port => port)
+    @server.mount_proc(mount_point, server_proc)
+  end
+
+  def port
+    @server.config[:Port]
+  end
+
+  def start
+    @thread = Thread.new { @server.start }
+  end
+
+  def stop
+    @thread.exit if @thread
+  end
+
+  def self.wrap(port = 3333, mount_point='/oai')
+    server = self.new(port, mount_point)
+    begin
+      server.start
+      yield(server)
+    ensure
+      server.stop
+    end
+  end
+
+  protected
+
+  def server_proc
+    Proc.new do |req, res|
+      begin
+        res.body = @provider.process_request(req.query)
+        res.status = 200
+        res['Content-Type'] = 'text/xml'
+      rescue => err
+        puts err
+        puts err.backtrace.join("\n")
+        res.body = err.backtrace.join("\n")
+        res.status = 500
+      end
+    end
+  end
+
+end

data/test/client/helpers/test_wrapper.rb

@@ -0,0 +1,6 @@
+unless $provider_server
+  $provider_server = ProviderServer.new(3333, '/oai')
+  $provider_server.start
+  sleep 0.2
+end
+

data/test/client/tc_exception.rb

@@ -0,0 +1,36 @@
+require 'test_helper_client'
+
+class ExceptionTest < Test::Unit::TestCase
+
+  def test_not_found
+    client = OAI::Client.new 'http://www.google.com'
+    assert_raises(OAI::Exception) { client.identify }
+  end
+
+  def test_xml_error
+    client = OAI::Client.new 'http://www.yahoo.com'
+    begin
+      client.identify
+    rescue OAI::Exception => e
+      assert_match /response not well formed XML/, e.to_s, 'xml error'
+    end
+  end
+
+  def test_oai_error
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    assert_raises(OAI::ResumptionTokenException) do
+      client.list_identifiers :resumption_token => 'bogus'
+    end
+  end
+
+  # must pass in options as a hash
+  def test_parameter_error
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    assert_raises(OAI::ArgumentException) {client.get_record('foo')}
+    assert_raises(OAI::ArgumentException) {client.list_identifiers('foo')}
+    assert_raises(OAI::ArgumentException) {client.list_records('foo')}
+    assert_raises(OAI::ArgumentException) {client.list_metadata_formats('foo')}
+    assert_raises(OAI::ArgumentException) {client.list_sets('foo')}
+  end
+
+end

data/test/client/tc_get_record.rb

@@ -0,0 +1,39 @@
+require 'test_helper_client'
+
+class GetRecordTest < Test::Unit::TestCase
+
+  def test_get_one
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    response = client.get_record :identifier => 'oai:test:3'
+    assert_kind_of OAI::GetRecordResponse, response
+    assert_kind_of OAI::Record, response.record
+    assert_kind_of REXML::Element, response.record._source
+    assert_kind_of REXML::Element, response.record.metadata
+    assert_kind_of OAI::Header, response.record.header
+    assert_kind_of REXML::Element, response.record.about
+
+    # minimal check that the header is working
+    assert_equal 'oai:test:3', response.record.header.identifier
+
+    # minimal check that the metadata is working
+    #assert 'en', response.record.metadata.elements['.//dc:language'].text
+    assert_equal 'Ruby OAI test data', response.record.about.elements['.//dc:publisher'].text
+  end
+
+  def test_missing_identifier
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    begin
+      client.get_record :metadata_prefix => 'oai_dc'
+      flunk 'invalid get_record did not throw OAI::Exception'
+    rescue OAI::ArgumentException => e
+      assert_match /The request includes illegal arguments/, e.to_s
+    end
+  end
+
+  def test_deleted_record
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    record = client.get_record :identifier => 'oai:test:275'
+    assert record.deleted?
+  end
+
+end

data/test/client/tc_http_client.rb

@@ -0,0 +1,129 @@
+require 'test_helper_client'
+require 'webrick'
+
+class HttpClientTest < Test::Unit::TestCase
+  def test_user_agent_and_from_headers
+    faraday_stub = Faraday.new do |builder|
+      builder.adapter :test do |stub|
+        stub.get('/echo') { |env| [200, {}, Marshal.dump(env)] }
+      end
+    end
+
+    client = OAI::Client.new 'http://localhost:3333/oai', :headers => { 'From' => 'oai@example.com', 'User-Agent' => 'ruby-oai' }, :http => faraday_stub
+
+    response = client.send(:get, '/echo')
+    env = Marshal.load(response)
+
+    assert_equal('oai@example.com', env.request_headers['From'])
+    assert_equal('ruby-oai', env.request_headers['User-Agent'])
+  end
+
+  def test_pluggable_http_client
+    oai_response = <<-eos
+    <Identify>
+      <repositoryName>Mock OAI Provider</repositoryName>
+      <baseURL>http://nowhere.example.com</baseURL>
+    </Identify>
+eos
+
+    faraday_stub = Faraday.new do |builder|
+      builder.adapter :test do |stub|
+        stub.get('/oai?verb=Identify') { [200, {}, oai_response] }
+      end
+    end
+    client = OAI::Client.new 'http://localhost:3333/oai', :http => faraday_stub
+    response = client.identify
+
+    assert_kind_of OAI::IdentifyResponse, response
+    assert_equal 'Mock OAI Provider [http://nowhere.example.com]', response.to_s
+
+  end
+
+  def test_http_client_handles_trailing_slash_redirects
+    # First, test that this works when mocking out Faraday client
+    oai_response = <<-eos
+      <Identify>
+        <repositoryName>Mock OAI Provider</repositoryName>
+        <baseURL>http://nowhere.example.com</baseURL>
+      </Identify>
+    eos
+
+    stubs = TrailingSlashAwareStubs.new do |stub|
+      stub.get('/oai/?verb=Identify') { [200, {}, oai_response] }
+      stub.get('/oai?verb=Identify') {
+        [301, {
+          'Location' => 'http://localhost:3334/oai/?verb=Identify'
+        }, '']
+      }
+    end
+
+    faraday_stub = Faraday.new do |builder|
+      require 'faraday/follow_redirects'
+      builder.use Faraday::FollowRedirects::Middleware
+      builder.adapter :test, stubs
+    end
+
+    client = OAI::Client.new 'http://localhost:3334/oai', :http => faraday_stub
+    response = client.identify
+
+    assert_kind_of OAI::IdentifyResponse, response
+    assert_equal 'Mock OAI Provider [http://nowhere.example.com]', response.to_s
+    assert_equal 2, stubs.consumed[:get].length
+    assert_equal stubs.consumed[:get].first.path, '/oai'
+    assert_equal stubs.consumed[:get].last.path, '/oai/'
+
+    # Now try it with a real server and default Faraday client
+    TrailingSlashProviderServer.wrap(3334) do |server|
+      client = OAI::Client.new "http://localhost:#{server.port}/oai"
+      response = client.identify
+
+      assert_kind_of OAI::IdentifyResponse, response
+      assert_equal 'Complex Provider [http://localhost]', response.to_s
+      assert_equal 2, server.consumed.length
+      assert_equal server.consumed.first.path, '/oai'
+      assert_equal server.consumed.last.path, '/oai/'
+    end
+  end
+
+  private
+
+  class TrailingSlashProviderServer < ProviderServer
+    def server_proc
+      Proc.new do |req, res|
+        @consumed << req
+        case req.path
+        when "/oai/"
+          begin
+            res.body = @provider.process_request(req.query)
+            res.status = 200
+            res['Content-Type'] = 'text/xml'
+          rescue => err
+            puts err
+            puts err.backtrace.join("\n")
+            res.body = err.backtrace.join("\n")
+            res.status = 500
+          end
+        else
+          res.body = ''
+          res.status = 301
+          res['Location'] = "http://localhost:#{port}/oai/?#{req.query_string}"
+        end
+        res
+      end
+    end
+  end
+
+  class TrailingSlashAwareStubs < Faraday::Adapter::Test::Stubs
+    attr_reader :consumed
+
+    # ensure leading, but not trailing slash
+    def normalize_path(path)
+      path = '/' + path if path.index('/') != 0
+      #path = path.sub('?', '/?')
+      #path = path + '/' unless $&
+      path.gsub('//', '/')
+    end
+
+  end
+end
+

data/test/client/tc_identify.rb

@@ -0,0 +1,13 @@
+require 'test_helper_client'
+
+class IdentifyTest < Test::Unit::TestCase
+
+  def test_ok
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    response = client.identify
+    assert_kind_of OAI::IdentifyResponse, response
+    assert_equal 'Complex Provider [http://localhost]', response.to_s
+    #assert_equal 'PubMed Central (PMC3 - NLM DTD) [http://www.pubmedcentral.gov/oai/oai.cgi]', response.to_s
+  end
+
+end

data/test/client/tc_libxml.rb

@@ -0,0 +1,61 @@
+require 'test_helper_client'
+
+class LibXMLTest < Test::Unit::TestCase
+
+  def test_oai_exception
+    return unless have_libxml
+
+    uri = 'http://localhost:3333/oai'
+    client = OAI::Client.new uri, :parser => 'libxml'
+    assert_raises(OAI::IdException) {client.get_record(:identifier => 'nosuchid')}
+  end
+
+  def test_list_records
+    return unless have_libxml
+
+    # since there is regex magic going on to remove default oai namespaces
+    # it's worth trying a few different oai targets
+    oai_targets = %w{
+      http://localhost:3333/oai
+    }
+
+    #oai_targets = %w{
+    #  http://etd.caltech.edu:80/ETD-db/OAI/oai
+    #  http://ir.library.oregonstate.edu/dspace-oai/request
+    #  http://memory.loc.gov/cgi-bin/oai2_0
+    #  http://libeprints.open.ac.uk/perl/oai2
+    #}
+
+
+    oai_targets.each do |uri|
+      client = OAI::Client.new uri, :parser => 'libxml'
+      records = client.list_records
+      records.each do |record|
+        assert record.header.identifier
+        next if record.deleted?
+        assert_kind_of LibXML::XML::Node, record.metadata
+      end
+    end
+  end
+
+  def test_deleted_record
+    return unless have_libxml
+
+    uri = 'http://localhost:3333/oai'
+    client = OAI::Client.new(uri, :parser => 'libxml')
+    response = client.get_record :identifier => 'oai:test:275'
+    assert response.record.deleted?
+  end
+
+  private
+
+  def have_libxml
+    begin
+      require 'xml/libxml'
+      return true
+    rescue LoadError
+      return false
+    end
+  end
+
+end

data/test/{tc_list_identifiers.rb → client/tc_list_identifiers.rb}

@@ -1,12 +1,12 @@
-require 'date'
+require 'test_helper_client'
 
 class ListIdentifiersTest < Test::Unit::TestCase
 
   def test_list_with_resumption_token
-    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi' 
+    client = OAI::Client.new 'http://localhost:3333/oai'
 
     # get a list of identifier headers
-    response = client.list_identifiers :metadata_prefix => 'oai_dc' 
+    response = client.list_identifiers :metadata_prefix => 'oai_dc'
     assert_kind_of OAI::ListIdentifiersResponse, response
     assert_kind_of OAI::Response, response
     assert response.entries.size > 0
@@ -27,17 +27,36 @@ class ListIdentifiersTest < Test::Unit::TestCase
     assert_not_equal response.entries[0].identifier, first_identifier
   end
 
+  def test_list_full
+    client = OAI::Client.new 'http://localhost:3333/oai'
+
+    # get a list of identifier headers
+    response = client.list_identifiers :metadata_prefix => 'oai_dc'
+    assert_kind_of OAI::ListIdentifiersResponse, response
+    assert_kind_of OAI::Response, response
+    assert response.respond_to?(:full), "Should expose :full"
+
+    # Check that it runs through the pages
+    assert_equal 1150, response.full.count
+    response.full.each do |header|
+      assert_kind_of OAI::Header, header
+      assert header.identifier
+      assert header.datestamp
+      assert header.set_spec
+    end
+  end
+
   def test_list_with_date_range
-    client = OAI::Client.new 'http://alcme.oclc.org/xtcat/servlet/OAIHandler'
-    from_date = Date.new(2001,1,1)
-    until_date  = Date.new(2006,1,1)
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    from_date = Date.new(1998,1,1)
+    until_date  = Date.new(2002,1,1)
     response = client.list_identifiers :from => from_date, :until => until_date
     assert response.entries.size > 0
   end
 
   def test_list_with_datetime_range
     # xtcat should support higher granularity
-    client = OAI::Client.new 'http://alcme.oclc.org/xtcat/servlet/OAIHandler'
+    client = OAI::Client.new 'http://localhost:3333/oai'
     from_date = DateTime.new(2001,1,1)
     until_date = DateTime.now
     response = client.list_identifiers :from => from_date, :until => until_date
@@ -45,8 +64,8 @@ class ListIdentifiersTest < Test::Unit::TestCase
   end
 
   def test_invalid_argument
-    client = OAI::Client.new 'http://arXiv.org/oai2'
-    assert_raise(OAI::Exception) {client.list_identifiers :foo => 'bar'}
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    assert_raise(OAI::ArgumentException) {client.list_identifiers :foo => 'bar'}
   end
 
 end

data/test/{tc_list_metadata_formats.rb → client/tc_list_metadata_formats.rb}

@@ -1,6 +1,8 @@
+require 'test_helper_client'
+
 class ListMetadataFormatsTest < Test::Unit::TestCase
   def test_list
-    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi' 
+    client = OAI::Client.new 'http://localhost:3333/oai'
     response = client.list_metadata_formats
     assert_kind_of OAI::ListMetadataFormatsResponse, response
     assert response.entries.size > 0
@@ -11,5 +13,6 @@ class ListMetadataFormatsTest < Test::Unit::TestCase
     assert_equal 'http://www.openarchives.org/OAI/2.0/oai_dc.xsd', format.schema
     assert_equal 'http://www.openarchives.org/OAI/2.0/oai_dc/', format.namespace
   end
+
 end
 

data/test/client/tc_list_records.rb

@@ -0,0 +1,26 @@
+require 'test_helper_client'
+
+class ListRecordsTest < Test::Unit::TestCase
+
+  def test_list
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    response = client.list_records
+    assert_kind_of OAI::ListRecordsResponse, response
+    assert response.entries.size > 0
+    assert_kind_of OAI::Record,  response.entries[0]
+  end
+
+  def test_list_full
+    client = OAI::Client.new 'http://localhost:3333/oai'
+
+    response = client.list_records
+    assert_kind_of OAI::ListRecordsResponse, response
+
+    # Check that it runs through the pages
+    assert_equal 1150, response.full.count
+    response.full.each do |record|
+      assert_kind_of OAI::Record, record
+    end
+  end
+
+end

data/test/client/tc_list_sets.rb

@@ -0,0 +1,34 @@
+require 'test_helper_client'
+
+class ListSetsTest < Test::Unit::TestCase
+
+  def test_list
+    client = OAI::Client.new 'http://localhost:3333/oai'
+    response = client.list_sets
+    assert_kind_of OAI::ListSetsResponse, response
+    assert response.entries.size > 0
+    assert_kind_of OAI::Set,  response.entries[0]
+
+    # test iterator
+    for set in response
+      assert_kind_of OAI::Set, set
+    end
+  end
+
+  def test_list_full
+    client = OAI::Client.new 'http://localhost:3333/oai'
+
+    response = client.list_sets
+    assert_kind_of OAI::ListSetsResponse, response
+    assert_kind_of OAI::Response, response
+    assert response.respond_to?(:full), "Should expose :full"
+
+    # This won't page, but it should work anyway
+    assert_equal 6, response.full.count
+    response.full.each do |set|
+      assert_kind_of OAI::Set, set
+    end
+  end
+
+end
+

data/test/client/tc_low_resolution_dates.rb

@@ -0,0 +1,14 @@
+require 'test_helper_client'
+
+class LowResolutionDatesTest < Test::Unit::TestCase
+
+  def test_low_res_date_parsing
+    client = OAI::Client.new 'http://localhost:3333/oai'
+
+    date = Date.new 2003, 1, 1
+
+    # get a list of identifier headers
+    assert_nothing_raised { client.list_identifiers :from => date }
+  end
+
+end

data/test/client/tc_utf8_escaping.rb

@@ -0,0 +1,19 @@
+require 'test_helper_client'
+
+class UTF8Test < Test::Unit::TestCase
+  def client
+    @client ||= OAI::Client.new 'http://localhost:3333/oai'
+  end
+
+  def test_escaping_invalid_utf_8_characters
+    invalid_utf_8 = [2, 3, 4, 104, 5, 101, 6, 108, 66897, 108, 66535, 111, 1114112, 33, 55234123, 33].pack("U*")
+    invalid_utf_8 = invalid_utf_8.force_encoding("binary") if invalid_utf_8.respond_to? :force_encoding
+    assert_equal("hello!!", client.send(:strip_invalid_utf_8_chars, invalid_utf_8).gsub(/\?/, ''))
+  end
+
+  def test_unescaped_ampersand_content_correction
+    src = '<test>Frankie & Johnny <character>&#9829;</character></test>'
+    expected = '<test>Frankie &amp; Johnny <character>&#9829;</character></test>'
+    assert_equal(expected, client.sanitize_xml(src))
+  end
+end

data/test/client/tc_xpath.rb

@@ -0,0 +1,26 @@
+require 'test_helper_client'
+
+class XpathTest < Test::Unit::TestCase
+  include OAI::XPath
+
+  def test_rexml
+    require 'rexml/document'
+    doc = REXML::Document.new(File.new('test/test.xml'))
+    assert_equal xpath(doc, './/responseDate'), '2006-09-11T14:33:15Z'
+    assert_equal xpath(doc, './/foobar'), nil
+  end
+
+  def test_libxml
+    begin
+      require 'xml/libxml'
+    rescue LoadError
+      # libxml not available so nothing to test!
+      return
+    end
+
+    doc = LibXML::XML::Document.file('test/test.xml')
+    assert_equal xpath(doc, './/responseDate'), '2006-09-11T14:33:15Z'
+    assert_equal xpath(doc, './/foobar'), nil
+  end
+
+end

data/test/client/test_helper_client.rb

@@ -0,0 +1,5 @@
+require 'oai'
+require 'test/unit'
+
+require File.dirname(__FILE__) + '/helpers/provider'
+require File.dirname(__FILE__) + '/helpers/test_wrapper'