oai 0.0.2 → 0.0.3

data/lib/oai/client.rb

@@ -1,6 +1,5 @@
 require 'uri'
 require 'net/http'
-require 'rexml/document'
 require 'cgi'
 require 'date'
 
@@ -10,8 +9,10 @@ module OAI
   # a OAI-PMH server. The 6 OAI-PMH verbs translate directly to methods you 
   # can call on a OAI::Client object. Verb arguments are passed as a hash:
   #
-  #   client = OAI::Client.new ''http://www.pubmedcentral.gov/oai/oai.cgi'
-  #   client.list_identifiers :metadata_prefix => 'oai_dc'
+  #   client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+  #   record = client.get_record :identifier => 'oai:pubmedcentral.gov:13901'
+  #   for identifier in client.list_identifiers :metadata_prefix => 'oai_dc'
+  #     puts identifier.
   #
   # It is worth noting that the api uses methods and parameter names with 
   # underscores in them rather than studly caps. So above list_identifiers 
@@ -37,15 +38,40 @@ module OAI
     # If you want to see debugging messages on STDERR use:
     #
     #   client = OAI::Harvester.new 'http://example.com', :debug => true
+    #
+    # By default OAI verbs called on the client will return REXML::Element
+    # objects for metadata records, however if you wish you can use the
+    # :parser option to indicate you want to use 'libxml' instead, and get
+    # back XML::Node objects
+    #
+    #   client = OAI::Harvester.new 'http://example.com', :parser => 'libxml'
     
     def initialize(base_url, options={})
       @base = URI.parse base_url
-      @debug = options[:debug]
+      @debug = options.fetch(:debug, false)
+      @parser = options.fetch(:parser, 'rexml')
+      
+      # load appropriate parser
+      case @parser
+      when 'libxml'
+        begin
+	  require 'rubygems'
+          require 'xml/libxml'
+        rescue
+          raise OAI::Exception.new("xml/libxml not available")
+        end
+      when 'rexml'
+        require 'rexml/document'
+        require 'rexml/xpath'
+      else
+        raise OAI::Exception.new("unknown parser: #{@parser}")
+      end
     end
 
     # Equivalent to a Identify request. You'll get back a OAI::IdentifyResponse
     # object which is essentially just a wrapper around a REXML::Document 
-    # for the response.
+    # for the response. If you are created your client using the libxml 
+    # parser then you will get an XML::Node object instead.
     
     def identify
       return IdentifyResponse.new(do_request(:verb => 'Identify'))
@@ -55,8 +81,7 @@ module OAI
     # object is returned to you. 
     
     def list_metadata_formats(opts={})
-      opts[:verb] = 'ListMetadataFormats'
-      verify_verb_arguments opts, [:verb, :identifier]
+      sanitize_verb_arguments 'ListMetadataFormats', opts, [:verb, :identifier]
       return ListMetadataFormatsResponse.new(do_request(opts))
     end
 
@@ -65,9 +90,9 @@ module OAI
     # supported by the server.
     
     def list_identifiers(opts={})
-      opts[:verb] = 'ListIdentifiers'
+      sanitize_verb_arguments 'ListIdentifiers', opts, 
+        [:verb, :from, :until, :metadata_prefix, :set, :resumption_token]
       add_default_metadata_prefix opts
-      verify_verb_arguments opts, [:verb, :from, :until, :metadata_prefix, :set,         :resumption_token]
       return ListIdentifiersResponse.new(do_request(opts)) 
     end
 
@@ -76,9 +101,9 @@ module OAI
     # which you can extract a OAI::Record object from.
     
     def get_record(opts={})
-      opts[:verb] = 'GetRecord'
+      sanitize_verb_arguments 'GetRecord', opts, 
+        [:verb, :identifier, :metadata_prefix]
       add_default_metadata_prefix opts
-      verify_verb_arguments opts, [:verb, :identifier, :metadata_prefix]
       return GetRecordResponse.new(do_request(opts))
     end
 
@@ -90,10 +115,9 @@ module OAI
     #   end
     
     def list_records(opts={})
-      opts[:verb] = 'ListRecords'
-      add_default_metadata_prefix opts
-      verify_verb_arguments opts, [:verb, :from, :until, :set, 
+      sanitize_verb_arguments 'ListRecords', opts, [:verb, :from, :until, :set, 
         :resumption_token, :metadata_prefix]
+      add_default_metadata_prefix opts
       return ListRecordsResponse.new(do_request(opts))
     end
 
@@ -106,8 +130,7 @@ module OAI
     #   end
     
     def list_sets(opts={})
-      opts[:verb] = 'ListSets'
-      verify_verb_arguments opts, [:verb, :resumptionToken]
+      sanitize_verb_arguments 'ListSets', opts, [:verb, :resumptionToken]
       return ListSetsResponse.new(do_request(opts))
     end
 
@@ -131,18 +154,41 @@ module OAI
       uri.query = parts.join('&')
       debug("doing request: #{uri.to_s}")
 
-      # fire off the request and return an REXML::Document object
+      # fire off the request and return appropriate DOM object
       begin
         xml = Net::HTTP.get(uri)
-        debug("got response: #{xml}")
-        return REXML::Document.new(xml)
-      rescue REXML::ParseException => e
-        raise OAI::Exception, 'response not well formed XML: '+e, caller
-      rescue SystemCallError=> e
+	if @parser == 'libxml' 
+          # remove default namespace for oai-pmh since libxml
+          # isn't able to use our xpaths to get at them 
+          # if you know a way around thins please let me know
+          xml = xml.gsub(
+            /xmlns=\"http:\/\/www.openarchives.org\/OAI\/.\..\/\"/, '') 
+        end
+        return load_document(xml)
+      rescue StandardError => e
         raise OAI::Exception, 'HTTP level error during OAI request: '+e, caller
       end
     end
 
+    def load_document(xml)
+      case @parser
+      when 'libxml'
+        begin
+          parser = XML::Parser.new()
+          parser.string = xml
+          return parser.parse
+        rescue XML::Parser::ParseError => e
+          raise OAI::Exception, 'response not well formed XML: '+e, caller
+        end
+      when 'rexml'
+        begin
+          return REXML::Document.new(xml)
+        rescue REXML::ParseException => e
+          raise OAI::Exception, 'response not well formed XML: '+e, caller
+        end
+      end
+    end
+
     # convert foo_bar to fooBar thus allowing our ruby code to use
     # the typical underscore idiom
     def studly(s)
@@ -160,7 +206,17 @@ module OAI
       end
     end
 
-    def verify_verb_arguments(opts, valid_opts)
+    def sanitize_verb_arguments(verb, opts, valid_opts)
+      # opts could mistakenly not be a hash if the method was called wrong
+      # client.get_record(12) instead of client.get_record(:identifier => 12)
+      unless opts.kind_of?(Hash)
+        raise OAI::Exception.new("method options must be passed as a hash") 
+      end
+
+      # add the verb
+      opts[:verb] = verb
+
+      # make sure options aren't using studly caps, and that they're legit
       opts.keys.each do |opt|
         if opt =~ /[A-Z]/
           raise OAI::Exception.new("#{opt} should use underscores")

data/lib/oai/get_record.rb

@@ -5,7 +5,11 @@ module OAI
 
     def initialize(doc)
       super doc
-      @record = OAI::Record.new(xpath_first(doc, './/record'))
+      @record = OAI::Record.new(xpath_first(doc, './/GetRecord/record'))
+    end
+
+    def deleted?
+      return @record.deleted?
     end
   end
 end

data/lib/oai/header.rb

@@ -4,9 +4,15 @@ module OAI
     attr_accessor :identifier, :datestamp, :set_spec
 
     def initialize(element)
+      @status = get_attribute(element, 'status')
       @identifier = xpath(element, './/identifier')
       @datestamp = xpath(element, './/datestamp')
       @set_spec = xpath(element, './/setSpec')
     end
+
+    def deleted?
+      return true unless @status == 'deleted'
+    end
+
   end
 end

data/lib/oai/identify.rb

@@ -21,6 +21,8 @@ module OAI
     end
 
     # returns REXML::Element nodes for each description section
+    # if the OAI::Client was configured to use libxml then you will
+    # instead get a XML::Node object.
     def descriptions
       return xpath_all(doc, './/Identify/description')
     end

data/lib/oai/record.rb

@@ -4,6 +4,9 @@ module OAI
   # or ListRecords request. Each record will have a header and metadata
   # attribute. The header is a OAI::Header object and the metadata is 
   # a REXML::Element object for that chunk of XML. 
+  #
+  # Note: if your OAI::Client was configured to use the 'libxml' parser
+  # metadata will return a XML::Node object instead.
   
   class Record
     include OAI::XPath
@@ -13,5 +16,11 @@ module OAI
       @header = OAI::Header.new xpath_first(element, './/header')
       @metadata = xpath_first(element, './/metadata')
     end
+
+    # a convenience method which digs into the header status attribute
+    # and returns true if the value is set to 'deleted'
+    def deleted?
+      return @header.deleted?
+    end
   end
 end

data/lib/oai/response.rb

@@ -9,11 +9,17 @@ module OAI
 
       # throw an exception if there was an error
       error = xpath_first(doc, './/error')
-      if error
-        message = error.text
-        code = error.attributes['code']
-        raise OAI::Exception.new("#{message} [#{code}]")
+      return unless error
+
+      case error.class.to_s
+        when 'REXML::Element'
+          message = error.text
+          code = error.attributes['code']
+        when 'XML::Node'
+          message = error.content
+          code = error.property('code')
       end
+      raise OAI::Exception.new("#{message} [#{code}]")
     end
 
   end

data/lib/oai/xpath.rb

@@ -1,21 +1,64 @@
-require 'rexml/xpath'
-
 module OAI
   module XPath
+
+    # get all matching nodes
     def xpath_all(doc, path)
-      return REXML::XPath.match(doc, path)
+      case parser_type(doc)
+      when 'libxml'
+        return doc.find(path)
+      when 'rexml'
+        return REXML::XPath.match(doc, path)
+      end
+      return []
     end
 
+    # get first matching node
     def xpath_first(doc, path)
       elements = xpath_all(doc, path)
       return elements[0] if elements != nil
       return nil
     end
 
+    # get text for first matching node
     def xpath(doc, path)
-      e = xpath_first(doc, path)
-      return e.text if e != nil
+      el = xpath_first(doc, path)
+      return unless el
+      case parser_type(doc)
+      when 'libxml'
+        return el.content
+      when 'rexml'
+        return el.text 
+      end
       return nil
     end
+
+    # figure out an attribute
+    def get_attribute(node, attr_name)
+      case node.class.to_s
+      when 'REXML::Element'
+        return node.attribute(attr_name)
+      when 'XML::Node'
+        return node.property(attr_name)
+      end
+      return nil
+    end
+
+    private 
+   
+    # figure out what sort of object we should do xpath on
+    def parser_type(x)
+      case x.class.to_s
+      when 'XML::Document'
+        return 'libxml'
+      when 'XML::Node'
+        return 'libxml'
+      when 'XML::Node::Set'
+	return 'libxml'
+      when 'REXML::Element'
+        return 'rexml'
+      when 'REXML::Document'
+        return 'rexml'
+      end
+    end
   end
 end

data/lib/test.rb

@@ -0,0 +1,25 @@
+require 'oai'
+
+buffer = ""
+start_time = Time.now()
+
+client = OAI::Client.new 'http://digitalcollections.library.oregonstate.edu/cgi-bin/oai.exe', :parser =>'libxml'
+
+last_check = Date.new(2006,9,5)
+records = client.list_records
+# :set => 'archives', :metadata_prefix => 'oai_dc', :from => last_check 
+
+x = 0
+records.each do |record|
+  #fields = record.serialize_metadata(record.metadata, "oai_dc", "Oai_Dc")
+  #puts "Primary Title: " + fields.title[0] + "\n"
+  puts "Identifier: " +  record.header.identifier + "\n"
+  x += 1
+end
+
+end_time = Time.now()
+
+puts buffer
+puts "Time to run: " + (end_time - start_time).to_s + "\n"
+puts "Records returned: " + x.to_s
+

data/test/tc_exception.rb

@@ -6,16 +6,33 @@ class ExceptionTest < Test::Unit::TestCase
       client.identify
       flunk 'did not throw expected exception'
     rescue OAI::Exception => e
-      assert_match /Connection refused/, e.to_s, 'include error message'
+      assert_match /^HTTP level error/, e.to_s, 'include error message'
     end
   end
 
   def test_xml_error
-    client = OAI::Client.new 'http://www.google.com'
+    client = OAI::Client.new 'http://www.yahoo.com'
     begin 
       client.identify
     rescue OAI::Exception => e
       assert_match /response not well formed XML/, e.to_s, 'xml error'
     end
   end
+
+  def test_oai_error
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    assert_raises(OAI::Exception) do
+      client.list_identifiers :resumption_token => 'bogus'
+    end
+  end
+
+  # must pass in options as a hash 
+  def test_parameter_error
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    assert_raises(OAI::Exception) {client.get_record('foo')}
+    assert_raises(OAI::Exception) {client.list_identifiers('foo')}
+    assert_raises(OAI::Exception) {client.list_records('foo')}
+    assert_raises(OAI::Exception) {client.list_metadata_formats('foo')}
+    assert_raises(OAI::Exception) {client.list_sets('foo')}
+  end
 end

data/test/tc_get_record.rb

@@ -24,4 +24,10 @@ class GetRecordTest < Test::Unit::TestCase
       assert_match /The request includes illegal arguments/, e.to_s
     end
   end
+
+  def test_deleted_record
+    client = OAI::Client.new 'http://ir.library.oregonstate.edu/dspace-oai/request'
+    record = client.get_record :identifier => 'oai:ir.library.oregonstate.edu:1957/19' 
+    assert record.deleted?
+  end
 end

data/test/tc_identify.rb

@@ -3,6 +3,6 @@ class IdentifyTest < Test::Unit::TestCase
     client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
     response = client.identify
     assert_kind_of OAI::IdentifyResponse, response
-    assert_equal 'PubMed Central (PMC3 - NLM DTD) [http://www.pubmedcentral.nih.gov:80/oai/oai.cgi]', response.to_s
+    assert_equal 'PubMed Central (PMC3 - NLM DTD) [http://www.pubmedcentral.gov/oai/oai.cgi]', response.to_s
   end
 end

data/test/tc_libxml.rb

@@ -0,0 +1,51 @@
+class LibXMLTest < Test::Unit::TestCase
+
+  def test_oai_exception
+    return unless have_libxml
+
+    uri = 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    client = OAI::Client.new uri, :parser => 'libxml'
+    assert_raises(OAI::Exception) {client.get_record(:identifier => 'nosuchid')}
+  end
+
+  def test_list_records
+    return unless have_libxml
+
+    # since there is regex magic going on to remove default oai namespaces 
+    # it's worth trying a few different oai targets
+    oai_targets = %w{
+      http://etd.caltech.edu:80/ETD-db/OAI/oai
+      http://ir.library.oregonstate.edu/dspace-oai/request
+      http://libeprints.open.ac.uk/perl/oai2
+      http://memory.loc.gov/cgi-bin/oai2_0
+    }
+
+    oai_targets.each do |uri|
+      client = OAI::Client.new uri, :parser => 'libxml'
+      records = client.list_records
+      records.each do |record|
+        assert record.header.identifier
+        next unless record.deleted?
+        assert_kind_of XML::Node, record.metadata
+      end
+    end
+  end
+
+  def test_deleted_record
+    uri = 'http://ir.library.oregonstate.edu/dspace-oai/request'
+    client = OAI::Client.new(uri, :parser => 'libxml')
+    record = client.get_record :identifier => 'oai:ir.library.oregonstate.edu:1957/19' 
+  end
+
+  private
+
+  def have_libxml
+    begin
+      require 'xml/libxml'
+      return true
+    rescue
+      return false
+    end
+  end
+
+end

data/test/tc_list_identifiers.rb

@@ -1,5 +1,3 @@
-require 'date'
-
 class ListIdentifiersTest < Test::Unit::TestCase
 
   def test_list_with_resumption_token

data/test/tc_xpath.rb

@@ -0,0 +1,29 @@
+require 'oai/xpath'
+
+class XpathTest < Test::Unit::TestCase
+  include OAI::XPath
+
+  def test_rexml
+    require 'rexml/document'
+    doc = REXML::Document.new(File.new('test/test.xml'))
+    assert_equal xpath(doc, './/responseDate'), '2006-09-11T14:33:15Z'
+    assert_equal xpath(doc, './/foobar'), nil
+  end
+
+  def test_libxml
+    begin 
+      require 'xml/libxml'
+    rescue
+      # libxml not available so nothing to test!
+      return
+    end
+
+    doc = XML::Document.file('test/test.xml')
+    assert_equal xpath(doc, './/responseDate'), '2006-09-11T14:33:15Z'
+    assert_equal xpath(doc, './/foobar'), nil
+  end
+
+end
+
+__END__
+

data/test/test.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OAI-PMH xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+  <responseDate>2006-09-11T14:33:15Z</responseDate>
+  <request verb="Identify">http://www.pubmedcentral.gov/oai/oai.cgi</request>
+  <Identify>
+    <repositoryName>PubMed Central (PMC3 - NLM DTD)</repositoryName>
+    <baseURL>http://www.pubmedcentral.gov/oai/oai.cgi</baseURL>
+    <protocolVersion>2.0</protocolVersion>
+    <adminEmail>oai@ncbi.nlm.nih.gov</adminEmail>
+    <earliestDatestamp>1999-01-01</earliestDatestamp>
+    <deletedRecord>no</deletedRecord>
+    <granularity>YYYY-MM-DD</granularity>
+    <description>
+      <oai-identifier xmlns="http://www.openarchives.org/OAI/2.0/oai-identifier" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai-identifier http://www.openarchives.org/OAI/2.0/oai-identifier.xsd">
+        <scheme>oai</scheme>
+        <repositoryIdentifier>pubmedcentral.gov</repositoryIdentifier>
+        <delimiter>:</delimiter>
+        <sampleIdentifier>oai:pubmedcentral.gov:13900</sampleIdentifier>
+      </oai-identifier>
+    </description>
+  </Identify>
+</OAI-PMH>

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.8.11
+rubygems_version: 0.9.0
 specification_version: 1
 name: oai
 version: !ruby/object:Gem::Version 
-  version: 0.0.2
-date: 2006-05-15 00:00:00 -05:00
+  version: 0.0.3
+date: 2006-09-19 00:00:00 -04:00
 summary: A ruby library for working with the Open Archive Initiative Protocol for Metadata Harvesting (OAI-PMH)
 require_paths: 
-  - lib
+- lib
 email: ehs@pobox.com
 homepage: http://www.textualize.com/ruby-marc
 rubyforge_project: 
@@ -18,45 +18,55 @@ bindir: bin
 has_rdoc: true
 required_ruby_version: !ruby/object:Gem::Version::Requirement 
   requirements: 
-    - 
-      - ">"
-      - !ruby/object:Gem::Version 
-        version: 0.0.0
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
   version: 
 platform: ruby
 signing_key: 
 cert_chain: 
+post_install_message: 
 authors: 
-  - Ed Summers
+- Ed Summers
 files: 
-  - lib/oai
-  - lib/oai.rb
-  - lib/oai/client.rb
-  - lib/oai/exception.rb
-  - lib/oai/get_record.rb
-  - lib/oai/header.rb
-  - lib/oai/identify.rb
-  - lib/oai/list_identifiers.rb
-  - lib/oai/list_metadata_formats.rb
-  - lib/oai/list_records.rb
-  - lib/oai/list_sets.rb
-  - lib/oai/metadata_format.rb
-  - lib/oai/record.rb
-  - lib/oai/response.rb
-  - lib/oai/set.rb
-  - lib/oai/xpath.rb
-  - test/tc_exception.rb
-  - test/tc_get_record.rb
-  - test/tc_identify.rb
-  - test/tc_list_identifiers.rb
-  - test/tc_list_metadata_formats.rb
-  - test/tc_list_records.rb
-  - test/tc_list_sets.rb
-test_files: 
-  - test.rb
+- lib/oai
+- lib/oai.rb
+- lib/test.rb
+- lib/oai/client.rb
+- lib/oai/exception.rb
+- lib/oai/get_record.rb
+- lib/oai/header.rb
+- lib/oai/identify.rb
+- lib/oai/list_identifiers.rb
+- lib/oai/list_metadata_formats.rb
+- lib/oai/list_records.rb
+- lib/oai/list_sets.rb
+- lib/oai/metadata_format.rb
+- lib/oai/record.rb
+- lib/oai/response.rb
+- lib/oai/set.rb
+- lib/oai/xpath.rb
+- test/tc_exception.rb
+- test/tc_get_record.rb
+- test/tc_identify.rb
+- test/tc_libxml.rb
+- test/tc_list_identifiers.rb
+- test/tc_list_metadata_formats.rb
+- test/tc_list_records.rb
+- test/tc_list_sets.rb
+- test/tc_xpath.rb
+- test/test.xml
+test_files: []
+
 rdoc_options: []
+
 extra_rdoc_files: []
+
 executables: []
+
 extensions: []
+
 requirements: []
-dependencies: []
+
+dependencies: []
+

data/test.rb

@@ -1,13 +0,0 @@
-#!/usr/bin/env ruby
-
-$LOAD_PATH.unshift 'lib'
-
-require 'test/unit'
-require 'oai'
-require 'test/tc_list_identifiers'
-require 'test/tc_list_metadata_formats'
-require 'test/tc_identify'
-require 'test/tc_get_record'
-require 'test/tc_list_records'
-require 'test/tc_list_sets'
-require 'test/tc_exception'