oai 0.0.1

data/lib/oai.rb

@@ -0,0 +1,14 @@
+require 'oai/xpath'
+require 'oai/response'
+require 'oai/exception'
+require 'oai/header'
+require 'oai/record'
+require 'oai/set'
+require 'oai/metadata_format'
+require 'oai/client'
+require 'oai/identify'
+require 'oai/list_identifiers'
+require 'oai/list_metadata_formats'
+require 'oai/get_record'
+require 'oai/list_records'
+require 'oai/list_sets'

data/lib/oai/client.rb

@@ -0,0 +1,163 @@
+require 'uri'
+require 'net/http'
+require 'rexml/document'
+require 'cgi'
+
+module OAI
+
+  # A OAI::Client provides a client api for issuing OAI-PMH verbs against
+  # a OAI-PMH server. The 6 OAI-PMH verbs translate directly to methods you 
+  # can call on a OAI::Client object. Verb arguments are passed as a hash:
+  #
+  #   client = OAI::Harvester.new ''http://www.pubmedcentral.gov/oai/oai.cgi'
+  #   client.list_identifiers :metadata_prefix => 'oai_dc'
+  #
+  # It is worth noting that the api uses methods and parameter names with 
+  # underscores in them rather than studly caps. So above list_identifiers 
+  # and metadata_prefix are used instead of the listIdentifiers and 
+  # metadataPrefix used in the OAI-PMH specification.
+  #
+  # Also, the from and until arguments which specify dates should be passed
+  # in as Date or DateTime objects depending on the granularity supported
+  # by the server.
+  #
+  # For detailed information on the arguments that can be used please consult
+  # the OAI-PMH docs at:
+  #
+  #     http://www.openarchives.org/OAI/openarchivesprotocol.html
+  
+  class Client
+
+    # The constructor which must be passed a valid base url for an oai 
+    # service:
+    #
+    #   client = OAI::Harvseter.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    
+    def initialize(base_url)
+      @base = URI.parse base_url
+    end
+
+    # Equivalent to a Identify request. You'll get back a OAI::IdentifyResponse
+    # object which is essentially just a wrapper around a REXML::Document 
+    # for the response.
+    
+    def identify
+      return IdentifyResponse.new(do_request(:verb => 'Identify'))
+    end
+
+    # Equivalent to a ListMetadataFormats request. A ListMetadataFormatsResponse
+    # object is returned to you. 
+    
+    def list_metadata_formats(opts={})
+      opts[:verb] = 'ListMetadataFormats'
+      verify_verb_arguments opts, [:verb, :identifier]
+      return ListMetadataFormatsResponse.new(do_request(opts))
+    end
+
+    # Equivalent to a ListIdentifiers request. Pass in :from, :until arguments
+    # as Date or DateTime objects as appropriate depending on the granularity 
+    # supported by the server.
+    
+    def list_identifiers(opts={})
+      opts[:verb] = 'ListIdentifiers'
+      add_default_metadata_prefix opts
+      verify_verb_arguments opts, [:verb, :from, :until, :metadata_prefix, :set,         :resumption_token]
+      return ListIdentifiersResponse.new(do_request(opts)) 
+    end
+
+    # Equivalent to a GetRecord request. You must supply an identifier 
+    # argument. You should get back a OAI::GetRecordResponse object
+    # which you can extract a OAI::Record object from.
+    
+    def get_record(opts={})
+      opts[:verb] = 'GetRecord'
+      add_default_metadata_prefix opts
+      verify_verb_arguments opts, [:verb, :identifier, :metadata_prefix]
+      return GetRecordResponse.new(do_request(opts))
+    end
+
+    # Equivalent to the ListRecords request. A ListRecordsResponse
+    # will be returned which you can use to iterate through records
+    #
+    #   for record in client.list_records
+    #     puts record.metadata
+    #   end
+    
+    def list_records(opts={})
+      opts[:verb] = 'ListRecords'
+      add_default_metadata_prefix opts
+      verify_verb_arguments opts, [:verb, :from, :until, :set, 
+        :resumption_token, :metadata_prefix]
+      return ListRecordsResponse.new(do_request(opts))
+    end
+
+    # Equivalent to the ListSets request. A ListSetsResponse object
+    # will be returned which you can use for iterating through the 
+    # OAI::Set objects
+    #
+    #   for set in client.list_sets
+    #     puts set
+    #   end
+    
+    def list_sets(opts={})
+      opts[:verb] = 'ListSets'
+      verify_verb_arguments opts, [:verb, :resumptionToken]
+      return ListSetsResponse.new(do_request(opts))
+    end
+
+    private 
+
+    def do_request(hash)
+      uri = @base.clone
+
+      # build up the query string
+      parts = hash.entries.map do |entry|
+        key = studly(entry[0].to_s)
+        value = entry[1]
+        # dates get stringified using ISO8601, strings are url encoded
+        value = case value
+          when DateTime then value.strftime('%Y-%m-%dT%H:%M:%SZ'); 
+          when Date then value.strftime('%Y-%m-%d')
+          else CGI.escape(entry[1].to_s)
+        end
+        "#{key}=#{value}"
+      end
+      uri.query = parts.join('&')
+
+      # fire off the request and return an REXML::Document object
+      begin
+        xml = Net::HTTP.get(uri)
+        return REXML::Document.new(xml)
+      rescue
+        raise OAI::Exception, 'error during oai operation', caller
+      end
+    end
+
+    # convert foo_bar to fooBar thus allowing our ruby code to use
+    # the typical underscore idiom
+    def studly(s)
+      s.gsub(/_(\w)/) do |match|
+        match.sub! '_', ''
+        match.upcase
+      end
+    end
+
+    # add a metadata prefix unless it's there or we are working with 
+    # a resumption token, and having one added could cause problems
+    def add_default_metadata_prefix(opts)
+      unless opts.has_key? :metadata_prefix or opts.has_key? :resumption_token
+        opts[:metadata_prefix] = 'oai_dc'
+      end
+    end
+
+    def verify_verb_arguments(opts, valid_opts)
+      opts.keys.each do |opt|
+        if opt =~ /[A-Z]/
+          raise OAI::Exception.new("#{opt} should use underscores")
+        elsif not valid_opts.include? opt 
+          raise OAI::Exception.new("invalid option #{opt} in #{opts['verb']}")
+        end
+      end
+    end
+  end
+end

data/lib/oai/exception.rb

@@ -0,0 +1,4 @@
+module OAI
+  class Exception < RuntimeError
+  end
+end

data/lib/oai/get_record.rb

@@ -0,0 +1,11 @@
+module OAI
+  class GetRecordResponse < Response
+    include OAI::XPath
+    attr_accessor :record
+
+    def initialize(doc)
+      super doc
+      @record = OAI::Record.new(xpath_first(doc, './/record'))
+    end
+  end
+end

data/lib/oai/header.rb

@@ -0,0 +1,12 @@
+module OAI
+  class Header
+    include OAI::XPath
+    attr_accessor :identifier, :datestamp, :set_spec
+
+    def initialize(element)
+      @identifier = xpath(element, './/identifier')
+      @datestamp = xpath(element, './/datestamp')
+      @set_spec = xpath(element, './/setSpec')
+    end
+  end
+end

data/lib/oai/identify.rb

@@ -0,0 +1,28 @@
+module OAI
+  class IdentifyResponse < Response
+    include OAI::XPath
+    attr_accessor :repository_name, :base_url, :protocol, :admin_email, 
+      :earliest_datestamp, :deleted_record, :granularity, :compression
+
+    def initialize(doc)
+      super doc
+      @repository_name = xpath(doc, './/Identify/repositoryName')
+      @base_url = xpath(doc, './/Identify/baseURL')
+      @protocol = xpath(doc, './/Identify/protocol')
+      @admin_email = xpath(doc, './/Identify/adminEmail')
+      @earliest_datestamp = xpath(doc, './/Identify/earliestDatestamp')
+      @deleted_record = xpath(doc, './/Identify/deletedRecord')
+      @granularity = xpath(doc, './/Identify/granularity')
+      @compression = xpath(doc, '..//Identify/compression')
+    end
+
+    def to_s
+      return "#{@repository_name} [#{@base_url}]"
+    end
+
+    # returns REXML::Element nodes for each description section
+    def descriptions
+      return xpath_all(doc, './/Identify/description')
+    end
+  end
+end

data/lib/oai/list_identifiers.rb

@@ -0,0 +1,12 @@
+module OAI
+  class ListIdentifiersResponse < Response
+    include Enumerable
+    include OAI::XPath
+
+    def each
+      for header_element in xpath_all(@doc, './/ListIdentifiers/header')
+        yield OAI::Header.new(header_element)
+      end
+    end
+  end
+end

data/lib/oai/list_metadata_formats.rb

@@ -0,0 +1,12 @@
+module OAI
+  class ListMetadataFormatsResponse < Response
+    include Enumerable
+    include OAI::XPath
+
+    def each 
+      for format in xpath_all(@doc, './/metadataFormat')
+        yield MetadataFormat.new(format)
+      end
+    end
+  end
+end

data/lib/oai/list_records.rb

@@ -0,0 +1,21 @@
+module OAI
+
+  # allows for iteration across a list of records
+  #
+  #   for record in client.list_records :metadata_prefix => 'oai_dc':
+  #     puts record.metadata
+  #   end
+  #
+  # you'll need to handle resumption tokens
+  
+  class ListRecordsResponse < Response
+    include OAI::XPath
+    include Enumerable
+
+    def each
+      for record_element in xpath_all(@doc, './/ListRecords/record')
+        yield OAI::Record.new(record_element)
+      end
+    end
+  end
+end

data/lib/oai/list_sets.rb

@@ -0,0 +1,19 @@
+module OAI
+
+  # allows for iteration of the sets found in a oai-pmh server
+  #
+  #   for set in client.list_sets
+  #     puts set
+  #   end
+  
+  class ListSetsResponse < Response
+    include OAI::XPath
+    include Enumerable
+
+    def each
+      for set_element in xpath_all(@doc, './/set')
+        yield OAI::Set.new(set_element)
+      end
+    end
+  end
+end

data/lib/oai/metadata_format.rb

@@ -0,0 +1,12 @@
+module OAI
+  class MetadataFormat
+    include OAI::XPath
+    attr_accessor :prefix, :schema, :namespace
+
+    def initialize(element)
+      @prefix = xpath(element, './/metadataPrefix')
+      @schema = xpath(element, './/schema')
+      @namespace = xpath(element, './/metadataNamespace')
+    end
+  end
+end

data/lib/oai/record.rb

@@ -0,0 +1,11 @@
+module OAI
+  class Record
+    include OAI::XPath
+    attr_accessor :header, :metadata
+
+    def initialize(element)
+      @header = OAI::Header.new xpath_first(element, './/header')
+      @metadata = xpath_first(element, './/metadata')
+    end
+  end
+end

data/lib/oai/response.rb

@@ -0,0 +1,20 @@
+module OAI
+  class Response
+    include OAI::XPath
+    attr_reader :doc, :resumption_token
+
+    def initialize(doc)
+      @doc = doc
+      @resumption_token = xpath(doc, './/resumptionToken')
+
+      # throw an exception if there was an error
+      error = xpath_first(doc, './/error')
+      if error
+        message = error.text
+        code = error.attributes['code']
+        raise OAI::Exception.new("#{message} [#{code}]")
+      end
+    end
+
+  end
+end

data/lib/oai/set.rb

@@ -0,0 +1,20 @@
+module OAI
+  
+  # bundles up information about a set retrieved during a 
+  # ListSets request
+  
+  class Set
+    include OAI::XPath
+    attr_accessor :name, :spec, :description
+
+    def initialize(element)
+      @name = xpath(element, './/setName')
+      @spec = xpath(element, './/setSpec')
+      @description = xpath_first(element, './/setDescription')
+    end
+
+    def to_s
+      "#{@name} [#{@spec}]"
+    end
+  end
+end

data/lib/oai/xpath.rb

@@ -0,0 +1,21 @@
+require 'rexml/xpath'
+
+module OAI
+  module XPath
+    def xpath_all(doc, path)
+      return REXML::XPath.match(doc, path)
+    end
+
+    def xpath_first(doc, path)
+      elements = xpath_all(doc, path)
+      return elements[0] if elements != nil
+      return nil
+    end
+
+    def xpath(doc, path)
+      e = xpath_first(doc, path)
+      return e.text if e != nil
+      return nil
+    end
+  end
+end

data/test.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift 'lib'
+
+require 'test/unit'
+require 'oai'
+require 'test/tc_list_identifiers'
+require 'test/tc_list_metadata_formats'
+require 'test/tc_identify'
+require 'test/tc_get_record'
+require 'test/tc_list_records'
+require 'test/tc_list_sets'

data/test/tc_get_record.rb

@@ -0,0 +1,27 @@
+class GetRecordTest < Test::Unit::TestCase
+  def test_get_one
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    response = client.get_record :identifier => 'oai:pubmedcentral.gov:13901'
+    assert_kind_of OAI::GetRecordResponse, response
+    assert_kind_of OAI::Record, response.record
+    assert_kind_of REXML::Element, response.record.metadata
+    assert_kind_of OAI::Header, response.record.header
+
+    # minimal check that the header is working
+    assert_equal 'oai:pubmedcentral.gov:13901', 
+      response.record.header.identifier
+
+    # minimal check that the metadata is working
+    assert 'en', response.record.metadata.elements['.//dc:language'].text
+  end
+
+  def test_missing_identifier
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    begin
+      client.get_record :metadata_prefix => 'oai_dc'
+      flunk 'invalid get_record did not throw OAI::Exception'
+    rescue OAI::Exception => e
+      assert_match /The request includes illegal arguments/, e.to_s
+    end
+  end
+end

data/test/tc_identify.rb

@@ -0,0 +1,8 @@
+class IdentifyTest < Test::Unit::TestCase
+  def test_ok
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    response = client.identify
+    assert_kind_of OAI::IdentifyResponse, response
+    assert_equal 'PubMed Central (PMC3 - NLM DTD) [http://www.pubmedcentral.nih.gov:80/oai/oai.cgi]', response.to_s
+  end
+end

data/test/tc_list_identifiers.rb

@@ -0,0 +1,52 @@
+require 'date'
+
+class ListIdentifiersTest < Test::Unit::TestCase
+
+  def test_list_with_resumption_token
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi' 
+
+    # get a list of identifier headers
+    response = client.list_identifiers :metadata_prefix => 'oai_dc' 
+    assert_kind_of OAI::ListIdentifiersResponse, response
+    assert_kind_of OAI::Response, response
+    assert response.entries.size > 0
+
+    # make sure header is put together reasonably
+    header = response.entries[0]
+    assert_kind_of OAI::Header, header
+    assert header.identifier
+    assert header.datestamp
+    assert header.set_spec
+
+    # exercise a resumption token and make sure first identifier is different
+    first_identifier = response.entries[0].identifier
+    token = response.resumption_token
+    assert_not_nil token
+    response = client.list_identifiers :resumption_token => token
+    assert response.entries.size > 0
+    assert_not_equal response.entries[0].identifier, first_identifier
+  end
+
+  def test_list_with_date_range
+    client = OAI::Client.new 'http://alcme.oclc.org/xtcat/servlet/OAIHandler'
+    from_date = Date.new(2001,1,1)
+    until_date  = Date.new(2006,1,1)
+    response = client.list_identifiers :from => from_date, :until => until_date
+    assert response.entries.size > 0
+  end
+
+  def test_list_with_datetime_range
+    # xtcat should support higher granularity
+    client = OAI::Client.new 'http://alcme.oclc.org/xtcat/servlet/OAIHandler'
+    from_date = DateTime.new(2001,1,1)
+    until_date = DateTime.now
+    response = client.list_identifiers :from => from_date, :until => until_date
+    assert response.entries.size > 0
+  end
+
+  def test_invalid_argument
+    client = OAI::Client.new 'http://arXiv.org/oai2'
+    assert_raise(OAI::Exception) {client.list_identifiers :foo => 'bar'}
+  end
+
+end

data/test/tc_list_metadata_formats.rb

@@ -0,0 +1,15 @@
+class ListMetadataFormatsTest < Test::Unit::TestCase
+  def test_list
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi' 
+    response = client.list_metadata_formats
+    assert_kind_of OAI::ListMetadataFormatsResponse, response
+    assert response.entries.size > 0
+
+    format = response.entries[0]
+    assert_kind_of OAI::MetadataFormat, format
+    assert_equal 'oai_dc', format.prefix
+    assert_equal 'http://www.openarchives.org/OAI/2.0/oai_dc.xsd', format.schema
+    assert_equal 'http://www.openarchives.org/OAI/2.0/oai_dc/', format.namespace
+  end
+end
+

data/test/tc_list_records.rb

@@ -0,0 +1,9 @@
+class GetRecordsTest < Test::Unit::TestCase
+  def test_get_records
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    response = client.list_records 
+    assert_kind_of OAI::ListRecordsResponse, response
+    assert response.entries.size > 0
+    assert_kind_of OAI::Record,  response.entries[0]
+  end
+end

data/test/tc_list_sets.rb

@@ -0,0 +1,17 @@
+class ListSetsTest < Test::Unit::TestCase
+
+  def test_list
+    client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    response = client.list_sets
+    assert_kind_of OAI::ListSetsResponse, response
+    assert response.entries.size > 0
+    assert_kind_of OAI::Set,  response.entries[0]
+  
+    # test iterator
+    for set in response
+      assert_kind_of OAI::Set, set
+    end
+  end
+
+end
+

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: oai
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+date: 2006-04-20 00:00:00 -05:00
+summary: A ruby library for working with the Open Archive Initiative Protocol for Metadata Harvesting (OAI-PMH)
+require_paths: 
+  - lib
+email: ehs@pobox.com
+homepage: http://www.textualize.com/ruby-marc
+rubyforge_project: 
+description: 
+autorequire: oai
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+  - Ed Summers
+files: 
+  - lib/oai
+  - lib/oai.rb
+  - lib/oai/client.rb
+  - lib/oai/exception.rb
+  - lib/oai/get_record.rb
+  - lib/oai/header.rb
+  - lib/oai/identify.rb
+  - lib/oai/list_identifiers.rb
+  - lib/oai/list_metadata_formats.rb
+  - lib/oai/list_records.rb
+  - lib/oai/list_sets.rb
+  - lib/oai/metadata_format.rb
+  - lib/oai/record.rb
+  - lib/oai/response.rb
+  - lib/oai/set.rb
+  - lib/oai/xpath.rb
+  - test/tc_get_record.rb
+  - test/tc_identify.rb
+  - test/tc_list_identifiers.rb
+  - test/tc_list_metadata_formats.rb
+  - test/tc_list_records.rb
+  - test/tc_list_sets.rb
+test_files: 
+  - test.rb
+rdoc_options: []
+extra_rdoc_files: []
+executables: []
+extensions: []
+requirements: []
+dependencies: []