oai 0.0.1 → 1.3.0

data/lib/oai/client.rb

@@ -1,163 +1,368 @@
+# encoding: us-ascii
+
+# External dependencies
 require 'uri'
-require 'net/http'
-require 'rexml/document'
+require 'faraday'
 require 'cgi'
 
+if not defined?(OAI::Const::VERBS)
+  # Shared stuff
+  require 'oai/exception'
+  require 'oai/constants'
+  require 'oai/xpath'
+  require 'oai/set'
+end
+
+# Localize requires so user can select a subset of functionality
+require 'oai/client/metadata_format'
+require 'oai/client/response'
+require 'oai/client/header'
+require 'oai/client/record'
+require 'oai/client/identify'
+require 'oai/client/get_record'
+require 'oai/client/resumable'
+require 'oai/client/list_identifiers'
+require 'oai/client/list_metadata_formats'
+require 'oai/client/list_records'
+require 'oai/client/list_sets'
+
 module OAI
 
-  # A OAI::Client provides a client api for issuing OAI-PMH verbs against
-  # a OAI-PMH server. The 6 OAI-PMH verbs translate directly to methods you 
-  # can call on a OAI::Client object. Verb arguments are passed as a hash:
+  # A `OAI::Client` provides a client api for issuing OAI-PMH verbs against
+  # a OAI-PMH server. The 6 OAI-PMH verbs translate directly to methods you
+  # can call on a `OAI::Client` object. Verb arguments are passed as a hash:
   #
-  #   client = OAI::Harvester.new ''http://www.pubmedcentral.gov/oai/oai.cgi'
-  #   client.list_identifiers :metadata_prefix => 'oai_dc'
+  # ```ruby
+  #   client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+  #   record = client.get_record :identifier => 'oai:pubmedcentral.gov:13901'
+  #   for identifier in client.list_identifiers
+  #     puts identifier
+  #   end
+  # ```
   #
-  # It is worth noting that the api uses methods and parameter names with 
-  # underscores in them rather than studly caps. So above list_identifiers 
-  # and metadata_prefix are used instead of the listIdentifiers and 
-  # metadataPrefix used in the OAI-PMH specification.
+  # It is worth noting that the API uses methods and parameter names with
+  # underscores in them rather than studly caps. So above `list_identifiers`
+  # and `metadata_prefix` are used instead of the `listIdentifiers` and
+  # `metadataPrefix` used in the OAI-PMH specification.
   #
   # Also, the from and until arguments which specify dates should be passed
-  # in as Date or DateTime objects depending on the granularity supported
+  # in as `Date` or `DateTime` objects depending on the granularity supported
   # by the server.
   #
   # For detailed information on the arguments that can be used please consult
-  # the OAI-PMH docs at:
-  #
-  #     http://www.openarchives.org/OAI/openarchivesprotocol.html
-  
-  class Client
+  # the OAI-PMH docs at
+  # <http://www.openarchives.org/OAI/openarchivesprotocol.html>.
 
-    # The constructor which must be passed a valid base url for an oai 
+  class Client
+    UNESCAPED_AMPERSAND = /&(?!(?:amp|lt|gt|quot|apos|\#\d+);)/
+    # The constructor which must be passed a valid base url for an oai
     # service:
     #
-    #   client = OAI::Harvseter.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
-    
-    def initialize(base_url)
+    #     client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    #
+    # If you want to see debugging messages on `STDERR` use:
+    #
+    #     client = OAI::Client.new 'http://example.com', :debug => true
+    #
+    # By default OAI verbs called on the client will return `REXML::Element`
+    # objects for metadata records, however if you wish you can use the
+    # `:parser` option to indicate you want to use `libxml` instead, and get
+    # back `XML::Node` objects
+    #
+    #     client = OAI::Client.new 'http://example.com', :parser => 'libxml'
+    #
+    # You can configure the Faraday HTTP client by providing an alternate
+    # Faraday instance:
+    #
+    # ```ruby
+    # client = OAI::Client.new 'http://example.com', :http => Faraday.new {|c|}
+    # ```
+    #
+    # ### HIGH PERFORMANCE
+    #
+    # If you want to supercharge this api install `libxml-ruby >= 0.3.8` and
+    # use the `:parser` option when you construct your `OAI::Client`.
+    #
+    def initialize(base_url, options={})
       @base = URI.parse base_url
+      @debug = options.fetch(:debug, false)
+      @parser = options.fetch(:parser, 'rexml')
+      @headers = options.fetch(:headers, {})
+
+      @http_client = options.fetch(:http) do
+        Faraday.new(:url => @base.clone) do |builder|
+          follow_redirects = options.fetch(:redirects, true)
+          follow_redirects = 5 if follow_redirects == true
+
+          if follow_redirects
+            require 'faraday/follow_redirects'
+            builder.use Faraday::FollowRedirects::Middleware
+            builder.response :follow_redirects, :limit => follow_redirects.to_i
+          end
+          builder.adapter :net_http
+        end
+      end
+
+      # load appropriate parser
+      case @parser
+      when 'libxml'
+        begin
+          require 'rubygems'
+          require 'xml/libxml'
+        rescue
+          raise OAI::Exception.new("xml/libxml not available")
+        end
+      when 'rexml'
+        require 'rexml/document'
+        require 'rexml/xpath'
+      else
+        raise OAI::Exception.new("unknown parser: #{@parser}")
+      end
     end
 
-    # Equivalent to a Identify request. You'll get back a OAI::IdentifyResponse
-    # object which is essentially just a wrapper around a REXML::Document 
-    # for the response.
-    
+    # Equivalent to a `Identify` request.
+    # You'll get back a `OAI::IdentifyResponse`
+    # object which is essentially just a wrapper around a `REXML::Document`
+    # for the response. If you created your client using the `libxml`
+    # parser then you will get an `XML::Node` object instead.
     def identify
-      return IdentifyResponse.new(do_request(:verb => 'Identify'))
+      OAI::IdentifyResponse.new(do_request('Identify'))
     end
 
-    # Equivalent to a ListMetadataFormats request. A ListMetadataFormatsResponse
-    # object is returned to you. 
-    
+    # Equivalent to a `ListMetadataFormats` request.
+    # A `ListMetadataFormatsResponse` object is returned to you.
+
     def list_metadata_formats(opts={})
-      opts[:verb] = 'ListMetadataFormats'
-      verify_verb_arguments opts, [:verb, :identifier]
-      return ListMetadataFormatsResponse.new(do_request(opts))
+      OAI::ListMetadataFormatsResponse.new(do_request('ListMetadataFormats', opts))
     end
 
-    # Equivalent to a ListIdentifiers request. Pass in :from, :until arguments
-    # as Date or DateTime objects as appropriate depending on the granularity 
-    # supported by the server.
-    
+    # Equivalent to a `ListIdentifiers` request. Pass in `:from`,
+    # `:until` arguments as `Date` or `DateTime` objects as appropriate
+    # depending on the granularity supported by the server.
+    #
+    # You can use seamless resumption with this verb, which allows you to
+    # mitigate (to some extent) the lack of a `Count` verb:
+    #
+    #     client.list_identifiers.full.count # Don't try this on PubMed though!
+    #
     def list_identifiers(opts={})
-      opts[:verb] = 'ListIdentifiers'
-      add_default_metadata_prefix opts
-      verify_verb_arguments opts, [:verb, :from, :until, :metadata_prefix, :set,         :resumption_token]
-      return ListIdentifiersResponse.new(do_request(opts)) 
+      do_resumable(OAI::ListIdentifiersResponse, 'ListIdentifiers', opts)
     end
 
-    # Equivalent to a GetRecord request. You must supply an identifier 
-    # argument. You should get back a OAI::GetRecordResponse object
-    # which you can extract a OAI::Record object from.
-    
+    # Equivalent to a `GetRecord` request. You must supply an `:identifier`
+    # argument. You should get back a `OAI::GetRecordResponse` object
+    # which you can extract a `OAI::Record` object from.
     def get_record(opts={})
-      opts[:verb] = 'GetRecord'
-      add_default_metadata_prefix opts
-      verify_verb_arguments opts, [:verb, :identifier, :metadata_prefix]
-      return GetRecordResponse.new(do_request(opts))
+      OAI::GetRecordResponse.new(do_request('GetRecord', opts))
     end
 
-    # Equivalent to the ListRecords request. A ListRecordsResponse
+    # Equivalent to the `ListRecords` request. A `ListRecordsResponse`
     # will be returned which you can use to iterate through records
     #
-    #   for record in client.list_records
-    #     puts record.metadata
-    #   end
-    
+    #     response = client.list_records
+    #     response.each do |record|
+    #       puts record.metadata
+    #     end
+    #
+    # Alternately, you can use seamless resumption to avoid handling
+    # resumption tokens:
+    #
+    #     client.list_records.full.each do |record|
+    #       puts record.metadata
+    #     end
+    #
+    # ### Memory Use
+    # `:full` will avoid storing more than one page of records in
+    # memory, but your use it in ways that override that behaviour. Be careful
+    # to avoid using `client.list_records.full.entries` unless you really want
+    # to hold all the records in the feed in memory!
     def list_records(opts={})
-      opts[:verb] = 'ListRecords'
-      add_default_metadata_prefix opts
-      verify_verb_arguments opts, [:verb, :from, :until, :set, 
-        :resumption_token, :metadata_prefix]
-      return ListRecordsResponse.new(do_request(opts))
+      do_resumable(OAI::ListRecordsResponse, 'ListRecords', opts)
     end
 
-    # Equivalent to the ListSets request. A ListSetsResponse object
-    # will be returned which you can use for iterating through the 
-    # OAI::Set objects
+    # Equivalent to the `ListSets` request. A `ListSetsResponse` object
+    # will be returned which you can use for iterating through the
+    # `OAI::Set` objects
+    #
+    #     for set in client.list_sets
+    #       puts set
+    #     end
     #
-    #   for set in client.list_sets
-    #     puts set
-    #   end
-    
+    # A large number of sets is not unusual for some OAI-PMH feeds, so
+    # using seamless resumption may be preferable:
+    #
+    #     client.list_sets.full.each do |set|
+    #       puts set
+    #     end
     def list_sets(opts={})
-      opts[:verb] = 'ListSets'
-      verify_verb_arguments opts, [:verb, :resumptionToken]
-      return ListSetsResponse.new(do_request(opts))
+      do_resumable(OAI::ListSetsResponse, 'ListSets', opts)
     end
 
-    private 
+    def sanitize_xml(xml)
+      xml = strip_invalid_utf_8_chars(xml)
+      xml = strip_invalid_xml_chars(xml)
+      if @parser == 'libxml'
+        # remove default namespace for oai-pmh since libxml
+        # isn't able to use our xpaths to get at them
+        # if you know a way around thins please let me know
+        xml = xml.gsub(
+          /xmlns=\"http:\/\/www.openarchives.org\/OAI\/.\..\/\"/, '')
+      end
+      xml
+    end
 
-    def do_request(hash)
-      uri = @base.clone
+    private
 
-      # build up the query string
-      parts = hash.entries.map do |entry|
-        key = studly(entry[0].to_s)
-        value = entry[1]
-        # dates get stringified using ISO8601, strings are url encoded
-        value = case value
-          when DateTime then value.strftime('%Y-%m-%dT%H:%M:%SZ'); 
-          when Date then value.strftime('%Y-%m-%d')
-          else CGI.escape(entry[1].to_s)
-        end
-        "#{key}=#{value}"
+    def do_request(verb, opts = nil)
+      # fire off the request and return appropriate DOM object
+      uri = build_uri(verb, opts)
+      return load_document(get(uri))
+    end
+
+    def do_resumable(responseClass, verb, opts)
+      responseClass.new(do_request(verb, opts)) do |response|
+        responseClass.new \
+          do_request(verb, :resumption_token => response.resumption_token)
       end
-      uri.query = parts.join('&')
-
-      # fire off the request and return an REXML::Document object
-      begin
-        xml = Net::HTTP.get(uri)
-        return REXML::Document.new(xml)
-      rescue
-        raise OAI::Exception, 'error during oai operation', caller
+    end
+
+    def build_uri(verb, opts)
+      opts = validate_options(verb, opts)
+      uri = @base
+      uri.query = "verb=" << verb
+      opts.each_pair { |k,v| uri.query << '&' << externalize(k) << '=' << encode(v) }
+      uri
+    end
+
+    def encode(value)
+      return CGI.escape(value) unless value.respond_to?(:strftime)
+      if value.kind_of?(DateTime)
+        Time.parse(value.asctime).utc.xmlschema
+      elsif value.kind_of?(Time)
+        value.utc.xmlschema
+      else # Assume something date like
+        value.strftime('%Y-%m-%d')
       end
     end
 
-    # convert foo_bar to fooBar thus allowing our ruby code to use
-    # the typical underscore idiom
-    def studly(s)
-      s.gsub(/_(\w)/) do |match|
-        match.sub! '_', ''
-        match.upcase
+    def load_document(xml)
+      xml = sanitize_xml(xml)
+      case @parser
+      when 'libxml'
+        begin
+          parser = XML::Parser.string(xml)
+          return parser.parse
+        rescue XML::Error => e
+          raise OAI::Exception, 'response not well formed XML: '+e, caller
+        end
+      when 'rexml'
+        begin
+          return REXML::Document.new(xml)
+        rescue REXML::ParseException => e
+          raise OAI::Exception, 'response not well formed XML: '+e.message, caller
+        end
       end
     end
 
-    # add a metadata prefix unless it's there or we are working with 
-    # a resumption token, and having one added could cause problems
-    def add_default_metadata_prefix(opts)
-      unless opts.has_key? :metadata_prefix or opts.has_key? :resumption_token
-        opts[:metadata_prefix] = 'oai_dc'
+    # Do the actual HTTP get, following any temporary redirects
+    def get(uri)
+      response = @http_client.get do |req|
+        req.url uri
+        req.headers.merge! @headers
       end
+
+      response.body
     end
 
-    def verify_verb_arguments(opts, valid_opts)
-      opts.keys.each do |opt|
-        if opt =~ /[A-Z]/
-          raise OAI::Exception.new("#{opt} should use underscores")
-        elsif not valid_opts.include? opt 
-          raise OAI::Exception.new("invalid option #{opt} in #{opts['verb']}")
-        end
+    def debug(msg)
+      $stderr.print("#{msg}\n") if @debug
+    end
+
+    # Massage the standard OAI options to make them a bit more palatable.
+    def validate_options(verb, opts = {})
+      raise OAI::VerbException.new unless Const::VERBS.keys.include?(verb)
+
+      return {} if opts.nil?
+
+      raise OAI::ArgumentException.new unless opts.respond_to?(:keys)
+
+      realopts = {}
+      # Internalize the hash
+      opts.keys.each do |key|
+        realopts[key.to_s.gsub(/([A-Z])/, '_\1').downcase.intern] = opts.delete(key)
+      end
+
+      return realopts if is_resumption?(realopts)
+
+      # add in a default metadataPrefix if none exists
+      if(Const::VERBS[verb].include?(:metadata_prefix))
+        realopts[:metadata_prefix] ||= 'oai_dc'
+      end
+
+      # Convert date formated strings in dates.
+      #realopts[:from] = parse_date(realopts[:from]) if realopts[:from]
+      #realopts[:until] = parse_date(realopts[:until]) if realopts[:until]
+
+      # check for any bad options
+      unless (realopts.keys - OAI::Const::VERBS[verb]).empty?
+        raise OAI::ArgumentException.new
       end
+      realopts
+    end
+
+    def is_resumption?(opts)
+      if opts.keys.include?(:resumption_token)
+        return true if 1 == opts.keys.size
+        raise OAI::ArgumentException.new
+      end
+    end
+
+    # Convert our internal representations back into standard OAI options
+    def externalize(value)
+      value.to_s.gsub(/_[a-z]/) { |m| m.sub("_", '').capitalize }
+    end
+
+    def parse_date(value)
+      return value if value.respond_to?(:strftime)
+
+      Date.parse(value) # This will raise an exception for badly formatted dates
+      Time.parse(value).utc # Sadly, this will not
+    rescue
+      raise OAI::ArgumentException.new, "unparsable date: '#{value}'"
+    end
+
+    # Strip out invalid UTF-8 characters.  Regex from the W3C, inverted.
+    # http://www.w3.org/International/questions/qa-forms-utf-8.en.php
+    #
+    # Regex is from WebCollab:
+    #   http://webcollab.sourceforge.net/unicode.html
+    def strip_invalid_utf_8_chars(xml)
+      return nil unless xml
+
+      # If it's in a specific encoding other than BINARY, it may trigger
+      # an exception to try to gsub these illegal bytes. Temporarily
+      # put it in BINARY. NOTE: We're not totally sure what's going on
+      # with encodings in this gem in general, it might not be totally reasonable.
+      orig_encoding = xml.encoding
+      xml.force_encoding("BINARY")
+
+      xml = xml.gsub(/[\x00-\x08\x10\x0B\x0C\x0E-\x19\x7F]
+                             | [\x00-\x7F][\x80-\xBF]+
+                             | ([\xC0\xC1]|[\xF0-\xFF])[\x80-\xBF]*
+                             | [\xC2-\xDF]((?![\x80-\xBF])|[\x80-\xBF]{2,})
+                             | [\xE0-\xEF](([\x80-\xBF](?![\x80-\xBF]))
+                             | (?![\x80-\xBF]{2})|[\x80-\xBF]{3,})/x, '?')\
+                .gsub(/\xE0[\x80-\x9F][\x80-\xBF]
+                       | \xED[\xA0-\xBF][\x80-\xBF]/,'?')
+
+      xml.force_encoding(orig_encoding)
+
+      xml
+    end
+
+    def strip_invalid_xml_chars(xml)
+      return xml unless xml =~ UNESCAPED_AMPERSAND
+      xml.gsub(UNESCAPED_AMPERSAND, '&amp;')
     end
   end
 end

data/lib/oai/constants.rb

@@ -0,0 +1,34 @@
+module OAI
+
+  module Const
+    # OAI defines six verbs with various allowable options.
+    VERBS = {
+      'Identify' => [],
+      'ListMetadataFormats' => [:identifier],
+      'ListSets' => [:resumption_token],  # unused currently
+      'GetRecord' => [:identifier, :from, :until, :set, :metadata_prefix],
+      'ListIdentifiers' => [:from, :until, :set, :metadata_prefix, :resumption_token],
+      'ListRecords' => [:from, :until, :set, :metadata_prefix, :resumption_token]
+    }.freeze
+    
+    RESERVED_WORDS = %w{type id}
+    
+    # Two granularities are supported in OIA-PMH, daily or seconds. 
+    module Granularity
+      LOW = 'YYYY-MM-DD'
+      HIGH = 'YYYY-MM-DDThh:mm:ssZ'
+    end
+    
+    # Repositories can support three different schemes for dealing with deletions.
+    # * NO - No deletions allowed
+    # * TRANSIENT - Deletions are supported but may not be permanently maintained.
+    # * PERSISTENT - Deletions are supported and are permanently maintained.
+    module Delete
+      NO = :no
+      TRANSIENT = :transient
+      PERSISTENT = :persistent
+    end
+        
+  end
+  
+end

data/lib/oai/exception.rb

@@ -1,4 +1,83 @@
 module OAI
+
+  # Standard error responses for problems serving OAI content.  These
+  # messages will be wrapped in an XML response to the client.
+
   class Exception < RuntimeError
+    CODE = nil
+    MESSAGE = nil
+
+    attr_reader :code
+
+    @@codes = {}
+
+    def self.register_exception_code(code, exception_class)
+      @@codes[code] = exception_class if exception_class.superclass == OAI::Exception
+    end
+
+    def self.for(message: nil, code: nil)
+      @@codes.fetch(code, Exception).new(message)
+    end
+
+    def initialize(message = nil, code = nil)
+      super(message || self.class::MESSAGE)
+      @code = code || self.class::CODE
+    end
   end
-end
+
+  class ArgumentException < Exception
+    CODE = 'badArgument'
+    MESSAGE = 'The request includes ' \
+      'illegal arguments, is missing required arguments, includes a ' \
+      'repeated argument, or values for arguments have an illegal syntax.'
+    register_exception_code(CODE, self)
+  end
+
+  class VerbException < Exception
+    CODE = 'badVerb'
+    MESSAGE = 'Value of the verb argument is not a legal OAI-PMH '\
+      'verb, the verb argument is missing, or the verb argument is repeated.'
+    register_exception_code(CODE, self)
+  end
+
+  class FormatException < Exception
+    CODE = 'cannotDisseminateFormat'
+    MESSAGE = 'The metadata format identified by '\
+        'the value given for the metadataPrefix argument is not supported '\
+        'by the item or by the repository.'
+    register_exception_code(CODE, self)
+  end
+
+  class IdException < Exception
+    CODE = 'idDoesNotExist'
+    MESSAGE = 'The value of the identifier argument is '\
+        'unknown or illegal in this repository.'
+    register_exception_code(CODE, self)
+  end
+
+  class NoMatchException < Exception
+    CODE = 'noRecordsMatch'
+    MESSAGE = 'The combination of the values of the from, '\
+      'until, set and metadataPrefix arguments results in an empty list.'
+    register_exception_code(CODE, self)
+  end
+
+  class MetadataFormatException < Exception
+    CODE = 'noMetadataFormats'
+    MESSAGE = 'There are no metadata formats available '\
+        'for the specified item.'
+    register_exception_code(CODE, self)
+  end
+
+  class SetException < Exception
+    CODE = 'noSetHierarchy'
+    MESSAGE = 'This repository does not support sets.'
+    register_exception_code(CODE, self)
+  end
+
+  class ResumptionTokenException < Exception
+    CODE = 'badResumptionToken'
+    MESSAGE = 'The value of the resumptionToken argument is invalid or expired.'
+    register_exception_code(CODE, self)
+  end
+end

data/lib/oai/harvester/config.rb

@@ -0,0 +1,41 @@
+#
+#  Created by William Groppe on 2006-11-05.
+#  Copyright (c) 2006. All rights reserved.
+
+module OAI
+  module Harvester
+
+    LOW_RESOLUTION = OAI::Const::Granularity::LOW
+  
+    class Config < OpenStruct
+    
+      PERIODS = %w(daily weekly monthly)
+      GLOBAL = "/etc/oai/harvester.yml"
+    
+      def self.load
+        config = find_config
+        File.exists?(config) ? new(YAML.load_file(config)) : new
+      end
+
+      def save
+        config = Config.find_config
+        open(config, 'w') do |out|
+          YAML.dump(@table, out)
+        end
+      end
+
+      private 
+      # Shamelessly lifted from Camping
+      def self.find_config
+        if home = ENV['HOME'] # POSIX
+          return GLOBAL if File.exists?(GLOBAL) && File.writable?(GLOBAL)
+          FileUtils.mkdir_p File.join(home, '.oai')
+          File.join(home, '.oai/harvester.yml')
+        elsif home = ENV['APPDATA'] # MSWIN
+          File.join(home, 'oai/harvester.yml')
+        end
+      end
+    
+    end
+  end
+end