oai_talia 0.0.13

data/lib/oai/client.rb

@@ -0,0 +1,301 @@
+# External dependencies
+require 'uri'
+require 'net/http'
+require 'cgi'
+require 'iconv'
+
+if not defined?(OAI::Const::VERBS)
+  # Shared stuff
+  require 'oai/exception'
+  require 'oai/constants'
+  require 'oai/xpath'
+  require 'oai/set'
+end
+
+# Localize requires so user can select a subset of functionality
+require 'oai/client/metadata_format'
+require 'oai/client/response'
+require 'oai/client/header'
+require 'oai/client/record' 
+require 'oai/client/identify'
+require 'oai/client/get_record'
+require 'oai/client/list_identifiers'
+require 'oai/client/list_metadata_formats'
+require 'oai/client/list_records'
+require 'oai/client/list_sets'
+
+module OAI
+
+  # A OAI::Client provides a client api for issuing OAI-PMH verbs against
+  # a OAI-PMH server. The 6 OAI-PMH verbs translate directly to methods you 
+  # can call on a OAI::Client object. Verb arguments are passed as a hash:
+  #
+  #   client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+  #   record = client.get_record :identifier => 'oai:pubmedcentral.gov:13901'
+  #   for identifier in client.list_identifiers
+  #     puts identifier
+  #   end
+  #
+  # It is worth noting that the api uses methods and parameter names with 
+  # underscores in them rather than studly caps. So above list_identifiers 
+  # and metadata_prefix are used instead of the listIdentifiers and 
+  # metadataPrefix used in the OAI-PMH specification.
+  #
+  # Also, the from and until arguments which specify dates should be passed
+  # in as Date or DateTime objects depending on the granularity supported
+  # by the server.
+  #
+  # For detailed information on the arguments that can be used please consult
+  # the OAI-PMH docs at:
+  #
+  #     http://www.openarchives.org/OAI/openarchivesprotocol.html
+  
+  class Client
+
+    # The constructor which must be passed a valid base url for an oai 
+    # service:
+    #
+    #   client = OAI::Client.new 'http://www.pubmedcentral.gov/oai/oai.cgi'
+    #
+    # If you want to see debugging messages on STDERR use:
+    #
+    #   client = OAI::Client.new 'http://example.com', :debug => true
+    #
+    # By default OAI verbs called on the client will return REXML::Element
+    # objects for metadata records, however if you wish you can use the
+    # :parser option to indicate you want to use 'libxml' instead, and get
+    # back XML::Node objects
+    #
+    #   client = OAI::Client.new 'http://example.com', :parser => 'libxml'
+    #
+    # === HIGH PERFORMANCE
+    #
+    # If you want to supercharge this api install libxml-ruby >= 0.3.8 and 
+    # use the :parser option when you construct your OAI::Client.
+    #
+    def initialize(base_url, options={})
+      @base = URI.parse base_url
+      @debug = options.fetch(:debug, false)
+      @parser = options.fetch(:parser, 'rexml')
+      @follow_redirects = options.fetch(:redirects, true)
+      
+      # load appropriate parser
+      case @parser
+      when 'libxml'
+        begin
+          require 'rubygems'
+          require 'xml/libxml'
+        rescue
+          raise OAI::Exception.new("xml/libxml not available")
+        end
+      when 'rexml'
+        require 'rexml/document'
+        require 'rexml/xpath'
+      else
+        raise OAI::Exception.new("unknown parser: #{@parser}")
+      end
+    end
+
+    # Equivalent to a Identify request. You'll get back a OAI::IdentifyResponse
+    # object which is essentially just a wrapper around a REXML::Document 
+    # for the response. If you are created your client using the libxml 
+    # parser then you will get an XML::Node object instead.
+    
+    def identify
+      return OAI::IdentifyResponse.new(do_request('Identify'))
+    end
+
+    # Equivalent to a ListMetadataFormats request. A ListMetadataFormatsResponse
+    # object is returned to you. 
+    
+    def list_metadata_formats(opts={})
+      return OAI::ListMetadataFormatsResponse.new(do_request('ListMetadataFormats', opts))
+    end
+
+    # Equivalent to a ListIdentifiers request. Pass in :from, :until arguments
+    # as Date or DateTime objects as appropriate depending on the granularity 
+    # supported by the server.
+    
+    def list_identifiers(opts={})
+      return OAI::ListIdentifiersResponse.new(do_request('ListIdentifiers', opts)) 
+    end
+
+    # Equivalent to a GetRecord request. You must supply an identifier 
+    # argument. You should get back a OAI::GetRecordResponse object
+    # which you can extract a OAI::Record object from.
+    
+    def get_record(opts={})
+      return OAI::GetRecordResponse.new(do_request('GetRecord', opts))
+    end
+
+    # Equivalent to the ListRecords request. A ListRecordsResponse
+    # will be returned which you can use to iterate through records
+    #
+    #   for record in client.list_records
+    #     puts record.metadata
+    #   end
+    
+    def list_records(opts={})
+      return OAI::ListRecordsResponse.new(do_request('ListRecords', opts))
+    end
+
+    # Equivalent to the ListSets request. A ListSetsResponse object
+    # will be returned which you can use for iterating through the 
+    # OAI::Set objects
+    #
+    #   for set in client.list_sets
+    #     puts set
+    #   end
+    
+    def list_sets(opts={})
+      return OAI::ListSetsResponse.new(do_request('ListSets', opts))
+    end
+
+    private 
+
+    def do_request(verb, opts = nil)
+      # fire off the request and return appropriate DOM object
+      uri = build_uri(verb, opts)
+      xml = strip_invalid_utf_8_chars(get(uri))
+      if @parser == 'libxml' 
+        # remove default namespace for oai-pmh since libxml
+        # isn't able to use our xpaths to get at them 
+        # if you know a way around thins please let me know
+        xml = xml.gsub(
+          /xmlns=\"http:\/\/www.openarchives.org\/OAI\/.\..\/\"/, '') 
+      end
+      return load_document(xml)
+    end
+    
+    def build_uri(verb, opts)
+      opts = validate_options(verb, opts)
+      uri = @base.clone
+      uri.query = "verb=" << verb
+      opts.each_pair { |k,v| uri.query << '&' << externalize(k) << '=' << encode(v) }
+      uri
+    end
+    
+    def encode(value)
+      return CGI.escape(value) unless value.respond_to?(:strftime)
+      if value.kind_of?(DateTime)
+        Time.parse(value.asctime).utc.xmlschema
+      elsif value.kind_of?(Time)
+        value.utc.xmlschema
+      else # Assume something date like
+        value.strftime('%Y-%m-%d')
+      end
+    end
+
+    def load_document(xml)
+      case @parser
+      when 'libxml'
+        begin
+          parser = XML::Parser.new()
+          parser.string = xml
+          return parser.parse
+        rescue XML::Parser::ParseError => e
+          raise OAI::Exception, 'response not well formed XML: '+e, caller
+        end
+      when 'rexml'
+        begin
+          return REXML::Document.new(xml)
+        rescue REXML::ParseException => e
+          raise OAI::Exception, 'response not well formed XML: '+e.message, caller
+        end
+      end
+    end
+
+    # Do the actual HTTP get, following any temporary redirects
+    def get(uri)
+      response = Net::HTTP.get_response(uri)
+      case response
+      when Net::HTTPSuccess
+        return response.body
+      when Net::HTTPMovedPermanently
+        if @follow_redirects
+          response = get(URI.parse(response['location']))
+        else
+          raise ArgumentError, "Permanently Redirected to [#{response['location']}]"
+        end
+      when Net::HTTPTemporaryRedirect, Net::HTTPFound
+        response = get(URI.parse(response['location']))
+      else
+        raise ArgumentError, "#{response.code_type} [#{response.code}]"
+      end
+    end
+
+    def debug(msg)
+      $stderr.print("#{msg}\n") if @debug
+    end
+    
+    # Massage the standard OAI options to make them a bit more palatable.
+    def validate_options(verb, opts = {})
+      raise OAI::VerbException.new unless Const::VERBS.keys.include?(verb)
+
+      return {} if opts.nil?
+
+      raise OAI::ArgumentException.new unless opts.respond_to?(:keys)
+      
+      realopts = {}
+      # Internalize the hash
+      opts.keys.each do |key|
+        realopts[key.to_s.gsub(/([A-Z])/, '_\1').downcase.intern] = opts.delete(key)
+      end
+      
+      return realopts if is_resumption?(realopts)
+      
+      # add in a default metadataPrefix if none exists
+      if(Const::VERBS[verb].include?(:metadata_prefix))
+        realopts[:metadata_prefix] ||= 'oai_dc'
+      end
+      
+      # Convert date formated strings in dates.
+      #realopts[:from] = parse_date(realopts[:from]) if realopts[:from]
+      #realopts[:until] = parse_date(realopts[:until]) if realopts[:until]
+
+      # check for any bad options
+      unless (realopts.keys - OAI::Const::VERBS[verb]).empty?
+        raise OAI::ArgumentException.new
+      end
+      realopts
+    end
+    
+    def is_resumption?(opts)
+      if opts.keys.include?(:resumption_token) 
+        return true if 1 == opts.keys.size
+        raise OAI::ArgumentException.new
+      end
+    end
+        
+    # Convert our internal representations back into standard OAI options
+    def externalize(value)
+      value.to_s.gsub(/_[a-z]/) { |m| m.sub("_", '').capitalize }
+    end
+    
+    def parse_date(value)
+      return value if value.respond_to?(:strftime)
+      
+      Date.parse(value) # This will raise an exception for badly formatted dates
+      Time.parse(value).utc # Sadly, this will not
+    rescue
+      raise OAI::ArgumentError.new 
+    end
+    
+    # Strip out invalid UTF-8 characters.  Regex from the W3C, inverted.
+    # http://www.w3.org/International/questions/qa-forms-utf-8.en.php
+    #
+    # Regex is from WebCollab: 
+    #   http://webcollab.sourceforge.net/unicode.html
+    def strip_invalid_utf_8_chars(xml)
+      simple_bytes = xml.gsub(/[\x00-\x08\x10\x0B\x0C\x0E-\x19\x7F]
+                             | [\x00-\x7F][\x80-\xBF]+
+                             | ([\xC0\xC1]|[\xF0-\xFF])[\x80-\xBF]*
+                             | [\xC2-\xDF]((?![\x80-\xBF])|[\x80-\xBF]{2,})
+                             | [\xE0-\xEF](([\x80-\xBF](?![\x80-\xBF]))
+                             | (?![\x80-\xBF]{2})|[\x80-\xBF]{3,})/x, '?')
+      simple_bytes.gsub(/\xE0[\x80-\x9F][\x80-\xBF]
+                       | \xED[\xA0-\xBF][\x80-\xBF]/,'?')
+    end
+        
+  end
+end

data/lib/oai/constants.rb

@@ -0,0 +1,34 @@
+module OAI
+
+  module Const
+    # OAI defines six verbs with various allowable options.
+    VERBS = {
+      'Identify' => [],
+      'ListMetadataFormats' => [:identifier],
+      'ListSets' => [:resumption_token],  # unused currently
+      'GetRecord' => [:identifier, :from, :until, :set, :metadata_prefix],
+      'ListIdentifiers' => [:from, :until, :set, :metadata_prefix, :resumption_token],
+      'ListRecords' => [:from, :until, :set, :metadata_prefix, :resumption_token]
+    }.freeze
+    
+    RESERVED_WORDS = %w{type id}
+    
+    # Two granularities are supported in OIA-PMH, daily or seconds. 
+    module Granularity
+      LOW = 'YYYY-MM-DD'
+      HIGH = 'YYYY-MM-DDThh:mm:ssZ'
+    end
+    
+    # Repositories can support three different schemes for dealing with deletions.
+    # * NO - No deletions allowed
+    # * TRANSIENT - Deletions are supported but may not be permanently maintained.
+    # * PERSISTENT - Deletions are supported and are permanently maintained.
+    module Delete
+      NO = :no
+      TRANSIENT = :transient
+      PERSISTENT = :persistent
+    end
+        
+  end
+  
+end

data/lib/oai/exception.rb

@@ -0,0 +1,75 @@
+module OAI
+
+  # Standard error responses for problems serving OAI content.  These
+  # messages will be wrapped in an XML response to the client.
+
+  class Exception < RuntimeError
+    attr_reader :code
+
+    def initialize(message, code = nil)
+      super(message)
+      @code = code
+    end
+  end
+
+  class ArgumentException < Exception
+    def initialize()
+      super('The request includes ' \
+      'illegal arguments, is missing required arguments, includes a ' \
+      'repeated argument, or values for arguments have an illegal syntax.',
+      'badArgument')
+    end
+  end
+
+  class VerbException < Exception
+    def initialize()
+      super('Value of the verb argument is not a legal OAI-PMH '\
+      'verb, the verb argument is missing, or the verb argument is repeated.',
+      'badVerb')
+    end
+  end
+
+  class FormatException < Exception
+    def initialize()
+      super('The metadata format identified by '\
+        'the value given for the metadataPrefix argument is not supported '\
+        'by the item or by the repository.', 'cannotDisseminateFormat')
+    end
+  end
+
+  class IdException < Exception
+    def initialize()
+      super('The value of the identifier argument is '\
+        'unknown or illegal in this repository.', 'idDoesNotExist')
+    end
+  end
+
+  class NoMatchException < Exception
+    def initialize()
+      super('The combination of the values of the from, '\
+      'until, set and metadataPrefix arguments results in an empty list.',
+      'noRecordsMatch')
+    end
+  end
+
+  class MetadataFormatException < Exception
+    def initialize()
+      super('There are no metadata formats available '\
+        'for the specified item.', 'noMetadataFormats')
+    end
+  end
+
+  class SetException < Exception
+    def initialize()
+      super('This repository does not support sets.', 'noSetHierarchy')
+    end
+  end
+
+  class ResumptionTokenException < Exception
+    def initialize()
+      super('The value of the resumptionToken argument is invalid or expired.',
+        'badResumptionToken')
+    end
+  end
+
+end

data/lib/oai/harvester/config.rb

@@ -0,0 +1,41 @@
+#
+#  Created by William Groppe on 2006-11-05.
+#  Copyright (c) 2006. All rights reserved.
+
+module OAI
+  module Harvester
+
+    LOW_RESOLUTION = "YYYY-MM-DD"
+  
+    class Config < OpenStruct
+    
+      PERIODS = %w(daily weekly monthly)
+      GLOBAL = "/etc/oai/harvester.yml"
+    
+      def self.load
+        config = find_config
+        File.exists?(config) ? new(YAML.load_file(config)) : new
+      end
+
+      def save
+        config = Config.find_config
+        open(config, 'w') do |out|
+          YAML.dump(@table, out)
+        end
+      end
+
+      private 
+      # Shamelessly lifted from Camping
+      def self.find_config
+        if home = ENV['HOME'] # POSIX
+          return GLOBAL if File.exists?(GLOBAL) && File.writable?(GLOBAL)
+          FileUtils.mkdir_p File.join(home, '.oai')
+          File.join(home, '.oai/harvester.yml')
+        elsif home = ENV['APPDATA'] # MSWIN
+          File.join(home, 'oai/harvester.yml')
+        end
+      end
+    
+    end
+  end
+end

data/lib/oai/harvester/harvest.rb

@@ -0,0 +1,150 @@
+#
+#  Created by William Groppe on 2006-11-03.
+
+module OAI
+  module Harvester
+  
+    class Harvest
+    
+      def initialize(config = nil, directory = nil, date = nil)
+        @config = config || Config.load
+        @directory = directory || @config.storage
+        @from = date
+        @from.freeze
+        @parser = defined?(XML::Document) ? 'libxml' : 'rexml'
+      end
+    
+      def start(sites = nil, interactive = false)
+        @interactive = interactive
+        sites = (@config.sites.keys rescue {}) unless sites
+        begin
+          sites.each do |site|
+            harvest(site)
+          end
+        ensure
+          @config.save
+        end
+      end
+    
+      private
+    
+      def harvest(site)
+        opts = build_options_hash(@config.sites[site])
+        harvest_time = Time.now.utc
+
+        if "YYYY-MM-DD" == granularity(opts[:url])
+          opts[:until] = harvest_time.strftime("%Y-%m-%d")
+          opts[:from] = @from.strftime("%Y-%m-%d") if @from
+        else
+          opts[:until] = harvest_time.xmlschema
+          opts[:from] = @from.xmlschema if @from
+        end
+
+        # Allow a from date to be passed in
+        opts[:from] = earliest(opts[:url]) unless opts[:from]
+        opts.delete(:set) if 'all' == opts[:set]
+        
+        begin
+          # Connect, and download
+          file, records = call(opts.delete(:url), opts)
+      
+          # Move document to storage directory
+          dir = File.join(@directory, date_based_directory(harvest_time))
+          FileUtils.mkdir_p dir
+          FileUtils.mv(file.path, 
+            File.join(dir, "#{site}-#{filename(Time.parse(opts[:from]), 
+            harvest_time)}.xml.gz"))
+          @config.sites[site]['last'] = harvest_time
+        rescue
+          raise $! unless $!.respond_to?(:code)
+          raise $! if not @interactive || "noRecordsMatch" != $!.code
+          puts "No new records available"
+        end
+      end
+    
+      def call(url, opts)
+        # Preserve original options
+        options = opts.dup
+        
+        records = 0;
+        client = OAI::Client.new(url, :parser => @parser)
+        provider_config = client.identify
+        
+        file = Tempfile.new('oai_data')
+        gz = Zlib::GzipWriter.new(file)
+        gz << "<? xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
+        gz << "<records>"
+        begin
+          response = client.list_records(options)
+          get_records(response.doc).each do |rec|
+            gz << rec
+            records += 1
+          end
+          puts "#{records} records retrieved" if @interactive
+
+          # Get a full response by iterating with the resumption tokens.  
+          # Not very Ruby like.  Should fix OAI::Client to handle resumption
+          # tokens internally.
+          while(response.resumption_token and not response.resumption_token.empty?)
+            puts "\nresumption token recieved, continuing" if @interactive
+            response = client.list_records(:resumption_token => 
+              response.resumption_token)
+              get_records(response.doc).each do |rec|
+                gz << rec
+                records += 1
+              end
+            puts "#{records} records retrieved" if @interactive
+          end
+
+            gz << "</records>"
+            
+        ensure
+          gz.close
+          file.close
+        end
+
+        [file, records]
+      end
+    
+      def get_records(doc)
+        doc.find("/OAI-PMH/ListRecords/record").to_a
+      end
+    
+      def build_options_hash(site)
+        options = {:url => site['url']}
+        options[:set] = site['set'] if site['set']
+        options[:from] = site['last'].utc.xmlschema if site['last']
+        options[:metadata_prefix] = site['prefix'] if site['prefix']
+        options
+      end
+    
+      def date_based_directory(time)
+        "#{time.strftime(DIRECTORY_LAYOUT)}"
+      end
+
+      def filename(from_time, until_time)
+        format = "%Y-%m-%d"
+        "#{from_time.strftime(format)}_til_#{until_time.strftime(format)}"\
+        "_at_#{until_time.strftime('%H-%M-%S')}"
+      end
+    
+      def granularity(url)
+        client = OAI::Client.new url
+        client.identify.granularity
+      end
+
+      # Get earliest timestamp from repository
+      def earliest(url)
+        client = OAI::Client.new url
+        identify = client.identify
+        if "YYYY-MM-DD" == identify.granularity
+          Time.parse(identify.earliest_datestamp).strftime("%Y-%m-%d")
+        else
+          Time.parse(identify.earliest_datestamp).xmlschema
+        end
+      end
+    
+    end
+  
+  end
+end

data/lib/oai/harvester/logging.rb

@@ -0,0 +1,70 @@
+# Reopen Harvest and add logging
+module OAI
+  module Harvester
+
+    class Harvest
+      alias_method :orig_start, :start
+      alias_method :orig_harvest, :harvest
+      alias_method :orig_call, :call
+      alias_method :orig_init, :initialize
+    
+      def initialize(*args)
+        orig_init(*args)
+        @summary = []
+        @logger = Logger.new(File.join(@config.logfile, "harvester.log"), 
+          shift_age = 'weekly') if @config.logfile
+        @logger.datetime_format = "%Y-%m-%d %H:%M"
+      
+        # Turn off logging if no logging directory is specified.
+        @logger.level = Logger::FATAL unless @config.logfile
+      end
+    
+      def start(sites = nil, interactive = false)
+        if not interactive
+          @logger.info { "Starting regular harvest" }
+          orig_start(sites)
+          begin
+            OAI::Harvester::
+              Mailer.send(@config.mail_server, @config.email, @summary)
+          rescue
+            @logger.error { "Error sending out summary email: #{$!}"}
+          end
+        else
+          @logger.info { "Starting interactive harvest"}
+          orig_start(sites, true)
+        end
+      end
+    
+      private
+    
+      def harvest(site)
+        begin
+          @logger.info { "Harvest of '#{site}' starting" }
+          @summary << "Harvest of '#{site}' attempted"
+          orig_harvest(site)
+        rescue OAI::Exception
+          if "noRecordsMatch" == $!.code
+            @logger.info "No new records available"
+            @summary << "'#{site}' had no new records."
+          else
+            @logger.error { "Harvesting of '#{site}' failed, message: #{$!}" }
+            @summary << "'#{site}' had an OAI Error! #{$!}"
+          end
+        rescue
+          @logger.error { "Harvesting of '#{site}' failed, message: #{$!}" }
+          @logger.error { "#{$!.backtrace.join('\n')}" }
+          @summary << "'#{site}' had an Error! #{$!}"
+        end
+      end
+    
+      def call(url, options)
+        @logger.info { "fetching: #{url} with options #{options.inspect}" }
+        file, records = orig_call(url, options)
+        @logger.info { "retrieved #{records} records" }
+        @summary << "Retrieved #{records} records."
+        return file, records
+      end
+    end
+    
+  end
+end

data/lib/oai/harvester/mailer.rb

@@ -0,0 +1,17 @@
+module OAI
+  module Harvester
+    
+    class Mailer
+    
+      def self.send(server = nil, email = nil, message = nil)
+        msg = %{Subject: Harvester Summary\n\n#{message.join("\n")}}
+        to = (email.map { |e| "'#{e}'"}).join(", ")
+        Net::SMTP.start(server) do |smtp|
+          smtp.send_message msg, "harvester@#{Socket.gethostname}", to
+        end
+      end
+    
+    end
+    
+  end
+end