oai 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74c8bf29b2463c130d3ec55f04a433fb828041fba20d9df6045bfec0ac03dbf3
-  data.tar.gz: 8cc85a82a2d1006499baa2a80a238eb50c04b7e9c33e64b0aa5e19daa7afabdd
+  metadata.gz: c0a80559d64aa25add07953e978e93f9abfbc89e03c67bf306e79f0d7b18b0f7
+  data.tar.gz: 7989ff1c6dec95c3965cb3adc1d0bc156d250b6129002be50193443994d03caf
 SHA512:
-  metadata.gz: fd82dfd51917453cea94d63549975bf60cb928c7c619262cb017a79b5dccfe6ae5d02e6a938aeaf4e3577417aa3644319d5e8c59ed9cdd3a992c4db49c886e1d
-  data.tar.gz: 4cfb1c2ca5ec4b81fbe9e21462714801e4aa2c36ce58ecbb01facafd0cbcd7e354ba66d0df53f1789bc19dc9d8eb795cc220605381c9d2d6c24d1a611f6a3505
+  metadata.gz: db8b7e74c65625da8b47b35ef23a7e620777dfefb0a6a8d63a6a7314534dbfe46606bf863aef4a839087547280700519f35e3b10bdf10bf70ba93959ae3125e2
+  data.tar.gz: ad14343ae9a61b6516f7e079db4afc0b7eddbb5e70d8d962221842a5fe4f8b49da63d37dc8eeb7cfe9e163617d4d2a3cae93e37e8943b784df7d0dc1eb1a7cce

data/README.md

@@ -1,9 +1,8 @@
 ruby-oai
 ========
+[![Build Status](https://github.com/code4lib/ruby-oai/workflows/CI/badge.svg)](https://github.com/code4lib/ruby-oai/actions)
 
-[![Build Status](https://travis-ci.org/code4lib/ruby-oai.svg?branch=master)](https://travis-ci.org/code4lib/ruby-oai)
-
-[![Gem Version](https://badge.fury.io/rb/kithe.svg)](https://badge.fury.io/rb/oai)
+[![Gem Version](https://badge.fury.io/rb/oai.svg)](https://badge.fury.io/rb/oai)
 
 ruby-oai is a Open Archives Protocol for Metadata Harvesting (OAI-PMH)
 library for Ruby. [OAI-PMH](http://openarchives.org) is a somewhat
@@ -47,6 +46,22 @@ For example to initiate a ListRecords request to pubmed you can:
   end
 ```
 
+### Retry-After
+This library depends on faraday, but allows a wide range of versions. Depending on the client application's installed version of faraday, there may be different middleware libraries required to support automatically retrying requests that are rate limited/denied with a `Retry-After` header. The OAI client can, however, accept an externally configured faraday http client for handling this. For example, to retry on `429 Too Many Requests`:
+
+```ruby
+require 'oai'
+require 'faraday_middleware' # if using faraday version < 2
+http_client = Faraday.new do |conn|
+    conn.request(:retry, max: 5, retry_statuses: 429)
+    conn.response(:follow_redirects, limit: 5)
+    conn.adapter :net_http
+end
+client = OAI::Client.new(base_url, http: http_client)
+opts = {from:'2012-03-01', until:'2012-04-01', metadata_prefix:'oai_dc'}
+puts client.list_records(opts).full.count
+```
+
 See {OAI::Client} for more details
 
 Server
@@ -97,7 +112,7 @@ There are also convenience tasks to run subsets of tests.
 We use [appraisal](https://github.com/thoughtbot/appraisal) to test ActiveRecord-related functionality under multiple versions of ActiveRecord. While the above commands will test with latest ActiveRecord (allowed in our .gemspec development dependency), you can test under a particular version defined in the [Appraisals](./Appraisals) file like so:
 
     $ bundle exec appraisal rails-52 rake test
-    $ bundle exec appraisal rails-60 rake test
+    $ bundle exec appraisal rails-70 rake test
 
 If you run into trouble with appraisal's gemfiles getting out of date and bundler complaining,
 try:

data/Rakefile

@@ -34,6 +34,13 @@ namespace :test do
     t.warning = false
   end
 
+  Rake::TestTask.new('harvester') do |t|
+    t.libs << ['lib', 'test/harvester']
+    t.pattern = 'test/harvester/tc_*.rb'
+    #t.verbose = true
+    t.warning = false
+  end
+
   Rake::TestTask.new('provider') do |t|
     t.libs << ['lib', 'test/provider']
     t.pattern = 'test/provider/tc_*.rb'

data/bin/oai

@@ -5,8 +5,6 @@
 
 require 'optparse'
 
-DIRECTORY_LAYOUT = "%Y/%m".freeze
-
 require 'oai/harvester'
 
 include OAI::Harvester

data/examples/models/file_model.rb

@@ -45,8 +45,8 @@ class FileModel < OAI::Provider::Model
     case selector
     when :all
       records = Dir["#{@directory}/*.xml"].sort.collect do |file|
-        File.new(file) unless File.stat(file).mtime.utc < opts[:from] or
-          File.stat(file).mtime.utc > opts[:until]
+        File.new(file) unless File.stat(file).mtime.utc < opts[:from].to_time or
+          File.stat(file).mtime.utc > opts[:until].to_time
       end
       records
     else

data/lib/oai/client/response.rb

@@ -37,16 +37,16 @@ module OAI
           message = error.content
           code = ""
           if defined?(error.property) == nil
-              code = error.attributes['code']
-           else
-	      begin
-	 	code = error["code"]
-	      rescue
-                code = error.property('code')
-	      end
+            code = error.attributes['code']
+         else
+           begin
+             code = error["code"]
+           rescue
+             code = error.property('code')
            end
+         end
       end
-      raise OAI::Exception.new(message, code)
+      raise OAI::Exception.for(message: message, code: code)
     end
 
   end

data/lib/oai/client.rb

@@ -54,7 +54,7 @@ module OAI
   # <http://www.openarchives.org/OAI/openarchivesprotocol.html>.
 
   class Client
-
+    UNESCAPED_AMPERSAND = /&(?!(?:amp|lt|gt|quot|apos|\#\d+);)/
     # The constructor which must be passed a valid base url for an oai
     # service:
     #
@@ -95,7 +95,8 @@ module OAI
           follow_redirects = 5 if follow_redirects == true
 
           if follow_redirects
-            require 'faraday_middleware'
+            require 'faraday/follow_redirects'
+            builder.use Faraday::FollowRedirects::Middleware
             builder.response :follow_redirects, :limit => follow_redirects.to_i
           end
           builder.adapter :net_http
@@ -197,12 +198,9 @@ module OAI
       do_resumable(OAI::ListSetsResponse, 'ListSets', opts)
     end
 
-    private
-
-    def do_request(verb, opts = nil)
-      # fire off the request and return appropriate DOM object
-      uri = build_uri(verb, opts)
-      xml = strip_invalid_utf_8_chars(get(uri))
+    def sanitize_xml(xml)
+      xml = strip_invalid_utf_8_chars(xml)
+      xml = strip_invalid_xml_chars(xml)
       if @parser == 'libxml'
         # remove default namespace for oai-pmh since libxml
         # isn't able to use our xpaths to get at them
@@ -210,7 +208,15 @@ module OAI
         xml = xml.gsub(
           /xmlns=\"http:\/\/www.openarchives.org\/OAI\/.\..\/\"/, '')
       end
-      return load_document(xml)
+      xml
+    end
+
+    private
+
+    def do_request(verb, opts = nil)
+      # fire off the request and return appropriate DOM object
+      uri = build_uri(verb, opts)
+      return load_document(get(uri))
     end
 
     def do_resumable(responseClass, verb, opts)
@@ -240,6 +246,7 @@ module OAI
     end
 
     def load_document(xml)
+      xml = sanitize_xml(xml)
       case @parser
       when 'libxml'
         begin
@@ -330,7 +337,16 @@ module OAI
     # Regex is from WebCollab:
     #   http://webcollab.sourceforge.net/unicode.html
     def strip_invalid_utf_8_chars(xml)
-      xml && xml.gsub(/[\x00-\x08\x10\x0B\x0C\x0E-\x19\x7F]
+      return nil unless xml
+
+      # If it's in a specific encoding other than BINARY, it may trigger
+      # an exception to try to gsub these illegal bytes. Temporarily
+      # put it in BINARY. NOTE: We're not totally sure what's going on
+      # with encodings in this gem in general, it might not be totally reasonable.
+      orig_encoding = xml.encoding
+      xml.force_encoding("BINARY")
+
+      xml = xml.gsub(/[\x00-\x08\x10\x0B\x0C\x0E-\x19\x7F]
                              | [\x00-\x7F][\x80-\xBF]+
                              | ([\xC0\xC1]|[\xF0-\xFF])[\x80-\xBF]*
                              | [\xC2-\xDF]((?![\x80-\xBF])|[\x80-\xBF]{2,})
@@ -338,7 +354,15 @@ module OAI
                              | (?![\x80-\xBF]{2})|[\x80-\xBF]{3,})/x, '?')\
                 .gsub(/\xE0[\x80-\x9F][\x80-\xBF]
                        | \xED[\xA0-\xBF][\x80-\xBF]/,'?')
+
+      xml.force_encoding(orig_encoding)
+
+      xml
     end
 
+    def strip_invalid_xml_chars(xml)
+      return xml unless xml =~ UNESCAPED_AMPERSAND
+      xml.gsub(UNESCAPED_AMPERSAND, '&amp;')
+    end
   end
 end

data/lib/oai/exception.rb

@@ -4,72 +4,80 @@ module OAI
   # messages will be wrapped in an XML response to the client.
 
   class Exception < RuntimeError
+    CODE = nil
+    MESSAGE = nil
+
     attr_reader :code
 
-    def initialize(message, code = nil)
-      super(message)
-      @code = code
+    @@codes = {}
+
+    def self.register_exception_code(code, exception_class)
+      @@codes[code] = exception_class if exception_class.superclass == OAI::Exception
+    end
+
+    def self.for(message: nil, code: nil)
+      @@codes.fetch(code, Exception).new(message)
+    end
+
+    def initialize(message = nil, code = nil)
+      super(message || self.class::MESSAGE)
+      @code = code || self.class::CODE
     end
   end
 
   class ArgumentException < Exception
-    def initialize()
-      super('The request includes ' \
+    CODE = 'badArgument'
+    MESSAGE = 'The request includes ' \
       'illegal arguments, is missing required arguments, includes a ' \
-      'repeated argument, or values for arguments have an illegal syntax.',
-      'badArgument')
-    end
+      'repeated argument, or values for arguments have an illegal syntax.'
+    register_exception_code(CODE, self)
   end
 
   class VerbException < Exception
-    def initialize()
-      super('Value of the verb argument is not a legal OAI-PMH '\
-      'verb, the verb argument is missing, or the verb argument is repeated.',
-      'badVerb')
-    end
+    CODE = 'badVerb'
+    MESSAGE = 'Value of the verb argument is not a legal OAI-PMH '\
+      'verb, the verb argument is missing, or the verb argument is repeated.'
+    register_exception_code(CODE, self)
   end
 
   class FormatException < Exception
-    def initialize()
-      super('The metadata format identified by '\
+    CODE = 'cannotDisseminateFormat'
+    MESSAGE = 'The metadata format identified by '\
         'the value given for the metadataPrefix argument is not supported '\
-        'by the item or by the repository.', 'cannotDisseminateFormat')
-    end
+        'by the item or by the repository.'
+    register_exception_code(CODE, self)
   end
 
   class IdException < Exception
-    def initialize()
-      super('The value of the identifier argument is '\
-        'unknown or illegal in this repository.', 'idDoesNotExist')
-    end
+    CODE = 'idDoesNotExist'
+    MESSAGE = 'The value of the identifier argument is '\
+        'unknown or illegal in this repository.'
+    register_exception_code(CODE, self)
   end
 
   class NoMatchException < Exception
-    def initialize()
-      super('The combination of the values of the from, '\
-      'until, set and metadataPrefix arguments results in an empty list.',
-      'noRecordsMatch')
-    end
+    CODE = 'noRecordsMatch'
+    MESSAGE = 'The combination of the values of the from, '\
+      'until, set and metadataPrefix arguments results in an empty list.'
+    register_exception_code(CODE, self)
   end
 
   class MetadataFormatException < Exception
-    def initialize()
-      super('There are no metadata formats available '\
-        'for the specified item.', 'noMetadataFormats')
-    end
+    CODE = 'noMetadataFormats'
+    MESSAGE = 'There are no metadata formats available '\
+        'for the specified item.'
+    register_exception_code(CODE, self)
   end
 
   class SetException < Exception
-    def initialize()
-      super('This repository does not support sets.', 'noSetHierarchy')
-    end
+    CODE = 'noSetHierarchy'
+    MESSAGE = 'This repository does not support sets.'
+    register_exception_code(CODE, self)
   end
 
   class ResumptionTokenException < Exception
-    def initialize()
-      super('The value of the resumptionToken argument is invalid or expired.',
-        'badResumptionToken')
-    end
+    CODE = 'badResumptionToken'
+    MESSAGE = 'The value of the resumptionToken argument is invalid or expired.'
+    register_exception_code(CODE, self)
   end
-
 end

data/lib/oai/harvester/config.rb

@@ -5,7 +5,7 @@
 module OAI
   module Harvester
 
-    LOW_RESOLUTION = "YYYY-MM-DD"
+    LOW_RESOLUTION = OAI::Const::Granularity::LOW
   
     class Config < OpenStruct
     

data/lib/oai/harvester/harvest.rb

@@ -3,14 +3,16 @@
 
 module OAI
   module Harvester
-  
     class Harvest
+      DIRECTORY_LAYOUT = "%Y/%m".freeze
     
-      def initialize(config = nil, directory = nil, date = nil)
+      def initialize(config = nil, directory = nil, date = nil, to = nil)
         @config = config || Config.load
         @directory = directory || @config.storage
         @from = date
         @from.freeze
+        @until = to
+        @until.freeze
         @parser = defined?(XML::Document) ? 'libxml' : 'rexml'
       end
     
@@ -30,9 +32,13 @@ module OAI
     
       def harvest(site)
         opts = build_options_hash(@config.sites[site])
-        harvest_time = Time.now.utc
+        if @until
+          harvest_time = @until.to_time.utc
+        else
+          harvest_time = Time.now.utc
+        end
 
-        if "YYYY-MM-DD" == granularity(opts[:url])
+        if OAI::Const::Granularity::LOW == granularity(opts[:url])
           opts[:until] = harvest_time.strftime("%Y-%m-%d")
           opts[:from] = @from.strftime("%Y-%m-%d") if @from
         else
@@ -43,22 +49,27 @@ module OAI
         # Allow a from date to be passed in
         opts[:from] = earliest(opts[:url]) unless opts[:from]
         opts.delete(:set) if 'all' == opts[:set]
-        
         begin
           # Connect, and download
           file, records = call(opts.delete(:url), opts)
       
-          # Move document to storage directory
-          dir = File.join(@directory, date_based_directory(harvest_time))
-          FileUtils.mkdir_p dir
-          FileUtils.mv(file.path, 
-            File.join(dir, "#{site}-#{filename(Time.parse(opts[:from]), 
-            harvest_time)}.xml.gz"))
+          # Move document to storage directory if configured
+          if @directory
+            directory_layout = @config.layouts[site] if @config.layouts
+            dir = File.join(@directory, date_based_directory(harvest_time, directory_layout))
+            FileUtils.mkdir_p dir
+            FileUtils.mv(file.path,
+              File.join(dir, "#{site}-#{filename(Time.parse(opts[:from]),
+              harvest_time)}.xml.gz"))
+          else
+            puts "no configured destination for temp file" if @interactive
+          end
           @config.sites[site]['last'] = harvest_time
-        rescue
-          raise $! unless $!.respond_to?(:code)
-          raise $! if not @interactive || "noRecordsMatch" != $!.code
-          puts "No new records available"
+        rescue OAI::NoMatchException
+          puts "No new records available" if @interactive
+        rescue OAI::Exception => ex
+          raise ex if not @interactive
+          puts ex.message
         end
       end
     
@@ -69,15 +80,15 @@ module OAI
         records = 0;
         client = OAI::Client.new(url, :parser => @parser)
         provider_config = client.identify
-        
+
         file = Tempfile.new('oai_data')
         gz = Zlib::GzipWriter.new(file)
         gz << "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
         gz << "<records>"
         begin
           response = client.list_records(options)
-          get_records(response.doc).each do |rec|
-            gz << rec
+          response.each do |rec|
+            gz << rec._source
             records += 1
           end
           puts "#{records} records retrieved" if @interactive
@@ -89,8 +100,8 @@ module OAI
             puts "\nresumption token recieved, continuing" if @interactive
             response = client.list_records(:resumption_token => 
               response.resumption_token)
-              get_records(response.doc).each do |rec|
-                gz << rec
+              response.each do |rec|
+                gz << rec._source
                 records += 1
               end
             puts "#{records} records retrieved" if @interactive
@@ -118,8 +129,9 @@ module OAI
         options
       end
     
-      def date_based_directory(time)
-        "#{time.strftime(DIRECTORY_LAYOUT)}"
+      def date_based_directory(time, directory_layout = nil)
+        directory_layout ||= Harvest::DIRECTORY_LAYOUT
+        "#{time.strftime(directory_layout)}"
       end
 
       def filename(from_time, until_time)
@@ -127,7 +139,7 @@ module OAI
         "#{from_time.strftime(format)}_til_#{until_time.strftime(format)}"\
         "_at_#{until_time.strftime('%H-%M-%S')}"
       end
-    
+
       def granularity(url)
         client = OAI::Client.new url
         client.identify.granularity
@@ -137,7 +149,7 @@ module OAI
       def earliest(url)
         client = OAI::Client.new url
         identify = client.identify
-        if "YYYY-MM-DD" == identify.granularity
+        if OAI::Const::Granularity::LOW == identify.granularity
           Time.parse(identify.earliest_datestamp).strftime("%Y-%m-%d")
         else
           Time.parse(identify.earliest_datestamp).xmlschema
@@ -147,4 +159,4 @@ module OAI
     end
   
   end
-end
+end

data/lib/oai/harvester/logging.rb

@@ -11,10 +11,9 @@ module OAI
       def initialize(*args)
         orig_init(*args)
         @summary = []
-        @logger = Logger.new(File.join(@config.logfile, "harvester.log"), 
-          shift_age = 'weekly') if @config.logfile
+        @logger = @config.logfile ? Logger.new(File.join(@config.logfile, "harvester.log"), 'weekly') : Logger.new(STDOUT)
         @logger.datetime_format = "%Y-%m-%d %H:%M"
-      
+
         # Turn off logging if no logging directory is specified.
         @logger.level = Logger::FATAL unless @config.logfile
       end
@@ -24,8 +23,7 @@ module OAI
           @logger.info { "Starting regular harvest" }
           orig_start(sites)
           begin
-            OAI::Harvester::
-              Mailer.send(@config.mail_server, @config.email, @summary)
+            OAI::Harvester::Mailer.send(@config.mail_server, @config.email, @summary) if @config.email
           rescue
             @logger.error { "Error sending out summary email: #{$!}"}
           end

data/lib/oai/harvester.rb

@@ -7,9 +7,12 @@ require 'logger'
 require 'fileutils'
 require 'ostruct'
 require 'readline'
-require 'chronic'
 require 'socket'
 
+if not defined?(OAI::Const::VERBS)
+  require 'oai/constants'
+end
+
 require 'oai/client'
 require 'oai/harvester/config'
 require 'oai/harvester/harvest'

data/lib/oai/provider/model/activerecord_caching_wrapper.rb

@@ -78,13 +78,7 @@ module OAI::Provider
       raise ResumptionTokenException.new unless @limit
 
       token = ResumptionToken.parse(token_string)
-      total = model.where(token_conditions(token)).count
-
-      if token.last * @limit + @limit < total
-        select_partial(token)
-      else
-        select_partial(token).records
-      end
+      select_partial(token)
     end
 
     # select a subset of the result set, and return it with a
@@ -102,10 +96,13 @@ module OAI::Provider
 
       raise ResumptionTokenException.new unless oaitoken
 
+      total = model.where(token_conditions(token)).count
+      # token offset should be nil if this is the last set
+      offset = (token.last * @limit + @limit >= total) ? nil : token.last + 1
       PartialResult.new(
         hydrate_records(
           oaitoken.entries.limit(@limit).offset(token.last * @limit)),
-        token.next(token.last + 1)
+        token.next(offset)
       )
     end