oai 0.0.1 → 1.3.0

data/lib/oai/provider/metadata_format.rb

@@ -0,0 +1,119 @@
+require 'singleton'
+
+module OAI::Provider::Metadata
+  # == Metadata Base Class
+  #
+  # MetadataFormat is the base class from which all other format classes 
+  # should inherit.  Format classes provide mapping of record fields into XML.
+  #
+  # * prefix - contains the metadata_prefix used to select the format
+  # * schema - location of the xml schema
+  # * namespace - location of the namespace document
+  # * element_namespace - the namespace portion of the XML elements
+  # * fields - list of fields in this metadata format
+  #
+  # See OAI::Metadata::DublinCore for an example
+  #
+  class Format
+    include Singleton
+    
+    attr_accessor :prefix, :schema, :namespace, :element_namespace, :fields
+    
+    # Provided a model, and a record belonging to that model this method
+    # will return an xml represention of the record.  This is the method
+    # that should be extended if you need to create more complex xml
+    # representations.
+    def encode(model, record)
+      if record.respond_to?("to_#{prefix}")
+        record.send("to_#{prefix}")
+      else
+        xml = Builder::XmlMarkup.new
+        map = model.respond_to?("map_#{prefix}") ? model.send("map_#{prefix}") : {}
+          xml.tag!("#{prefix}:#{element_namespace}", header_specification) do
+            fields.each do |field|
+              values = value_for(field, record, map)
+              if values.respond_to?(:each)
+                values.each do |value|
+                  xml.tag! "#{element_namespace}:#{field}", value
+                end
+              else
+                xml.tag! "#{element_namespace}:#{field}", values
+              end
+            end
+          end
+        xml.target!
+      end
+    end
+
+    private
+
+    # We try a bunch of different methods to get the data from the model.
+    #
+    # 1.  Check if the model defines a field mapping for the field of 
+    #     interest.
+    # 2.  Try calling the pluralized name method on the model.
+    # 3.  Try calling the singular name method on the model
+    def value_for(field, record, map)
+      method = map[field] ? map[field].to_s : field.to_s 
+     
+      if record.respond_to?(pluralize(method))
+        record.send pluralize(method)
+      elsif method != 'type' and record.respond_to?(method)
+        # TODO: this calls type, which is deprecated, should be 
+        record.send method
+      else
+        []
+      end
+    end
+
+    # Subclasses must override
+    def header_specification
+      raise NotImplementedError.new
+    end
+    
+    # Shamelessly lifted form ActiveSupport.  Thanks Rails community!
+    def pluralize(word)
+      # Use ActiveSupports pluralization if it's available.
+      return word.pluralize if word.respond_to?(:pluralize)
+      
+      # Otherwise use our own simple pluralization rules.
+      result = word.to_s.dup
+      
+      # Uncountable words
+      return result if %w(equipment information rice money species series fish sheep).include?(result)
+      
+      # Irregular words
+      { 'person' => 'people', 'man' => 'men', 'child' => 'children', 'sex' => 'sexes', 
+        'move' => 'moves', 'cow' => 'kine' }.each { |k,v| return v if word == k }
+      
+      rules.each { |(rule, replacement)| break if result.gsub!(rule, replacement) }
+      result
+    end
+    
+    def rules
+      [
+        [/$/, 's'],
+        [/s$/i, 's'],
+        [/(ax|test)is$/i, '\1es'],
+        [/(octop|vir)us$/i, '\1i'],
+        [/(alias|status)$/i, '\1es'],
+        [/(bu)s$/i, '\1ses'],
+        [/(buffal|tomat)o$/i, '\1oes'],
+        [/([ti])um$/i, '\1a'],
+        [/sis$/i, 'ses'],
+        [/(?:([^f])fe|([lr])f)$/i, '\1\2ves'],
+        [/(hive)$/i, '\1s'],
+        [/([^aeiouy]|qu)y$/i, '\1ies'],
+        [/(x|ch|ss|sh)$/i, '\1es'],
+        [/(matr|vert|ind)(?:ix|ex)$/i, '\1ices'],
+        [/([m|l])ouse$/i, '\1ice'],
+        [/^(ox)$/i, '\1en'],
+        [/(quiz)$/i, '\1zes']
+      ]
+    end
+
+  end
+  
+end
+
+Dir.glob(File.dirname(__FILE__) + '/metadata_format/*.rb').each {|lib| require lib}

data/lib/oai/provider/model/activerecord_caching_wrapper.rb

@@ -0,0 +1,129 @@
+require 'active_record'
+
+module OAI::Provider
+
+  # ActiveRecord model class in support of the caching wrapper.
+  class OaiToken < ActiveRecord::Base
+    has_many :entries, -> { order("record_id ASC") },
+      :class_name => 'OaiEntry', :dependent => :destroy
+
+    validates_uniqueness_of :token
+
+    # Make sanitize_sql a public method so we can make use of it.
+    public
+
+    def self.sanitize_sql(*arg)
+      super(*arg)
+    end
+
+    def new_record_before_save?
+      @new_record_before_save
+    end
+
+  end
+
+  # ActiveRecord model class in support of the caching wrapper.
+  class OaiEntry < ActiveRecord::Base
+    belongs_to :oai_token
+
+    validates_uniqueness_of :record_id, :scope => :oai_token
+  end
+
+  # This class wraps an ActiveRecord model and delegates all of the record
+  # selection/retrieval to the AR model.  It accepts options for specifying
+  # the update timestamp field, a timeout, and a limit.  The limit option
+  # is used for doing pagination with resumption tokens.  The timeout is
+  # used to expire old tokens from the cache.  Default timeout is 12 hours.
+  #
+  # The difference between ActiveRecordWrapper and this class is how the
+  # pagination is accomplished.  ActiveRecordWrapper encodes all the
+  # information in the token.  That approach should work 99% of the time.
+  # If you have an extremely active respository you may want to consider
+  # the caching wrapper.  The caching wrapper takes the entire result set
+  # from a request and caches it in another database table, well tables
+  # actually.  So the result returned to the client will always be
+  # internally consistent.
+  #
+  class ActiveRecordCachingWrapper < ActiveRecordWrapper
+
+    attr_reader :model, :timestamp_field, :expire
+
+    def initialize(model, options={})
+      @expire = options.delete(:timeout) || 12.hours
+      super(model, options)
+    end
+
+    def find(selector, options={})
+      sweep_cache
+      return next_set(options[:resumption_token]) if options[:resumption_token]
+
+      conditions = sql_conditions(options)
+
+      if :all == selector
+        total = model.where(conditions).count
+        if @limit && total > @limit
+          select_partial(
+            ResumptionToken.new(options.merge({:last => 0})))
+        else
+          model.where(conditions)
+        end
+      else
+        model.where(conditions).find(selector)
+      end
+    end
+
+    protected
+
+    def next_set(token_string)
+      raise ResumptionTokenException.new unless @limit
+
+      token = ResumptionToken.parse(token_string)
+      select_partial(token)
+    end
+
+    # select a subset of the result set, and return it with a
+    # resumption token to get the next subset
+    def select_partial(token)
+      oaitoken = OaiToken.find_by(token: token.to_s)
+
+      if 0 == token.last && oaitoken.nil?
+        oaitoken = OaiToken.create!(token: token.to_s)
+        OaiToken.connection.execute("insert into " +
+          "#{OaiEntry.table_name} (oai_token_id, record_id) " +
+          "select #{oaitoken.id}, id from #{model.table_name} where " +
+          "#{OaiToken.sanitize_sql(token_conditions(token))}")
+      end
+
+      raise ResumptionTokenException.new unless oaitoken
+
+      total = model.where(token_conditions(token)).count
+      # token offset should be nil if this is the last set
+      offset = (token.last * @limit + @limit >= total) ? nil : token.last + 1
+      PartialResult.new(
+        hydrate_records(
+          oaitoken.entries.limit(@limit).offset(token.last * @limit)),
+        token.next(offset)
+      )
+    end
+
+    def sweep_cache
+      OaiToken.where(["created_at < ?", Time.now - expire]).destroy_all
+    end
+
+    def hydrate_records(records)
+      model.find(records.collect {|r| r.record_id })
+    end
+
+    def token_conditions(token)
+      sql_conditions token.to_conditions_hash
+    end
+
+    private
+
+    def expires_at(creation)
+      created = Time.parse(creation.strftime("%Y-%m-%d %H:%M:%S"))
+      created.utc + expire
+    end
+
+  end
+end

data/lib/oai/provider/model/activerecord_wrapper.rb

@@ -0,0 +1,217 @@
+require 'active_record'
+
+module OAI::Provider
+  # This class wraps an ActiveRecord model and delegates all of the record
+  # selection/retrieval to the AR model.  It accepts options for specifying
+  # the update timestamp field, a timeout, and a limit.  The limit option
+  # is used for doing pagination with resumption tokens.  The
+  # expiration timeout is ignored, since all necessary information is
+  # encoded in the token.
+  #
+  class ActiveRecordWrapper < Model
+
+    attr_reader :model, :timestamp_field, :identifier_field
+
+    # If custom 'timestamp_field' is used, be aware this will be an ActiveRecord
+    # attribute that we will limit on, so perhaps should be indexe appropriately.
+    #
+    # If custom `identifier_field` is used, be aware this will be an ActiveRecord
+    # attribute that we will sort on, and use in WHERE clauses with `=` as well as
+    # greater than/less than, so should be indexed appropriately.
+    def initialize(model, options={})
+      @model = model
+      @timestamp_field = options.delete(:timestamp_field) || 'updated_at'
+      @identifier_field = options.delete(:identifier_field) || model.primary_key || "id"
+      @limit = options.delete(:limit) || 100
+
+      unless options.empty?
+        raise ArgumentError.new(
+          "Unsupported options [#{options.keys.join(', ')}]"
+        )
+      end
+    end
+
+    def earliest
+      earliest_obj = model.order("#{model.base_class.table_name}.#{timestamp_field} asc").first
+      earliest_obj.nil? ? Time.at(0) : earliest_obj.send(timestamp_field)
+    end
+
+    def latest
+      latest_obj = model.order("#{model.base_class.table_name}.#{timestamp_field} desc").first
+      latest_obj.nil? ? Time.now : latest_obj.send(timestamp_field)
+    end
+    # A model class is expected to provide a method Model.sets that
+    # returns all the sets the model supports.  See the
+    # activerecord_provider tests for an example.
+    def sets
+      model.respond_to?(:sets) ? model.sets : []
+    end
+
+    def find(selector, options={})
+      find_scope = find_scope(options)
+      return next_set(find_scope,
+        options[:resumption_token]) if options[:resumption_token]
+      conditions = sql_conditions(options)
+      if :all == selector
+        total = find_scope.where(conditions).count
+        if @limit && total > @limit
+          select_partial(find_scope,
+            ResumptionToken.new(options.merge({:last => 0})))
+        else
+          find_scope.where(conditions)
+        end
+      else
+        find_scope.where(conditions).where(identifier_field => selector).first
+      end
+    end
+
+    def deleted?(record)
+      if record.respond_to?(:deleted_at)
+        return record.deleted_at
+      elsif record.respond_to?(:deleted)
+        return record.deleted
+      end
+      false
+    end
+
+    def respond_to?(m, *args)
+      if m =~ /^map_/
+        model.respond_to?(m, *args)
+      else
+        super
+      end
+    end
+
+    def method_missing(m, *args, &block)
+      if m =~ /^map_/
+        model.send(m, *args, &block)
+      else
+        super
+      end
+    end
+
+    protected
+
+    def find_scope(options)
+      return model unless options.key?(:set)
+
+      # Find the set or return an empty scope
+      set = find_set_by_spec(options[:set])
+      return model.limit(0) if set.nil?
+
+      # If the set has a backward relationship, we'll use it
+      if set.class.respond_to?(:reflect_on_all_associations)
+        set.class.reflect_on_all_associations.each do |assoc|
+          return set.send(assoc.name) if assoc.klass == model
+        end
+      end
+
+      # Search the attributes for 'set'
+      if model.column_names.include?('set')
+        # Scope using the set attribute as the spec
+        model.where(set: options[:set])
+      else
+        # Default to empty set, as we've tried everything else
+        model.none
+      end
+    end
+
+    def find_set_by_spec(spec)
+      if sets.class == ActiveRecord::Relation
+        sets.find_by_spec(spec)
+      else
+        sets.detect {|set| set.spec == spec}
+      end
+    end
+
+    # Request the next set in this sequence.
+    def next_set(find_scope, token_string)
+      raise OAI::ResumptionTokenException.new unless @limit
+
+      token = ResumptionToken.parse(token_string)
+      select_partial(find_scope, token)
+    end
+
+    # select a subset of the result set, and return it with a
+    # resumption token to get the next subset
+    def select_partial(find_scope, token)
+      records = find_scope.where(token_conditions(token))
+        .limit(@limit)
+        .order("#{model.base_class.table_name}.#{identifier_field} asc")
+      raise OAI::ResumptionTokenException.new unless records
+
+      total = find_scope.where(token_conditions(token)).count
+      # token offset should be nil if this is the last set
+      offset = (@limit >= total) ? nil : records.last.send(identifier_field)
+      PartialResult.new(records, token.next(offset))
+    end
+
+    # build a sql conditions statement from the content
+    # of a resumption token.  It is very important not to
+    # miss any changes as records may change scope as the
+    # harvest is in progress.  To avoid loosing any changes
+    # the last 'id' of the previous set is used as the
+    # filter to the next set.
+    def token_conditions(token)
+      last_id = token.last_str
+      sql = sql_conditions token.to_conditions_hash
+
+      return sql if "0" == last_id
+      # Now add last id constraint
+      sql.first << " AND #{model.base_class.table_name}.#{identifier_field} > :id"
+      sql.last[:id] = last_id
+
+      return sql
+    end
+
+    # build a sql conditions statement from an OAI options hash
+    def sql_conditions(opts)
+      sql = []
+      esc_values = {}
+      if opts.has_key?(:from)
+        sql << "#{model.base_class.table_name}.#{timestamp_field} >= :from"
+        esc_values[:from] = parse_to_local(opts[:from])
+      end
+      if opts.has_key?(:until)
+        # Handle databases which store fractions of a second by rounding up
+        sql << "#{model.base_class.table_name}.#{timestamp_field} < :until"
+        esc_values[:until] = parse_to_local(opts[:until]) { |t| t + 1 }
+      end
+
+      return [sql.join(" AND "), esc_values]
+    end
+
+    private
+
+    def parse_to_local(time)
+      if time.respond_to?(:strftime)
+        time_obj = time
+      else
+        begin
+          if time[-1] == "Z"
+            time_obj = Time.strptime(time, "%Y-%m-%dT%H:%M:%S%Z")
+          else
+            time_obj = Date.strptime(time, "%Y-%m-%d")
+          end
+        rescue
+          raise OAI::ArgumentException.new, "unparsable date: '#{time}'"
+        end
+      end
+
+      time_obj = yield(time_obj) if block_given?
+
+      if time_obj.kind_of?(Date)
+        time_obj.strftime("%Y-%m-%d")
+      else
+        # Convert to same as DB - :local => :getlocal, :utc => :getutc
+        if ActiveRecord::VERSION::MAJOR >= 7
+          tzconv = "get#{ActiveRecord.default_timezone.to_s}".to_sym
+        else
+          tzconv = "get#{model.default_timezone.to_s}".to_sym
+        end
+        time_obj.send(tzconv).strftime("%Y-%m-%d %H:%M:%S")
+      end
+    end
+
+  end
+end

data/lib/oai/provider/model.rb

@@ -0,0 +1,80 @@
+module OAI::Provider
+  # = OAI::Provider::Model
+  #
+  # Model implementers should subclass OAI::Provider::Model and override
+  # Model#earliest, Model#latest, and Model#find.  Optionally Model#sets and
+  # Model#deleted? can be used to support sets and record deletions.  It
+  # is also the responsibility of the model implementer to account for
+  # resumption tokens if support is required.  Models that don't support
+  # resumption tokens should raise an exception if a limit is requested
+  # during initialization.
+  #
+  # earliest - should return the earliest update time in the repository.
+  # latest - should return the most recent update time in the repository.
+  # sets - should return an array of sets supported by the repository.
+  # deleted? - individual records returned should respond true or false
+  # when sent the deleted? message.
+  # available_formats - if overridden, individual records should return an
+  # array of prefixes for all formats in which that record is available,
+  # if other than ["oai_dc"]
+  # about - if overridden, should return a String or Array of XML Strings to
+  # insert into the OAI Record <about> chunks.
+  #
+  # == Resumption Tokens
+  #
+  # For examples of using resumption tokens see the
+  # ActiveRecordWrapper, and ActiveRecordCachingWrapper classes.
+  #
+  # There are several helper models for dealing with resumption tokens please
+  # see the ResumptionToken class for more details.
+  #
+  class Model
+    attr_reader :timestamp_field, :identifier_field, :limit
+
+    def initialize(limit = nil, timestamp_field = 'updated_at', identifier_field = 'id')
+      @limit = limit
+      @identifier_field = identifier_field
+      @timestamp_field = timestamp_field
+    end
+
+    # should return the earliest timestamp available from this model.
+    def earliest
+      raise NotImplementedError.new
+    end
+
+    # should return the latest timestamp available from this model.
+    def latest
+      raise NotImplementedError.new
+    end
+
+    def sets
+      nil
+    end
+
+    # find is the core method of a model, it returns records from the model
+    # bases on the parameters passed in.
+    #
+    # <tt>selector</tt> can be a singular id, or the symbol :all
+    # <tt>options</tt> is a hash of options to be used to constrain the query.
+    #
+    # Valid options:
+    # * :from => earliest timestamp to be included in the results
+    # * :until => latest timestamp to be included in the results
+    # * :set => the set from which to retrieve the results
+    # * :metadata_prefix => type of metadata requested (this may be useful if
+    #                       not all records are available in all formats)
+    def find(selector, options={})
+      raise NotImplementedError.new
+    end
+
+    def deleted?
+      false
+    end
+
+    # can return a String or Array of XML Strings add as OAI Record <about> chunks.
+    def about record
+      nil
+    end
+  end
+
+end

data/lib/oai/provider/partial_result.rb

@@ -0,0 +1,18 @@
+module OAI::Provider
+  # = OAI::Provider::PartialResult
+  #
+  # PartialResult is used for returning a set/page of results from a model
+  # that supports resumption tokens.  It should contain and array of
+  # records, and a resumption token for getting the next set/page.
+  #
+  class PartialResult
+    attr_reader :records, :token
+    
+    def initialize(records, token = nil)
+      @records = records
+      @token = token
+    end
+    
+  end
+
+end

data/lib/oai/provider/response/error.rb

@@ -0,0 +1,16 @@
+module OAI::Provider::Response
+  class Error < Base
+    
+    def initialize(provider, error)
+      super(provider)
+      @error = error
+    end
+    
+    def to_xml
+      response do |r|
+        r.error @error.to_s, :code => @error.code
+      end
+    end
+    
+  end
+end

data/lib/oai/provider/response/get_record.rb

@@ -0,0 +1,27 @@
+module OAI::Provider::Response
+
+  class GetRecord < RecordResponse
+    required_parameters :identifier, :metadata_prefix
+
+    def to_xml
+      id = extract_identifier(options.delete(:identifier))
+      unless record = provider.model.find(id, options)
+        raise OAI::IdException.new
+      end
+
+      response do |r|
+        r.GetRecord do
+          r.record do
+            header_for record
+            data_for record unless deleted?(record)
+            about_for record unless deleted?(record)
+          end
+        end
+      end
+    end
+
+  end
+
+end
+
+

data/lib/oai/provider/response/identify.rb

@@ -0,0 +1,37 @@
+module OAI::Provider::Response
+  
+  class Identify < Base
+    
+    def to_xml
+      response do |r|
+        r.Identify do
+          r.repositoryName provider.name
+          r.baseURL provider.url
+          r.protocolVersion 2.0
+          if provider.email and provider.email.respond_to?(:each)
+            provider.email.each { |address| r.adminEmail address }
+          else
+            r.adminEmail provider.email.to_s
+          end
+          r.earliestDatestamp Time.parse(provider.model.earliest.to_s).utc.xmlschema
+          r.deletedRecord provider.delete_support.to_s
+          r.granularity provider.granularity
+          r.description do
+            r.tag! 'oai-identifier', 'xmlns' => 'http://www.openarchives.org/OAI/2.0/oai-identifier', 'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance', 'xsi:schemaLocation' => 'http://www.openarchives.org/OAI/2.0/oai-identifier http://www.openarchives.org/OAI/2.0/oai-identifier.xsd' do
+              r.scheme 'oai'
+              r.repositoryIdentifier provider.prefix.gsub(/oai:/, '')
+              r.delimiter ':'
+              r.sampleIdentifier "#{provider.prefix}:#{provider.identifier}"
+            end
+          end
+          if provider.description
+            r.target! << provider.description
+          end
+        end
+
+      end
+    end
+    
+  end
+  
+end

data/lib/oai/provider/response/list_identifiers.rb

@@ -0,0 +1,30 @@
+module OAI::Provider::Response
+  
+  class ListIdentifiers < RecordResponse
+    
+    def to_xml
+      result = provider.model.find(:all, options)
+
+      # result may be an array of records, or a partial result
+      records = result.respond_to?(:records) ? result.records : result
+
+      raise OAI::NoMatchException.new if records.nil? or records.empty?
+      
+      response do |r|
+        r.ListIdentifiers do
+          records.each do |rec|
+            header_for rec
+          end
+
+          # append resumption token for getting next group of records
+          if result.respond_to?(:token)
+            r.target! << result.token.to_xml
+          end
+
+        end
+      end
+    end
+    
+  end
+  
+end