oai 0.0.3 → 0.0.4

data/lib/oai/provider/model/activerecord_caching_wrapper.rb

@@ -0,0 +1,135 @@
+require 'active_record'
+
+module OAI::Provider
+  
+  # ActiveRecord model class in support of the caching wrapper.
+  class OaiToken < ActiveRecord::Base
+    has_many :entries, :class_name => 'OaiEntry', 
+      :order => "record_id", :dependent => :destroy
+
+    validates_uniqueness_of :token
+    
+    # Make sanitize_sql a public method so we can make use of it.
+    public
+     
+    def self.sanitize_sql(*arg)
+      super(*arg)
+    end
+    
+    def new_record_before_save?
+      @new_record_before_save
+    end
+
+  end
+
+  # ActiveRecord model class in support of the caching wrapper.
+  class OaiEntry < ActiveRecord::Base
+    belongs_to :oai_token
+
+    validates_uniqueness_of :record_id, :scope => :oai_token
+  end
+  
+  # = OAI::Provider::ActiveRecordCachingWrapper
+  # 
+  # This class wraps an ActiveRecord model and delegates all of the record
+  # selection/retrieval to the AR model.  It accepts options for specifying
+  # the update timestamp field, a timeout, and a limit.  The limit option 
+  # is used for doing pagination with resumption tokens.  The timeout is
+  # used to expire old tokens from the cache.  Default timeout is 12 hours.
+  #
+  # The difference between ActiveRecordWrapper and this class is how the
+  # pagination is accomplished.  ActiveRecordWrapper encodes all the
+  # information in the token.  That approach should work 99% of the time.
+  # If you have an extremely active respository you may want to consider
+  # the caching wrapper.  The caching wrapper takes the entire result set
+  # from a request and caches it in another database table, well tables
+  # actually.  So the result returned to the client will always be 
+  # internally consistent.
+  #
+  class ActiveRecordCachingWrapper < ActiveRecordWrapper
+    
+    attr_reader :model, :timestamp_field, :expire
+    
+    def initialize(model, options={})
+      @expire = options.delete(:timeout) || 12.hours
+      super(model, options)
+    end
+    
+    def find(selector, options={})
+      sweep_cache
+      return next_set(options[:resumption_token]) if options[:resumption_token]
+
+      conditions = sql_conditions(options)
+
+      if :all == selector
+        total = model.count(:id, :conditions => conditions)
+        if @limit && total > @limit
+          select_partial(
+            ResumptionToken.new(options.merge({:last => 0})))
+        else
+          model.find(:all, :conditions => conditions)
+        end
+      else
+        model.find(selector, :conditions => conditions)
+      end
+    end
+  
+    protected 
+  
+    def next_set(token_string)
+      raise ResumptionTokenException.new unless @limit
+
+      token = ResumptionToken.parse(token_string)
+      total = model.count(:id, :conditions => token_conditions(token))
+
+      if token.last * @limit + @limit < total
+        select_partial(token)
+      else 
+        select_partial(token).records
+      end
+    end
+  
+    # select a subset of the result set, and return it with a
+    # resumption token to get the next subset
+    def select_partial(token)
+      if 0 == token.last
+        oaitoken = OaiToken.find_or_create_by_token(token.to_s)
+        if oaitoken.new_record_before_save?
+          OaiToken.connection.execute("insert into " +
+            "#{OaiEntry.table_name} (oai_token_id, record_id) " +
+            "select #{oaitoken.id}, id from #{model.table_name} where " +
+            "#{OaiToken.sanitize_sql(token_conditions(token))}")
+        end
+      end
+      
+      oaitoken = OaiToken.find_by_token(token.to_s)
+      
+      raise ResumptionTokenException.new unless oaitoken
+
+      PartialResult.new(
+        hydrate_records(oaitoken.entries.find(:all, :limit => @limit, 
+          :offset => token.last * @limit)), token.next(token.last + 1)
+      )
+    end
+    
+    def sweep_cache
+      OaiToken.destroy_all(["created_at < ?", Time.now - expire])
+    end
+    
+    def hydrate_records(records)
+      model.find(records.collect {|r| r.record_id })
+    end
+    
+    def token_conditions(token)
+      sql_conditions token.to_conditions_hash
+    end
+    
+    private
+    
+    def expires_at(creation)
+      created = Time.parse(creation.strftime("%Y-%m-%d %H:%M:%S"))
+      created.utc + expire
+    end
+
+  end
+end

data/lib/oai/provider/model/activerecord_wrapper.rb

@@ -0,0 +1,136 @@
+require 'active_record'
+
+module OAI::Provider
+  # = OAI::Provider::ActiveRecordWrapper
+  # 
+  # This class wraps an ActiveRecord model and delegates all of the record
+  # selection/retrieval to the AR model.  It accepts options for specifying
+  # the update timestamp field, a timeout, and a limit.  The limit option 
+  # is used for doing pagination with resumption tokens.  The
+  # expiration timeout is ignored, since all necessary information is
+  # encoded in the token.
+  #
+  class ActiveRecordWrapper < Model
+    
+    attr_reader :model, :timestamp_field
+    
+    def initialize(model, options={})
+      @model = model
+      @timestamp_field = options.delete(:timestamp_field) || 'updated_at'
+      @limit = options.delete(:limit)
+      
+      unless options.empty?
+        raise ArgumentException.new(
+          "Unsupported options [#{options.join(', ')}]"
+        )
+      end
+    end
+    
+    def earliest
+      model.find(:first, 
+        :order => "#{timestamp_field} asc").send(timestamp_field)
+    end
+    
+    def latest
+      model.find(:first, 
+        :order => "#{timestamp_field} desc").send(timestamp_field)
+    end
+
+    # A model class is expected to provide a method Model.sets that
+    # returns all the sets the model supports.  See the 
+    # activerecord_provider tests for an example.   
+    def sets
+      model.sets if model.respond_to?(:sets)
+    end
+    
+    def find(selector, options={})
+      return next_set(options[:resumption_token]) if options[:resumption_token]
+      conditions = sql_conditions(options)
+      
+      if :all == selector
+        total = model.count(:id, :conditions => conditions)
+        if @limit && total > @limit
+          select_partial(ResumptionToken.new(options.merge({:last => 0})))
+        else
+          model.find(:all, :conditions => conditions)
+        end
+      else
+        model.find(selector, :conditions => conditions)
+      end
+    end
+    
+    def deleted?(record)
+      if record.respond_to?(:deleted_at)
+        return record.deleted_at
+      elsif record.respond_to?(:deleted)
+        return record.deleted
+      end
+      false
+    end    
+    
+    protected
+    
+    # Request the next set in this sequence.
+    def next_set(token_string)
+      raise OAI::ResumptionTokenException.new unless @limit
+    
+      token = ResumptionToken.parse(token_string)
+      total = model.count(:id, :conditions => token_conditions(token))
+    
+      if @limit < total
+        select_partial(token)
+      else # end of result set
+        model.find(:all, 
+          :conditions => token_conditions(token), 
+          :limit => @limit, :order => "#{model.primary_key} asc")
+      end
+    end
+    
+    # select a subset of the result set, and return it with a
+    # resumption token to get the next subset
+    def select_partial(token)
+      records = model.find(:all, 
+        :conditions => token_conditions(token),
+        :limit => @limit, 
+        :order => "#{model.primary_key} asc")
+
+      raise OAI::ResumptionTokenException.new unless records
+
+      offset = records.last.send(model.primary_key.to_sym)
+      
+      PartialResult.new(records, token.next(offset))
+    end
+    
+    # build a sql conditions statement from the content
+    # of a resumption token.  It is very important not to
+    # miss any changes as records may change scope as the
+    # harvest is in progress.  To avoid loosing any changes
+    # the last 'id' of the previous set is used as the 
+    # filter to the next set.
+    def token_conditions(token)
+      last = token.last
+      sql = sql_conditions token.to_conditions_hash
+      
+      return sql if 0 == last
+      # Now add last id constraint
+      sql[0] << " AND #{model.primary_key} > ?"
+      sql << last
+      
+      return sql
+    end
+    
+    # build a sql conditions statement from an OAI options hash
+    def sql_conditions(opts)
+      sql = []
+      sql << "#{timestamp_field} >= ?" << "#{timestamp_field} <= ?" 
+      sql << "set = ?" if opts[:set]
+
+      esc_values = [sql.join(" AND ")]
+      esc_values << opts[:from].localtime << opts[:until].localtime
+      esc_values << opts[:set] if opts[:set]
+      
+      return esc_values
+    end
+    
+  end
+end

data/lib/oai/provider/partial_result.rb

@@ -0,0 +1,18 @@
+module OAI::Provider
+  # = OAI::Provider::PartialResult
+  #
+  # PartialResult is used for returning a set/page of results from a model
+  # that supports resumption tokens.  It should contain and array of
+  # records, and a resumption token for getting the next set/page.
+  #
+  class PartialResult
+    attr_reader :records, :token
+    
+    def initialize(records, token = nil)
+      @records = records
+      @token = token
+    end
+    
+  end
+
+end

data/lib/oai/provider/response.rb

@@ -0,0 +1,119 @@
+require 'builder' unless defined?(Builder)
+
+module OAI
+  module Provider
+    module Response
+  
+  class Base
+    attr_reader :provider, :options
+    
+    class << self
+      attr_reader :valid_options, :default_options, :required_options
+
+      def valid_parameters(*args)
+        @valid_options ||= []
+        @valid_options = (@valid_options + args.dup).uniq
+      end
+      
+      def default_parameters(options = {})
+        @default_options ||= {}
+        @default_options.merge! options.dup
+      end
+      
+      def required_parameters(*args)
+        valid_parameters(*args)
+        @required_options ||= []
+        @required_options = (@required_options + args.dup).uniq
+      end
+      
+    end 
+
+    def initialize(provider, options = {})
+      @provider = provider
+      @options = internalize(options)
+      raise OAI::ArgumentException.new unless valid?
+    end
+
+    def response
+      @builder = Builder::XmlMarkup.new
+      @builder.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      @builder.tag!('OAI-PMH', header) do 
+        @builder.responseDate Time.now.utc.xmlschema
+        @builder.request(provider.url, options)
+        yield @builder
+      end
+    end
+
+    private
+    
+    def header
+      { 
+        'xmlns' => "http://www.openarchives.org/OAI/2.0/",
+        'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
+        'xsi:schemaLocation' => %{http://www.openarchives.org/OAI/2.0/
+          http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd}
+      }
+    end
+    
+    def valid?
+      return true if resumption?
+      
+      return true if self.class.valid_options.nil? and options.empty?
+      
+      if self.class.required_options
+        return false unless (self.class.required_options - @options.keys).empty?
+      end
+
+      return false unless (@options.keys - self.class.valid_options).empty?
+
+      populate_defaults
+    end
+    
+    def populate_defaults
+      self.class.default_options.each do |k,v|
+        @options[k] = v.respond_to?(:call) ? v.call(self) : v if not @options[k]
+      end
+    end
+    
+    def resumption?
+      if @options.keys.include?(:resumption_token) 
+        return true if 1 == @options.keys.size
+        raise OAI::ArgumentException.new
+      end
+    end
+        
+    # Convert our internal representations back into standard OAI options
+    def externalize(value)
+      value.to_s.gsub(/_[a-z]/) { |m| m.sub("_", '').capitalize }
+    end
+    
+    def parse_date(value)
+      return value if value.respond_to?(:strftime)
+      
+      # Oddly Chronic doesn't parse an UTC encoded datetime.  
+      # Luckily Time does
+      dt = Chronic.parse(value) || Time.parse(value)
+      raise OAI::ArgumentError.new unless dt
+      
+      dt.utc
+    end
+    
+    def internalize(hash = {})
+      internal = {}
+      hash.keys.each do |key|
+        internal[key.to_s.gsub(/([A-Z])/, '_\1').downcase.intern] = hash[key].dup
+      end
+      
+      # Convert date formated strings into internal time values
+      # Convert date formated strings in dates.
+      internal[:from] = parse_date(internal[:from]) if internal[:from]
+      internal[:until] = parse_date(internal[:until]) if internal[:until]
+      
+      internal
+    end
+    
+  end
+  
+end
+end
+end

data/lib/oai/provider/response/error.rb

@@ -0,0 +1,16 @@
+module OAI::Provider::Response
+  class Error < Base
+    
+    def initialize(provider, error)
+      super(provider)
+      @error = error
+    end
+    
+    def to_xml
+      response do |r|
+        r.error @error.to_s, :code => @error.code
+      end
+    end
+    
+  end
+end

data/lib/oai/provider/response/get_record.rb

@@ -0,0 +1,32 @@
+module OAI::Provider::Response
+  
+  class GetRecord < RecordResponse
+    required_parameters :identifier
+    
+    def to_xml
+      id = extract_identifier(options.delete(:identifier))
+      unless record = provider.model.find(id, options)
+        raise OAI::IdException.new
+      end
+
+      response do |r|
+        r.GetRecord do
+          r.record do 
+            header_for record
+            data_for record unless deleted?(record)
+          end
+        end
+      end
+    end
+    
+    private
+    
+    def extract_identifier(id)
+      id.sub("#{provider.prefix}/", '')
+    end
+    
+  end
+
+end
+    
+