oai_talia 0.0.13

data/lib/oai/provider/model/activerecord_caching_wrapper.rb

@@ -0,0 +1,134 @@
+require 'active_record'
+
+module OAI::Provider
+  
+  # ActiveRecord model class in support of the caching wrapper.
+  class OaiToken < ActiveRecord::Base
+    has_many :entries, :class_name => 'OaiEntry', 
+      :order => "record_id", :dependent => :destroy
+
+    validates_uniqueness_of :token
+    
+    # Make sanitize_sql a public method so we can make use of it.
+    public
+     
+    def self.sanitize_sql(*arg)
+      super(*arg)
+    end
+    
+    def new_record_before_save?
+      @new_record_before_save
+    end
+
+  end
+
+  # ActiveRecord model class in support of the caching wrapper.
+  class OaiEntry < ActiveRecord::Base
+    belongs_to :oai_token
+
+    validates_uniqueness_of :record_id, :scope => :oai_token
+  end
+  
+  # = OAI::Provider::ActiveRecordCachingWrapper
+  # 
+  # This class wraps an ActiveRecord model and delegates all of the record
+  # selection/retrieval to the AR model.  It accepts options for specifying
+  # the update timestamp field, a timeout, and a limit.  The limit option 
+  # is used for doing pagination with resumption tokens.  The timeout is
+  # used to expire old tokens from the cache.  Default timeout is 12 hours.
+  #
+  # The difference between ActiveRecordWrapper and this class is how the
+  # pagination is accomplished.  ActiveRecordWrapper encodes all the
+  # information in the token.  That approach should work 99% of the time.
+  # If you have an extremely active respository you may want to consider
+  # the caching wrapper.  The caching wrapper takes the entire result set
+  # from a request and caches it in another database table, well tables
+  # actually.  So the result returned to the client will always be 
+  # internally consistent.
+  #
+  class ActiveRecordCachingWrapper < ActiveRecordWrapper
+    
+    attr_reader :model, :timestamp_field, :expire
+    
+    def initialize(model, options={})
+      @expire = options.delete(:timeout) || 12.hours
+      super(model, options)
+    end
+    
+    def find(selector, options={})
+      sweep_cache
+      return next_set(options[:resumption_token]) if options[:resumption_token]
+
+      conditions = sql_conditions(options)
+
+      if :all == selector
+        total = model.count(:id, :conditions => conditions)
+        if @limit && total > @limit
+          select_partial(
+            ResumptionToken.new(options.merge({:last => 0})))
+        else
+          model.find(:all, :conditions => conditions)
+        end
+      else
+        model.find(selector, :conditions => conditions)
+      end
+    end
+  
+    protected 
+  
+    def next_set(token_string)
+      raise ResumptionTokenException.new unless @limit
+
+      token = ResumptionToken.parse(token_string)
+      total = model.count(:id, :conditions => token_conditions(token))
+
+      if token.last * @limit + @limit < total
+        select_partial(token)
+      else 
+        select_partial(token).records
+      end
+    end
+  
+    # select a subset of the result set, and return it with a
+    # resumption token to get the next subset
+    def select_partial(token)
+      if 0 == token.last
+        oaitoken = OaiToken.find_or_create_by_token(token.to_s)
+        if oaitoken.new_record_before_save?
+          OaiToken.connection.execute("insert into " +
+            "#{OaiEntry.table_name} (oai_token_id, record_id) " +
+            "select #{oaitoken.id}, id from #{model.table_name} where " +
+            "#{OaiToken.sanitize_sql(token_conditions(token))}")
+        end
+      end
+      
+      oaitoken = OaiToken.find_by_token(token.to_s)
+      raise ResumptionTokenException.new unless oaitoken
+
+      PartialResult.new(
+        hydrate_records(oaitoken.entries.find(:all, :limit => @limit, 
+          :offset => token.last * @limit)), token.next(token.last + 1)
+      )
+    end
+    
+    def sweep_cache
+      OaiToken.destroy_all(["created_at < ?", Time.now - expire])
+    end
+    
+    def hydrate_records(records)
+      model.find(records.collect {|r| r.record_id })
+    end
+    
+    def token_conditions(token)
+      sql_conditions token.to_conditions_hash
+    end
+    
+    private
+    
+    def expires_at(creation)
+      created = Time.parse(creation.strftime("%Y-%m-%d %H:%M:%S"))
+      created.utc + expire
+    end
+
+  end
+end

data/lib/oai/provider/model/activerecord_wrapper.rb

@@ -0,0 +1,139 @@
+require 'active_record'
+module OAI::Provider
+  # = OAI::Provider::ActiveRecordWrapper
+  # 
+  # This class wraps an ActiveRecord model and delegates all of the record
+  # selection/retrieval to the AR model.  It accepts options for specifying
+  # the update timestamp field, a timeout, and a limit.  The limit option 
+  # is used for doing pagination with resumption tokens.  The
+  # expiration timeout is ignored, since all necessary information is
+  # encoded in the token.
+  #
+  class ActiveRecordWrapper < Model
+    
+    attr_reader :model, :timestamp_field
+    
+    def initialize(model, options={})
+      @model = model
+      @timestamp_field = options.delete(:timestamp_field) || 'updated_at'
+      @limit = options.delete(:limit)
+      
+      unless options.empty?
+        raise ArgumentError.new(
+          "Unsupported options [#{options.keys.join(', ')}]"
+        )
+      end
+    end
+    
+    def earliest
+      model.find(:first, 
+        :order => "#{timestamp_field} asc").send(timestamp_field)
+    end
+    
+    def latest
+      model.find(:first, 
+        :order => "#{timestamp_field} desc").send(timestamp_field)
+    end
+    # A model class is expected to provide a method Model.sets that
+    # returns all the sets the model supports.  See the 
+    # activerecord_provider tests for an example.   
+    def sets
+      model.sets if model.respond_to?(:sets)
+    end
+    
+    def find(selector, options={})
+      return next_set(options[:resumption_token]) if options[:resumption_token]
+      conditions = sql_conditions(options)
+      if :all == selector
+        total = model.count(:id, :conditions => conditions)
+        if @limit && total > @limit
+          select_partial(ResumptionToken.new(options.merge({:last => 0})))
+        else
+          model.find(:all, :conditions => conditions)
+        end
+      else
+        begin
+          model.find(selector, :conditions => conditions)
+        rescue ActiveRecord::RecordNotFound
+          raise OAI::IdException.new
+        end
+      end
+    end
+    
+    def deleted?(record)
+      if record.respond_to?(:deleted_at)
+        return record.deleted_at
+      elsif record.respond_to?(:deleted)
+        return record.deleted
+      end
+      false
+    end    
+    
+    protected
+    
+    # Request the next set in this sequence.
+    def next_set(token_string)
+      raise OAI::ResumptionTokenException.new unless @limit
+    
+      token = ResumptionToken.parse(token_string)
+      total = model.count(:id, :conditions => token_conditions(token))
+    
+      if @limit < total
+        select_partial(token)
+      else # end of result set
+        model.find(:all, 
+          :conditions => token_conditions(token), 
+          :limit => @limit, :order => "#{model.primary_key} asc")
+      end
+    end
+    
+    # select a subset of the result set, and return it with a
+    # resumption token to get the next subset
+    def select_partial(token)
+      records = model.find(:all, 
+        :conditions => token_conditions(token),
+        :limit => @limit, 
+        :order => "#{model.primary_key} asc")
+      raise OAI::ResumptionTokenException.new unless records
+      offset = records.last.send(model.primary_key.to_sym)
+      
+      PartialResult.new(records, token.next(offset))
+    end
+    
+    # build a sql conditions statement from the content
+    # of a resumption token.  It is very important not to
+    # miss any changes as records may change scope as the
+    # harvest is in progress.  To avoid loosing any changes
+    # the last 'id' of the previous set is used as the 
+    # filter to the next set.
+    def token_conditions(token)
+      last = token.last
+      sql = sql_conditions token.to_conditions_hash
+      
+      return sql if 0 == last
+      # Now add last id constraint
+      sql[0] << " AND #{model.primary_key} > ?"
+      sql << last
+      
+      return sql
+    end
+    
+    # build a sql conditions statement from an OAI options hash
+    def sql_conditions(opts)
+      sql = []
+      sql << "#{timestamp_field} >= ?" << "#{timestamp_field} <= ?" 
+      sql << "set = ?" if opts[:set]
+      esc_values = [sql.join(" AND ")]
+      esc_values << get_time(opts[:from]).localtime << get_time(opts[:until]).localtime #-- OAI 2.0 hack - UTC fix from record_responce 
+      esc_values << opts[:set] if opts[:set]
+      
+      return esc_values
+    end
+    
+    def get_time(time)
+      time.kind_of?(Time) ? time : Time.parse(time)
+    end
+    
+  end
+end
+

data/lib/oai/provider/model.rb

@@ -0,0 +1,74 @@
+module OAI::Provider
+  # = OAI::Provider::Model
+  #
+  # Model implementers should subclass OAI::Provider::Model and override 
+  # Model#earliest, Model#latest, and Model#find.  Optionally Model#sets and
+  # Model#deleted? can be used to support sets and record deletions.  It 
+  # is also the responsibility of the model implementer to account for 
+  # resumption tokens if support is required.  Models that don't support 
+  # resumption tokens should raise an exception if a limit is requested     
+  # during initialization.
+  #
+  # earliest - should return the earliest update time in the repository.
+  # latest - should return the most recent update time in the repository.
+  # sets - should return an array of sets supported by the repository.
+  # deleted? - individual records returned should respond true or false
+  # when sent the deleted? message.
+  # available_formats - if overridden, individual records should return an 
+  # array of prefixes for all formats in which that record is available, 
+  # if other than ["oai_dc"]
+  #
+  # == Resumption Tokens
+  #
+  # For examples of using resumption tokens see the
+  # ActiveRecordWrapper, and ActiveRecordCachingWrapper classes.
+  #
+  # There are several helper models for dealing with resumption tokens please
+  # see the ResumptionToken class for more details.
+  #
+
+  class Model
+    attr_reader :timestamp_field
+    
+    def initialize(limit = nil, timestamp_field = 'updated_at')
+      @limit = limit
+      @timestamp_field = timestamp_field
+    end
+
+    # should return the earliest timestamp available from this model.
+    def earliest
+      raise NotImplementedError.new
+    end
+    
+    # should return the latest timestamp available from this model.
+    def latest
+      raise NotImplementedError.new
+    end
+    
+    def sets
+      nil
+    end
+  
+    # find is the core method of a model, it returns records from the model
+    # bases on the parameters passed in.
+    #
+    # <tt>selector</tt> can be a singular id, or the symbol :all
+    # <tt>options</tt> is a hash of options to be used to constrain the query.
+    #
+    # Valid options:
+    # * :from => earliest timestamp to be included in the results
+    # * :until => latest timestamp to be included in the results
+    # * :set => the set from which to retrieve the results
+    # * :metadata_prefix => type of metadata requested (this may be useful if 
+    #                       not all records are available in all formats)
+    def find(selector, options={})
+      raise NotImplementedError.new
+    end
+    
+    def deleted?
+      false
+    end
+    
+  end
+  
+end

data/lib/oai/provider/partial_result.rb

@@ -0,0 +1,18 @@
+module OAI::Provider
+  # = OAI::Provider::PartialResult
+  #
+  # PartialResult is used for returning a set/page of results from a model
+  # that supports resumption tokens.  It should contain and array of
+  # records, and a resumption token for getting the next set/page.
+  #
+  class PartialResult
+    attr_reader :records, :token
+    
+    def initialize(records, token = nil)
+      @records = records
+      @token = token
+    end
+    
+  end
+
+end

data/lib/oai/provider/response/error.rb

@@ -0,0 +1,16 @@
+module OAI::Provider::Response
+  class Error < Base
+    
+    def initialize(provider, error)
+      super(provider)
+      @error = error
+    end
+    
+    def to_xml
+      response do |r|
+        r.error @error.to_s, :code => @error.code
+      end
+    end
+    
+  end
+end

data/lib/oai/provider/response/get_record.rb

@@ -0,0 +1,26 @@
+module OAI::Provider::Response
+  
+  class GetRecord < RecordResponse
+    required_parameters :identifier, :metadata_prefix
+    
+    def to_xml
+      id = extract_identifier(options.delete(:identifier))
+      unless record = provider.model.find(id, options)
+        raise OAI::IdException.new
+      end
+
+      response do |r|
+        r.GetRecord do
+          r.record do 
+            header_for record
+            data_for record unless deleted?(record)
+          end
+        end
+      end
+    end
+        
+  end
+
+end
+    
+    

data/lib/oai/provider/response/identify.rb

@@ -0,0 +1,25 @@
+module OAI::Provider::Response
+  
+  class Identify < Base
+    
+    def to_xml
+      response do |r|
+        r.Identify do
+          r.repositoryName provider.name
+          r.baseURL provider.url
+          r.protocolVersion 2.0
+          if provider.email and provider.email.respond_to?(:each)
+            provider.email.each { |address| r.adminEmail address }
+          else
+            r.adminEmail provider.email.to_s
+          end
+          r.earliestDatestamp Time.parse(provider.model.earliest.to_s).utc.xmlschema
+          r.deletedRecord provider.delete_support.to_s
+          r.granularity provider.granularity
+        end
+      end
+    end
+    
+  end
+  
+end

data/lib/oai/provider/response/list_identifiers.rb

@@ -0,0 +1,35 @@
+module OAI::Provider::Response
+  
+  class ListIdentifiers < RecordResponse
+    required_parameters :metadata_prefix
+    
+    def to_xml
+      result = provider.model.find(:all, options)
+
+      # result may be an array of records, or a partial result
+      records = result.respond_to?(:records) ? result.records : result
+      
+      raise OAI::NoMatchException.new if records.nil? or records.empty?
+      format = requested_format # not call this in each iteration
+      records.reject! do |r| 
+        !record_supports(r, format)
+      end
+      
+      response do |r|
+        r.ListIdentifiers do
+          records.each do |rec|
+            header_for rec
+          end
+
+          # append resumption token for getting next group of records
+          if result.respond_to?(:token)
+            r.target! << result.token.to_xml
+          end
+
+        end
+      end
+    end
+    
+  end
+  
+end

data/lib/oai/provider/response/list_metadata_formats.rb

@@ -0,0 +1,34 @@
+module OAI::Provider::Response
+  class ListMetadataFormats < RecordResponse
+    valid_parameters :identifier
+    
+    def to_xml
+      # Get a list of all the formats the provider understands.
+      formats = provider.formats.values
+      
+      # if it's a doc-specific request
+      if options.include?(:identifier)
+        id = extract_identifier(options[:identifier])
+        unless record = provider.model.find(id, options)
+          raise OAI::IdException.new
+        end
+      
+        # Remove any format that this particular record can't be provided in.
+        formats.reject! { |f| !record_supports(record, f.prefix) }
+      end
+      response do |r|
+        r.ListMetadataFormats do
+          formats.each do |format|
+            r.metadataFormat do 
+              r.metadataPrefix format.prefix
+              r.schema format.schema
+              r.metadataNamespace format.namespace
+            end
+          end
+        end
+      end
+    end
+    
+    
+  end  
+end

data/lib/oai/provider/response/list_records.rb

@@ -0,0 +1,34 @@
+module OAI::Provider::Response
+
+  class ListRecords < RecordResponse
+    required_parameters :metadata_prefix
+    
+    def to_xml
+      result = provider.model.find(:all, options)
+      # result may be an array of records, or a partial result
+      records = result.respond_to?(:records) ? result.records : result
+
+      raise OAI::NoMatchException.new if records.nil? or records.empty?
+
+      response do |r|
+        r.ListRecords do
+          records.each do |rec|
+            r.record do
+              header_for rec
+              data_for rec unless deleted?(rec)
+            end
+          end
+
+          # append resumption token for getting next group of records
+          if result.respond_to?(:token)
+            r.target! << result.token.to_xml
+          end
+
+        end
+      end
+    end
+    
+  end
+  
+end
+  

data/lib/oai/provider/response/list_sets.rb

@@ -0,0 +1,23 @@
+module OAI::Provider::Response
+  
+  class ListSets < Base
+    
+    def to_xml
+      raise OAI::SetException.new unless provider.model.sets
+
+      response do |r|
+        r.ListSets do
+          provider.model.sets.each do |set|
+            r.set do
+              r.setSpec set.spec
+              r.setName set.name
+              r.setDescription(set.description) if set.respond_to?(:description)
+            end
+          end
+        end
+      end
+    end
+
+  end  
+
+end

data/lib/oai/provider/response/record_response.rb

@@ -0,0 +1,70 @@
+module OAI::Provider::Response
+  class RecordResponse < Base
+    def self.inherited(klass)
+      klass.valid_parameters    :metadata_prefix, :from, :until, :set
+      klass.default_parameters  :from => Proc.new {|x| Time.parse(x.provider.model.earliest.to_s) },
+            :until => Proc.new {|x| Time.parse(x.provider.model.latest.to_s) }
+    end
+    
+    # emit record header
+    def header_for(record)
+      param = Hash.new
+      param[:status] = 'deleted' if deleted?(record)
+      @builder.header param do 
+        @builder.identifier identifier_for(record)
+        @builder.datestamp timestamp_for(record)
+        sets_for(record).each do |set|
+          @builder.setSpec set.spec
+        end
+      end
+    end
+    # metadata - core routine for delivering metadata records
+    #
+    def data_for(record)
+      @builder.metadata do
+        @builder.target! << provider.format(requested_format).encode(provider.model, record)
+      end
+    end
+    
+    private
+    
+    def identifier_for(record)
+      "#{provider.prefix}/#{record.id}"
+    end
+    
+    def timestamp_for(record)
+      record.send(provider.model.timestamp_field).utc.xmlschema
+    end
+    
+    def sets_for(record)
+      return [] unless record.respond_to?(:sets) and record.sets
+      record.sets.respond_to?(:each) ? record.sets : [record.sets]
+    end
+    
+    def requested_format
+      format = 
+      if options[:metadata_prefix]
+        options[:metadata_prefix]
+      elsif options[:resumption_token]
+        OAI::Provider::ResumptionToken.extract_format(options[:resumption_token])
+      end
+      raise OAI::FormatException.new unless provider.format_supported?(format)
+      
+      format
+    end
+    
+    def deleted?(record)
+      return record.deleted? if record.respond_to?(:deleted?)
+      return record.deleted if record.respond_to?(:deleted)
+      return record.deleted_at if record.respond_to?(:deleted_at)
+      false
+    end
+    
+    def record_supports(record, prefix)
+      prefix == 'oai_dc' or 
+      record.respond_to?("to_#{prefix}") or
+      record.respond_to?("map_#{prefix}")
+  end
+    
+end
+end