oai_repository 0.1.2 → 0.9.0

data/README.rdoc

@@ -3,3 +3,126 @@
 A Rails (3.1+) engine that allows you to expose your models through an OAI-PHM Data Provider interface.
 
 See http://www.oaforum.org/tutorial/ and http://www.openarchives.org/OAI/openarchivesprotocol.html#Repository
+
+== Installation
+
+If you are using Bundler with your Rails application, then simply add
+
+  gem "oai_repository"
+
+and then run bundle install as usual.
+
+Then run the generator
+
+  $ rails g oai_repository:install
+
+== Configuration
+
+The generator installs a configuration file at <tt>config/initializers/oai_repository.rb</tt>
+
+The following settings should be edited appropriately:
+
+  config.repository_name = 'Test repository'
+  config.repository_url = 'http://localhost:3000/oai_repository'
+  config.record_prefix = 'http://localhost:3000/'
+  config.admin_email = 'root@localhost'
+  config.limit = 100
+  config.models = [ Person, Instrument ]
+
+The values for <tt>config.models</tt> should be the class name of the ActiveRecord model class that is being identified with the given set. It doesn't actually _have_ to be an ActiveRecord model class, but it should act like one. You *must* supply at least one model.
+
+The following settings are optional:
+
+  config.sets = []
+  config.additional_formats = []
+
+The items of the sets list should be hash with value for spec, name, model, and optionally description. E.g.
+
+  config.sets = [
+    {
+      spec: 'class:party',
+      name: 'Parties',
+      model: Person
+    },
+    {
+      spec: 'class:service',
+      name: 'Services',
+      model: Instrument,
+      description: 'Things that are services'
+    }
+  ]
+
+By default, an OAI repository must be able to emit its records in OAI_DC (Dublin Core) format. If you want to provide other output formats for your repository
+(and those formats are subclasses of OAI::Provider::Metadata.Format) then
+you can specify them here. E.g.
+
+  require 'rifcs_format'
+
+  config.additional_formats = [
+    OAI::Provider::Metadata::RIFCS
+  ]
+
+== Instrumenting your Models
+
+=== OAI DC Format
+
+As a *bare* *minimum*, your model classes must implement the following method (or readable attribute)
+
+  oai_dc_identifier
+
+This must return a *unique* value for the *whole* *repository*.
+The format of the unique identifier *must* correspond to that of
+the URI (Uniform Resource Identifier) syntax. See http://www.openarchives.org/OAI/openarchivesprotocol.html#UniqueIdentifier for more details.
+
+You may also supply oai_dc_<value> where <value> is any of
+
+  title
+  creator
+  subject
+  description
+  publisher
+  contributor
+  date
+  type
+  format
+  source
+  language
+  relation
+  coverage
+  rights
+
+See http://www.openarchives.org/OAI/openarchivesprotocol.html#dublincore for a bit more information on the Dublin Core metadata format.
+
+=== OAI Sets
+
+A set is an optional construct for grouping items for the purpose of selective harvesting.
+
+You must fill the configuration item <tt>config.sets</tt> to list the sets your
+repository uses. This list will be shown in the output of a <tt>ListSets</tt> request.
+
+If you are grouping your records by set you have two implementation options in your model(s).
+
+If all records from a model will belong to a given set, then simply
+
+  include OaiRepository::Set
+
+in your model and all records will belong to the sets from your <tt>config.sets</tt> mapping.
+
+If you want to be selective about set membership, implement a <tt>sets</tt> method in your model that responds with the set that a record belongs to. E.g.
+
+  def sets
+    oai_sets = [ OAI::Set.new({name: 'Tools', spec: 'tools'}) ]
+    if name.match('multimeter')
+      oai_sets << OAI::Set.new({name: 'Meters', spec: 'meters'})
+    end
+    oai_sets
+  end
+
+
+== Mounting the Engine
+
+In your <tt>config/routes.rb</tt> add
+
+  mount OaiRepository::Engine => "/oai_repository"
+
+changing the path as desired.

data/lib/ar_wrapper_model.rb

@@ -4,9 +4,14 @@ class ARWrapperModel < OAI::Provider::Model
   def initialize(options={})
     @timestamp_field = options.delete(:timestamp_field) || 'updated_at'
     @limit = options.delete(:limit)
-    @sets_map = options.delete(:sets)
+    @models = options.delete(:models)
+    @sets_map = options.delete(:sets) || []
     @oai_dc_mapping = {}
 
+    if @models.nil? or @models.empty?
+      raise "models configuration value is required and must be non-empty"
+    end
+
     unless options.empty?
       raise ArgumentError.new(
         "Unsupported options [#{options.keys.join(', ')}]"
@@ -38,11 +43,11 @@ class ARWrapperModel < OAI::Provider::Model
   end
 
   def earliest
-    record_end(:asc)
+    first_record_date(:asc)
   end
 
   def latest
-    record_end(:desc)
+    first_record_date(:desc)
   end
 
   # selector can be id or :all
@@ -53,14 +58,13 @@ class ARWrapperModel < OAI::Provider::Model
       token = ResumptionToken.parse(options[:resumption_token])
     end
 
-    from   = token ? token.from   : options[:from]
-    to     = token ? token.until  : options[:until]
+    from   = token ? token.from.utc  : options[:from]
+    to     = token ? token.until.utc : options[:until]
     last   = token ? token.last   : 0
-    prefix = token ? token.prefix : options[:metadata_prefix]
     set    = token ? token.set    : options[:set]
+    prefix = token ? token.prefix : options[:metadata_prefix]
 
-    conditions = sql_conditions(:from => from, :until => to)
-    record_rows = get_record_rows(get_models(set), conditions)
+    record_rows = get_record_rows(set, :from => from, :until => to)
 
     return get_specific_record(record_rows, selector) if selector != :all
 
@@ -75,7 +79,7 @@ class ARWrapperModel < OAI::Provider::Model
 
   def get_specific_record(records, id)
     # TODO: optimise somehow
-    # This is terribly intensive :-(
+    # This scans all records :-(
     records.each do |record|
       obj = Object.const_get(record["type"]).find(record["id"])
       return obj if obj.oai_dc_identifier.eql?(id)
@@ -85,47 +89,51 @@ class ARWrapperModel < OAI::Provider::Model
 
   def get_paged_records(record_rows, options)
 
-    if record_rows.size < options[:last]
+    last = options[:last]
+    if record_rows.size < last
       raise OAI::ResumptionTokenException.new
     end
 
-    list = get_record_objects(record_rows[options[:last], @limit])
+    list = get_record_objects(record_rows[last, @limit])
 
-    if list.size < @limit
+    last += @limit
+    if last >= record_rows.size
       list
     else
-      options[:last] += @limit
+      options[:last] = last
       PartialResult.new(list, ResumptionToken.new(options))
     end
   end
 
-  def sql_conditions(opts)
-    from = Time.parse(opts[:from].to_s).localtime
-    to   = Time.parse(opts[:until].to_s).localtime.to_s
-    return "#{timestamp_field} >= #{ActiveRecord::Base.sanitize(from)} AND #{timestamp_field} <= #{ActiveRecord::Base.sanitize(to)}"
-  end
-
-  def get_models(set)
-    models =
-      if set.nil?
-        @sets_map.map{|s| s[:model]}
-      else
-        @sets_map.select{|s| s[:spec] == set}.map{|s| s[:model]}
+  def get_record_rows(set, options={})
+    union = []
+
+    from = options[:from]
+    # DateTime has microsecond precision, but we're parsing in dates with only
+    # second precision. In this case the microsecond value defaults to zero.
+    # Since some (most) of the records at the boundaries of a range will have
+    # non-zero microseconds (where the timestamp database field type has this 
+    # level of resolution), the range needs to be adjusted to cover this.
+    #
+    # E.g. if the range is 2012-01-01 12:00:00 to 2012-01-01 12:30:00 inclusive
+    # a record with timestamp 2012-01-01 12:30:00.000001 is probably expected
+    # to fall in the range.
+    #
+    # To do this we extend the upper bound by one second and then make this
+    # upper bound exclusive.
+    to   = options[:until] + 1.second
+
+    record_sql = @models.map do |m|
+      res = m.select("id, '#{m.name}' as type, #{timestamp_field}").where("#{timestamp_field} >= ? and #{timestamp_field} < ?", from.to_s(:db), to.to_s(:db))
+      if !(res.empty? or set.nil?)
+        res.select!{|record| record.sets.map(&:spec).include?(set.spec)}
       end
-
-    if models.empty?
-      raise OAI::NoMatchException.new
+      union += res unless res.empty?
     end
-    models
-  end
 
-  def get_record_rows(models, conditions)
-    record_sql = models.map do |m|
-      "select id, '#{m.name}' as type, #{timestamp_field} from #{m.table_name} where #{conditions}"
-    end.join(" UNION ")
+    raise OAI::NoMatchException.new if union.empty?
 
-    sorted_list_sql = "select t.id as id, t.type as type, t.updated_at as updated_at from (#{record_sql}) t order by t.updated_at desc"
-    records = ActiveRecord::Base.connection.execute(sorted_list_sql).to_a
+    union.sort! {|a,b| b.updated_at <=> a.updated_at}
   end
 
   def get_record_objects(records)
@@ -134,10 +142,10 @@ class ARWrapperModel < OAI::Provider::Model
     end
   end
 
-  def record_end(order)
+  def first_record_date(order)
     record = nil
-    @sets_map.each do |s|
-      r = s[:model].first(:order => "#{@timestamp_field} #{order.to_s}")
+    @models.each do |model|
+      r = model.first(:order => "#{@timestamp_field} #{order.to_s}")
       next if r.nil?
 
       if record.nil?
@@ -148,7 +156,9 @@ class ARWrapperModel < OAI::Provider::Model
         record = r
       end
     end
+
     raise OAI::NoMatchException if record.nil?
+
     record.send(@timestamp_field)
   end
 

data/lib/generators/templates/oai_repository.rb

@@ -25,9 +25,15 @@ OaiRepository.setup do |config|
   # The number of records shown at a time (when doing a ListRecords)
   config.limit = 100
 
-  # Map the name of the set to the ActiveRecord (or other) class name that
-  # will provide (at a minimum) the required oai_dc attributes/methods.
-  # E.g.
+  # The values for "models" should be the class name of the ActiveRecord model 
+  # class that is being identified with the given set. It doesn't actually have
+  # to be a ActiveRecord model class, but it should act like one.
+  #
+  # You must supply at least one model.
+  config.models = [ SupplyMe ]
+
+  # List the sets (and the ActiveRecord model they belong to). E.g.
+  #
   # config.sets = [
   #   {
   #     spec: 'class:party',
@@ -37,14 +43,11 @@ OaiRepository.setup do |config|
   #   {
   #     spec: 'class:service',
   #     name: 'Services',
-  #     description: 'Things that are services',
-  #     model: Instrument
+  #     model: Instrument,
+  #     description: 'Things that are services'
   #   }
   # ]
   #
-  # The "model" value should be the class name of the ActiveRecord model class
-  # that is being identified with the given set. It doesn't actually *have*
-  # to be a ActiveRecord model class, but it should act like one.
   config.sets = []
 
   # By default, an OAI repository must emit its records in OAI_DC (Dublin Core)

data/lib/oai_provider.rb

@@ -20,6 +20,7 @@ module OAIProvider
     provider_class.record_prefix OaiRepository.record_prefix
     provider_class.admin_email OaiRepository.admin_email
     provider_class.source_model ARWrapperModel.new(
+      models: OaiRepository.models,
       sets: OaiRepository.sets,
       limit: OaiRepository.limit,
       timestamp_field: OaiRepository.timestamp_field

data/lib/oai_repository.rb

@@ -18,6 +18,9 @@ module OaiRepository
   mattr_accessor :admin_email
   @@admin_email = 'root@localhost'
 
+  mattr_accessor :models
+  @@models = {}
+
   mattr_accessor :sets
   @@sets = {}
 
@@ -30,4 +33,20 @@ module OaiRepository
 
   mattr_accessor :timestamp_field
   @@timestamp_field = 'updated_at'
+
+  module Set
+
+    def sets
+      OaiRepository.sets.select {|s| s[:model] == self.class}.map{|set_obj|
+        OAI::Set.new(
+          {
+            name: set_obj[:name],
+            spec: set_obj[:spec],
+            description: set_obj[:description]
+          }
+        )
+      }
+    end
+
+  end
 end

data/lib/oai_repository/version.rb

@@ -1,3 +1,3 @@
 module OaiRepository
-  VERSION = "0.1.2"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oai_repository
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.9.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-16 00:00:00.000000000 Z
+date: 2012-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
-  requirement: &74598920 !ruby/object:Gem::Requirement
+  requirement: &78869700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,10 @@ dependencies:
         version: '3.1'
   type: :runtime
   prerelease: false
-  version_requirements: *74598920
+  version_requirements: *78869700
 - !ruby/object:Gem::Dependency
   name: oai
-  requirement: &74598390 !ruby/object:Gem::Requirement
+  requirement: &78868860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +33,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *74598390
+  version_requirements: *78868860
 - !ruby/object:Gem::Dependency
   name: sqlite3
-  requirement: &74597970 !ruby/object:Gem::Requirement
+  requirement: &78919450 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +44,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *74597970
+  version_requirements: *78919450
 - !ruby/object:Gem::Dependency
   name: rspec-rails
-  requirement: &74651040 !ruby/object:Gem::Requirement
+  requirement: &78918000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +55,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *74651040
+  version_requirements: *78918000
 - !ruby/object:Gem::Dependency
   name: capybara
-  requirement: &74645980 !ruby/object:Gem::Requirement
+  requirement: &78916060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,7 +66,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *74645980
+  version_requirements: *78916060
 description: An Engine for Rails (3.1+) that allows you to make your application an
   OAI-PMH Data Provider. See http://www.openarchives.org/pmh/ and http://www.openarchives.org/OAI/openarchivesprotocol.html#Repository
 email:
@@ -114,7 +114,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 387383629
+      hash: 493502575
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -123,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 387383629
+      hash: 493502575
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.15