oai 0.0.1 → 1.3.0

data/lib/oai/provider.rb

@@ -0,0 +1,487 @@
+require 'rexml/document'
+require 'singleton'
+require 'builder'
+
+if not defined?(OAI::Const::VERBS)
+  require 'oai/exception'
+  require 'oai/constants'
+  require 'oai/xpath'
+  require 'oai/set'
+end
+
+%w{ response metadata_format resumption_token model partial_result
+    response/record_response response/identify response/get_record
+    response/list_identifiers response/list_records
+    response/list_metadata_formats response/list_sets response/error
+  }.each { |lib| require File.dirname(__FILE__) + "/provider/#{lib}" }
+
+if defined?(ActiveRecord)
+  require File.dirname(__FILE__) + "/provider/model/activerecord_wrapper"
+  require File.dirname(__FILE__) + "/provider/model/activerecord_caching_wrapper"
+end
+
+# # OAI::Provider
+#
+# Open Archives Initiative - Protocol for Metadata Harvesting see
+# <http://www.openarchives.org/>
+#
+# ## Features
+# * Easily setup a simple repository
+# * Simple integration with ActiveRecord
+# * Dublin Core metadata format included
+# * Easily add addition metadata formats
+# * Adaptable to any data source
+# * Simple resumption token support
+#
+# ## Usage
+#
+# To create a functional provider either subclass {OAI::Provider::Base},
+# or reconfigure the defaults.
+#
+# ### Sub classing a provider
+#
+# ```ruby
+#  class MyProvider < Oai::Provider
+#    repository_name 'My little OAI provider'
+#    repository_url  'http://localhost/provider'
+#    record_prefix 'oai:localhost'
+#    admin_email 'root@localhost'   # String or Array
+#    source_model MyModel.new       # Subclass of OAI::Provider::Model
+#  end
+# ```
+#
+# ### Configuring the default provider
+#
+# ```ruby
+#  class Oai::Provider::Base
+#    repository_name 'My little OAI Provider'
+#    repository_url 'http://localhost/provider'
+#    record_prefix 'oai:localhost'
+#    admin_email 'root@localhost'
+#    # record_prefix will be automatically prepended to sample_id, so in this
+#    # case it becomes: oai:localhost:13900
+#    sample_id '13900'
+#    source_model MyModel.new
+#  end
+# ```
+#
+# The provider does allow a URL to be passed in at request processing time
+# in case the repository URL cannot be determined ahead of time.
+#
+# ## Integrating with frameworks
+#
+# ### Camping
+#
+# In the Models module of your camping application post model definition:
+#
+# ```ruby
+#   class CampingProvider < OAI::Provider::Base
+#     repository_name 'Camping Test OAI Repository'
+#     source_model ActiveRecordWrapper.new(YOUR_ACTIVE_RECORD_MODEL)
+#   end
+# ```
+#
+# In the Controllers module:
+#
+# ```ruby
+#   class Oai
+#     def get
+#       @headers['Content-Type'] = 'text/xml'
+#       provider = Models::CampingProvider.new
+#       provider.process_request(@input.merge(:url => "http:"+URL(Oai).to_s))
+#     end
+#   end
+# ```
+#
+# The provider will be available at "/oai"
+#
+# ### Rails
+#
+# At the bottom of environment.rb create a OAI Provider:
+#
+# ```ruby
+#   # forgive the standard blog example.
+#
+#   require 'oai'
+#   class BlogProvider < OAI::Provider::Base
+#     repository_name 'My little OAI Provider'
+#     repository_url 'http://localhost:3000/provider'
+#     record_prefix 'oai:blog'
+#     admin_email 'root@localhost'
+#     source_model OAI::Provider::ActiveRecordWrapper.new(Post)
+#     sample_id '13900' # record prefix used, so becomes oai:blog:13900
+#   end
+# ```
+#
+# Create a custom controller:
+#
+# ```ruby
+#   class OaiController < ApplicationController
+#     def index
+#       provider = BlogProvider.new
+#       response =  provider.process_request(oai_params.to_h)
+#       render :body => response, :content_type => 'text/xml'
+#     end
+#
+#     private
+#
+#     def oai_params
+#       params.permit(:verb, :identifier, :metadataPrefix, :set, :from, :until, :resumptionToken)
+#     end
+#   end
+# ```
+#
+# And route to it in your `config/routes.rb` file:
+#
+# ```ruby
+#    match 'oai', to: "oai#index", via: [:get, :post]
+# ```
+#
+# Special thanks to Jose Hales-Garcia for this solution.
+#
+# ### Leverage the Provider instance
+#
+# The traditional implementation of the OAI::Provider would pass the OAI::Provider
+# class to the different resposnes. This made it hard to inject context into a
+# common provider. Consider that we might have different request headers that
+# change the scope of the OAI::Provider queries.
+#
+# ```ruby
+# class InstanceProvider
+#   def initialize(options = {})
+#     super({ :provider_context => :instance_based })
+#     @controller = options.fetch(:controller)
+#   end
+#   attr_reader :controller
+# end
+#
+# class OaiController < ApplicationController
+#   def index
+#     provider = InstanceProvider.new({ :controller => self })
+#     request_body =  provider.process_request(oai_params.to_h)
+#     render :body => request_body, :content_type => 'text/xml'
+#   end
+# ```
+#
+# In the above example, the underlying response object will now receive an
+# instance of the InstanceProvider. Without the `super({ :provider_context => :instance_based })`
+# the response objects would have received the class InstanceProvider as the
+# given provider.
+#
+# ## Supporting custom metadata formats
+#
+# See {OAI::MetadataFormat} for details.
+#
+# ## ActiveRecord Integration
+#
+# ActiveRecord integration is provided by the `ActiveRecordWrapper` class.
+# It takes one required paramater, the class name of the AR class to wrap,
+# and optional hash of options.
+#
+# As of `oai` gem 1.0.0, Rails 5.2.x and Rails 6.0.x are supported.
+# Please check the .travis.yml file at root of repo to see what versions of ruby/rails
+# are being tested, in case this is out of date.
+#
+# Valid options include:
+#
+# * `timestamp_field` - Specifies the model field/method to use as the update
+#   filter.  Defaults to `updated_at`.
+# * `identifier_field` -- specifies the model field/method to use to get value to use
+#    as oai identifier (method return value should not include prefix)
+# * `limit` -           Maximum number of records to return in each page/set.
+#   Defaults to 100, set to `nil` for all records in one page. Otherwise
+#   the wrapper will paginate the result via resumption tokens.
+#   _Caution:  specifying too large a limit will adversely affect performance._
+#
+# Mapping from a ActiveRecord object to a specific metadata format follows
+# this set of rules:
+#
+# 1. Does `Model#to_{metadata_prefix}` exist?  If so just return the result.
+# 2. Does the model provide a map via `Model.map_{metadata_prefix}`?  If so
+#    use the map to generate the xml document.
+# 3. Loop thru the fields of the metadata format and check to see if the
+#    model responds to either the plural, or singular of the field.
+#
+# For maximum control of the xml metadata generated, it's usually best to
+# provide a `to_{metadata_prefix}` in the model.  If using Builder be sure
+# not to include any `instruct!` in the xml object.
+#
+# ### Explicit creation example
+#
+# ```ruby
+#  class Post < ActiveRecord::Base
+#    def to_oai_dc
+#      xml = Builder::XmlMarkup.new
+#      xml.tag!("oai_dc:dc",
+#        'xmlns:oai_dc' => "http://www.openarchives.org/OAI/2.0/oai_dc/",
+#        'xmlns:dc' => "http://purl.org/dc/elements/1.1/",
+#        'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
+#        'xsi:schemaLocation' =>
+#          %{http://www.openarchives.org/OAI/2.0/oai_dc/
+#            http://www.openarchives.org/OAI/2.0/oai_dc.xsd}) do
+#          xml.tag!('oai_dc:title', title)
+#          xml.tag!('oai_dc:description', text)
+#          xml.tag!('oai_dc:creator', user)
+#          tags.each do |tag|
+#            xml.tag!('oai_dc:subject', tag)
+#          end
+#      end
+#      xml.target!
+#    end
+#  end
+# ```
+#
+# ### Mapping Example
+#
+# ```ruby
+#  # Extremely contrived mapping
+#  class Post < ActiveRecord::Base
+#    def self.map_oai_dc
+#      {:subject => :tags,
+#       :description => :text,
+#       :creator => :user,
+#       :contibutor => :comments}
+#    end
+#  end
+# ```
+# ### Scopes for restrictions or eager-loading
+#
+# Instead of passing in a Model class to OAI::Provider::ActiveRecordWrapper, you can actually
+# pass in any scope (or ActiveRecord::Relation). This means you can use it for restrictions:
+#
+#     OAI::Provider::ActiveRecordWrapper.new(Post.where(published: true))
+#
+# Or eager-loading an association you will need to create serialization, to avoid n+1 query
+# performance problems:
+#
+#     OAI::Provider::ActiveRecordWrapper.new(Post.includes(:categories))
+#
+# Or both of those in combination, or anything else that returns an ActiveRecord::Relation,
+# including using custom scopes, etc.
+#
+# ### Sets?
+#
+# There is some code written to support oai-pmh "sets" in the ActiveRecord::Wrapper, but
+# it's somewhat inflexible, and not well-documented, and as I write this I don't understand
+# it enough to say more. See https://github.com/code4lib/ruby-oai/issues/67
+#
+module OAI::Provider
+  class Base
+    include OAI::Provider
+
+    class << self
+      attr_reader :formats
+      attr_accessor :name, :url, :prefix, :email, :delete_support, :granularity, :model, :identifier, :description
+
+      def register_format(format)
+        @formats ||= {}
+        @formats[format.prefix] = format
+      end
+
+      def format_supported?(prefix)
+        @formats.keys.include?(prefix)
+      end
+
+      def format(prefix)
+        if @formats[prefix].nil?
+          raise OAI::FormatException.new
+        else
+          @formats[prefix]
+        end
+      end
+
+      protected
+
+      def inherited(klass)
+        self.instance_variables.each do |iv|
+          klass.instance_variable_set(iv, self.instance_variable_get(iv))
+        end
+      end
+
+      alias_method :repository_name,    :name=
+      alias_method :repository_url,     :url=
+      alias_method :record_prefix,      :prefix=
+      alias_method :admin_email,        :email=
+      alias_method :deletion_support,   :delete_support=
+      alias_method :update_granularity, :granularity=
+      alias_method :source_model,       :model=
+      alias_method :sample_id,          :identifier=
+      alias_method :extra_description,  :description=
+
+    end
+
+    # Default configuration of a repository
+    Base.repository_name 'Open Archives Initiative Data Provider'
+    Base.repository_url 'unknown'
+    Base.record_prefix 'oai:localhost'
+    Base.admin_email 'nobody@localhost'
+    Base.deletion_support OAI::Const::Delete::TRANSIENT
+    Base.update_granularity OAI::Const::Granularity::HIGH
+    Base.sample_id '13900'
+
+    Base.register_format(OAI::Provider::Metadata::DublinCore.instance)
+
+    PROVIDER_CONTEXTS = {
+      :class_based => :class_based,
+      :instance_based => :instance_based
+    }
+
+    def initialize(options = {})
+      provider_context = options.fetch(:provider_context) { :class_based }
+      @provider_context = PROVIDER_CONTEXTS.fetch(provider_context)
+    end
+
+    # @note These are the accessor methods on the class. If you need to overwrite
+    # them on the instance level you can do that. However, an instance of this
+    # class won't be used unless you initialize with:
+    # { :provider_context => :instance_based }
+    attr_writer :name, :url, :prefix, :email, :delete_support, :granularity, :model, :identifier, :description
+
+    # The traditional interaction of a Provider has been to:
+    #
+    # 1) Assign attributes to the Provider class
+    # 2) Instantiate the Provider class
+    # 3) Call response instance methods for theProvider which pass
+    #    the Provider class and not the instance.
+    #
+    # The above behavior continues unless you initialize the Provider with
+    # { :provider_context => :instance_based }. If you do that, then the
+    # Provider behavior will be:
+    #
+    # 1) Assign attributes to Provider class
+    # 2) Instantiate the Provider class
+    # 3) Call response instance methods for theProvider which pass an
+    #    instance of the Provider to those response objects.
+    #    a) The instance will mirror all of the assigned Provider class
+    #       attributes, but allows for overriding and extending on a
+    #       case by case basis.
+    # (Dear reader, please note the second behavior is something most
+    # of us would've assumed to be the case, but for historic now lost
+    # reasons is not the case.)
+    def provider_context
+      if @provider_context == :instance_based
+        self
+      else
+        self.class
+      end
+    end
+
+    def format_supported?(*args)
+      self.class.format_supported?(*args)
+    end
+
+    def format(*args)
+      self.class.format(*args)
+    end
+
+    def formats
+      self.class.formats
+    end
+
+    def name
+      @name || self.class.name
+    end
+
+    def url
+      @url || self.class.url
+    end
+
+    def prefix
+      @prefix || self.class.prefix
+    end
+
+    def email
+      @email || self.class.email
+    end
+
+    def delete_support
+      @delete_support || self.class.delete_support
+    end
+
+    def granularity
+      @granularity || self.class.granularity
+    end
+
+    def model
+      @model || self.class.model
+    end
+
+    def identifier
+      @identifier || self.class.identifier
+    end
+
+    def description
+      @description || self.class.description
+    end
+
+    # Equivalent to '&verb=Identify', returns information about the repository
+    def identify(options = {})
+      Response::Identify.new(provider_context, options).to_xml
+    end
+
+    # Equivalent to '&verb=ListSets', returns a list of sets that are supported
+    # by the repository or an error if sets are not supported.
+    def list_sets(options = {})
+      Response::ListSets.new(provider_context, options).to_xml
+    end
+
+    # Equivalent to '&verb=ListMetadataFormats', returns a list of metadata formats
+    # supported by the repository.
+    def list_metadata_formats(options = {})
+      Response::ListMetadataFormats.new(provider_context, options).to_xml
+    end
+
+    # Equivalent to '&verb=ListIdentifiers', returns a list of record headers that
+    # meet the supplied criteria.
+    def list_identifiers(options = {})
+      Response::ListIdentifiers.new(provider_context, options).to_xml
+    end
+
+    # Equivalent to '&verb=ListRecords', returns a list of records that meet the
+    # supplied criteria.
+    def list_records(options = {})
+      Response::ListRecords.new(provider_context, options).to_xml
+    end
+
+    # Equivalent to '&verb=GetRecord', returns a record matching the required
+    # :identifier option
+    def get_record(options = {})
+      Response::GetRecord.new(provider_context, options).to_xml
+    end
+
+    #  xml_response = process_verb('ListRecords', :from => 'October 1, 2005',
+    #    :until => 'November 1, 2005')
+    #
+    # If you are implementing a web interface using process_request is the
+    # preferred way.
+    def process_request(params = {})
+      begin
+
+        # Allow the request to pass in a url
+        provider_context.url = params['url'] ? params.delete('url') : self.url
+
+        verb = params.delete('verb') || params.delete(:verb)
+
+        unless verb and OAI::Const::VERBS.keys.include?(verb)
+          raise OAI::VerbException.new
+        end
+
+        send(methodize(verb), params)
+
+      rescue => err
+        if err.respond_to?(:code)
+          Response::Error.new(self.class, err).to_xml
+        else
+          raise err
+        end
+      end
+    end
+
+    # Convert valid OAI-PMH verbs into ruby method calls
+    def methodize(verb)
+      verb.gsub(/[A-Z]/) {|m| "_#{m.downcase}"}.sub(/^\_/,'')
+    end
+
+  end
+
+end

data/lib/oai/set.rb

@@ -7,12 +7,21 @@ module OAI
     include OAI::XPath
     attr_accessor :name, :spec, :description
 
-    def initialize(element)
-      @name = xpath(element, './/setName')
-      @spec = xpath(element, './/setSpec')
-      @description = xpath_first(element, './/setDescription')
+    def initialize(values = {})
+      @name = values.delete(:name)
+      @spec = values.delete(:spec)
+      @description = values.delete(:description)
+      raise ArgumentException, "Invalid options" unless values.empty?
     end
-
+    
+    def self.parse(element)
+      set = self.new
+      set.name = set.xpath(element, './/setName')
+      set.spec = set.xpath(element, './/setSpec')
+      set.description = set.xpath_first(element, './/setDescription')
+      set
+    end
+    
     def to_s
       "#{@name} [#{@spec}]"
     end

data/lib/oai/xpath.rb

@@ -1,21 +1,75 @@
-require 'rexml/xpath'
-
 module OAI
   module XPath
+
+    # get all matching nodes
     def xpath_all(doc, path)
-      return REXML::XPath.match(doc, path)
+      case parser_type(doc)
+      when 'libxml'
+        return doc.find(path).to_a if doc.find(path)
+      when 'rexml'
+        return REXML::XPath.match(doc, path)
+      end
+      return []
     end
 
+    # get first matching node
     def xpath_first(doc, path)
       elements = xpath_all(doc, path)
       return elements[0] if elements != nil
       return nil
     end
 
+    # get text for first matching node
     def xpath(doc, path)
-      e = xpath_first(doc, path)
-      return e.text if e != nil
+      el = xpath_first(doc, path)
+      return unless el
+      case parser_type(doc)
+      when 'libxml'
+        return el.content
+      when 'rexml'
+        return el.text 
+      end
       return nil
     end
+
+    # figure out an attribute
+    def get_attribute(node, attr_name)
+      case node.class.to_s
+      when 'REXML::Element'
+        return node.attribute(attr_name)
+      when 'LibXML::XML::Node'
+  	#There has been a method shift between 0.5 and 0.7
+        if defined?(node.property) == nil
+          return node.attributes[attr_name]
+        else
+	  #node.property is being deprecated.  We'll eventually remove
+	  #this trap
+	  begin
+	    return node[attr_name]
+          rescue
+             return node.property(attr_name)
+          end
+        end	
+      end
+      return nil
+    end
+
+    private 
+   
+    # figure out what sort of object we should do xpath on
+    def parser_type(x)
+      case x.class.to_s
+      when 'LibXML::XML::Document'
+        return 'libxml'
+      when 'LibXML::XML::Node'
+        return 'libxml'
+      when 'LibXML::XML::Node::Set'
+	return 'libxml'
+      when 'REXML::Element'
+        return 'rexml'
+      when 'REXML::Document'
+        return 'rexml'
+      end
+    end
   end
 end

data/lib/oai.rb

@@ -1,14 +1,8 @@
-require 'oai/xpath'
-require 'oai/response'
-require 'oai/exception'
-require 'oai/header'
-require 'oai/record'
-require 'oai/set'
-require 'oai/metadata_format'
+require 'rubygems'
+require 'date'
+
+# Sub projects (client, provider) require their own libraries so the user
+# can selectively load them.
 require 'oai/client'
-require 'oai/identify'
-require 'oai/list_identifiers'
-require 'oai/list_metadata_formats'
-require 'oai/get_record'
-require 'oai/list_records'
-require 'oai/list_sets'
+require 'oai/provider'
+

data/lib/test.rb

@@ -0,0 +1,25 @@
+require 'oai'
+
+buffer = ""
+start_time = Time.now()
+
+client = OAI::Client.new 'http://digitalcollections.library.oregonstate.edu/cgi-bin/oai.exe', :parser =>'libxml'
+
+last_check = Date.new(2006,9,5)
+records = client.list_records
+# :set => 'archives', :metadata_prefix => 'oai_dc', :from => last_check 
+
+x = 0
+records.each do |record|
+  #fields = record.serialize_metadata(record.metadata, "oai_dc", "Oai_Dc")
+  #puts "Primary Title: " + fields.title[0] + "\n"
+  puts "Identifier: " +  record.header.identifier + "\n"
+  x += 1
+end
+
+end_time = Time.now()
+
+puts buffer
+puts "Time to run: " + (end_time - start_time).to_s + "\n"
+puts "Records returned: " + x.to_s
+

data/test/activerecord_provider/config/connection.rb

@@ -0,0 +1,30 @@
+require 'active_record'
+require 'logger'
+
+# Configure AR connection
+#ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+if RUBY_PLATFORM == "java"
+  require 'jdbc/sqlite3'
+  Jdbc::SQLite3.load_driver
+end
+
+ActiveRecord::Migration.verbose = false
+ActiveRecord::Base.establish_connection :adapter => "sqlite3",
+                                        :database => ":memory:"
+
+if ActiveRecord.version < Gem::Version.new("6.0.0")
+  ActiveRecord::MigrationContext.new(
+    File.join(File.dirname(__FILE__), '..', 'database')
+  ).migrate
+elsif ActiveRecord.version < Gem::Version.new("7.2.0")
+  ActiveRecord::MigrationContext.new(
+    File.join(File.dirname(__FILE__), '..', 'database'),
+    ActiveRecord::Base.connection.schema_migration
+  ).migrate
+else
+  ActiveRecord::MigrationContext.new(
+    File.join(File.dirname(__FILE__), '..', 'database')
+  ).migrate
+end
+