contentdm 0.1.20

data/README.rdoc

@@ -0,0 +1,36 @@
+= Introduction
+The ContentDm module for Ruby provides access to structured metadata in CONTENTdm collections
+via CONTENTdm's built-in OAI-PMH provider interface. The module turns Qualified Dublin Core
+metadata into a convenient hash structure. With proper authentication, it can also scrape
+collection-level field information from the CONTENTdm administrative interface and create
+formatted HTML output from retrieved records.
+
+= Examples
+
+  # Create a Harvester using the location of a CONTENTdm repository
+  harvester = ContentDm::Harvester.new('http://mycontentdm.example.com/')
+  
+  # Retrieve the list of collections
+  collections = harvester.collections  
+  => {"collection1" => "My First Collection", "collection2" => "My Second Collection"}
+  
+  # Retrieve a single record from collection1
+  record = harvester.get_record("collection1",16)
+  
+  # Retrieve all records from collection2
+  records = harvester.get_records("collection2")
+  
+Calling <tt>record#to_xml()</tt> or <tt>record#to_html()</tt> at this point will return 
+generic and arbitrarily-ordered markup, because the ContentDm::Mapper for 
+<tt>collection1</tt> hasn't been initialized.
+
+  # Initialize the Mapper for a single collection
+  ContentDm::Mapper.init_map("http://mycontentdm.example.com/", "collection1" :user => "my_contentdm_admin", :pass => "p@$$w0rd")
+  => #<ContentDm::Mapper ... >
+  
+  # Initialize Mappers for all collections on the server
+  ContentDm::Mapper.init_all("http://mycontentdm.example.com/", :user => "my_contentdm_admin", :pass => "p@$$w0rd")
+  => ["collection1", "collection2"]
+  
+Now <tt>record#to_xml()</tt> and <tt>record#to_html()</tt> will return markup consistent with
+the settings defined for the collection within CONTENTdm.

data/lib/contentdm.rb

@@ -0,0 +1,8 @@
+require 'contentdm/uri'
+require 'contentdm/harvester'
+require 'contentdm/mapper'
+require 'contentdm/record'
+
+module ContentDm
+  VERSION = '0.1.20'
+end

data/lib/contentdm/harvester.rb

@@ -0,0 +1,118 @@
+require 'rubygems'
+require 'nokogiri'
+require 'open-uri'
+require 'uri'
+
+module ContentDm #:nodoc:#
+  
+  class Harvester
+
+    extend URI
+    
+    OAI_PAGE_SIZE = 1000
+    
+    attr_reader :base_uri
+    attr_accessor :page_size
+    
+    # The constructor must be passed the URL of a CONTENTdm installation. This will usually
+    # be the root of the server on which CONTENTdm is installed.
+    def initialize(base_uri)
+      @base_uri = self.class.normalize(base_uri)
+      @page_size = 1000
+    end
+
+    # Convenience method which returns a single Record when passed a URL in
+    # one of two forms:
+    # * A CONTENTdm URL containing CISOROOT/CISOPTR values for the desired item
+    # * A CONTENTdm canonical URL in the form
+    #     http://path/to/contentdm/u?[collection],[ptr]
+    #   where <tt>[collection]</tt> is the CONTENTdm collection name, and <tt>[ptr]</tt> is the sequential
+    #   item ID within the collection.
+    def self.get_record(url)
+      base_uri = self.normalize(url)
+      params = {}
+      if args = url.match(/^(.+\/)u\/?\?\/(.+),(\d+)$/)
+        params[:base_url] = args[1]
+        params[:collection] = args[2]
+        params[:id] = args[3]
+      else
+        args = base_uri.query.split(/&/).inject({}) { |hash,arg|
+          (k,v) = arg.split(/\=/,2)
+          hash[k] = ::URI.decode(v)
+          hash
+        }
+        params[:base_url] = base_uri.merge('..')
+        params[:collection] = args['CISOROOT'][1..-1]
+        params[:id] = args['CISOPTR']
+      end
+      harvester = Harvester.new(params[:base_url])
+      harvester.get_record(params[:collection],params[:id])
+    end
+    
+    # Return a hash of collection IDs and collection names
+    def collections
+      response = Nokogiri::XML(open(@base_uri.merge('cgi-bin/oai.exe?verb=ListSets')))
+      sets = response.search('//xmlns:set',response.namespaces)
+      result = {}
+      sets.inject({}) { |hash,set| 
+        set_id = (set / 'setSpec').text()
+        set_desc = (set / 'setName').text()
+        hash[set_id] = set_desc
+        hash
+      }
+    end
+    
+    # Return a single Record given its collection ID and ordinal position
+    # within the collection
+    def get_record(collection, id)
+      oai_id = "oai:%s:%s/%d" % [@base_uri.host, collection, id]
+      response = get_response({ :verb => 'GetRecord', :identifier => oai_id, :metadataPrefix => 'qdc' })
+      record = parse_records(response).first
+      Record.new(record, { :base_uri => @base_uri, :collection => collection })
+    end
+
+    # Return an array of all the Records in a given collection
+    def get_records(collection, opts = {})
+      max = opts[:max].to_i
+      token = "#{collection}:#{opts[:from].to_s}:#{opts[:until].to_s}:qdc:#{opts[:first].to_i || 0}"
+      result = []
+      until token.nil? or ((max > 0) and (result.length >= max))
+        args = { :verb => 'ListRecords', :resumptionToken => token.to_s }
+        response = get_response(args)
+        token = response.search('/xmlns:OAI-PMH/xmlns:ListRecords/xmlns:resumptionToken/text()', response.namespaces).first
+        result += parse_records(response)
+      end
+      if result.length > max
+        result = result[0..max-1]
+      end
+      result.collect { |record|
+        Record.new(record, { :base_uri => @base_uri, :collection => collection })
+      }
+    end
+
+    private
+    def parse_records(response)
+      result = []
+      qdcs = response.search('//qdc:qualifieddc',{ 'qdc' => 'http://epubs.cclrc.ac.uk/xmlns/qdc/' })
+      qdcs.each { |qdc|
+        metadata = Hash.new { |h,k| h[k] = [] }
+        qdc.children.each { |child|
+          if child.element?
+            metadata[[child.namespace.prefix,child.name].join('.')] << child.text # unless child.text.empty?
+          end
+        }
+        result << metadata
+      }
+      result
+    end
+
+    def get_response(args)
+      path = 'cgi-bin/oai.exe'
+      query = args.collect { |k,v| [k.to_s,::URI.encode(v)].join('=') }.join('&')
+      uri = @base_uri.merge("#{path}?#{query}")
+      response = Nokogiri::XML(open(uri))
+    end
+
+  end
+
+end

data/lib/contentdm/mapper.rb

@@ -0,0 +1,258 @@
+require 'rubygems'
+require 'erb'
+require 'net/http'
+require 'nokogiri'
+require 'open-uri'
+require 'uri'
+
+module ContentDm
+  
+DEFAULT_TEMPLATE = %{<span>
+% field_order.each do |fieldname|
+%   unless data[fieldname].nil? or data[fieldname].empty?
+    <p>
+        <b><%= fieldname %>: </b>
+        <%= data[fieldname].to_a.join("; ") %>
+    </p>
+%   end 
+% end
+</span>}
+  
+# GenericMapper acts as a fallback formatter for instances when no other Mapper is defined
+class GenericMapper
+  
+  SaveOptions = Nokogiri::XML::Node::SaveOptions
+
+  # Serialize the given Record to a Qualified Dublin Core XML string
+  def to_xml(record, opts = {})
+    builder = Nokogiri::XML::Builder.new do |doc|
+      doc.qualifieddc('xmlns:qdc' => "http://epubs.cclrc.ac.uk/xmlns/qdc/", 
+        'xmlns:dc' => "http://purl.org/dc/elements/1.1/", 
+        'xmlns:dcterms' => "http://purl.org/dc/terms/") {
+          record.metadata.each_pair { |k,v|
+            (prefix,tag) = k.split(/\./)
+            if v.is_a?(Array)
+              v.each { |value|
+                doc[prefix].send(tag.to_sym) {
+                  doc.text(value)
+                }
+              }
+            else
+              doc[prefix].send(tag.to_sym) {
+                doc.text(v)
+              }
+            end
+          }
+        }
+    end
+    builder.to_xml
+  end
+  
+  # Serialize the given Record to an HTML string
+  def to_html(record, opts = {})
+    save_options = { :encoding => 'UTF-8', :save_with => (SaveOptions::AS_XML | SaveOptions::NO_DECLARATION), :indent => 2 }.merge(opts)
+    builder = Nokogiri::XML::Builder.new do |doc|
+      doc.span {
+        record.metadata.each_pair { |k,v|
+          unless v.nil? or v.to_s.empty?
+            (prefix,tag) = k.split(/\./)
+            # Convert from camelCase to Human Readable Label
+            tag = tag.gsub(/(\S)([A-Z])/,'\1 \2').gsub(/\b('?[a-z])/) { $1.capitalize }
+            doc.p {
+              doc.b {
+                doc.text "#{tag}:"
+              }
+              doc.text " "
+              if v.is_a?(Array)
+                doc.br
+                v.each { |value|
+                  doc.text value unless value.empty?
+                  doc.br
+                }
+              else
+                doc.text v
+              end
+            }
+          end
+        }
+      }
+    end
+    builder.to_xml(save_options)
+  end
+  
+end
+
+# A Mapper provides information about field label, visibility, and output order for a
+# specific CONTENTdm collection. This information can be screen-scraped from a 
+# CONTENTdm installation, or defined programatically.
+class Mapper < GenericMapper
+
+  extend URI
+  @@maps = {}
+  @@auto_init = true
+  
+  attr_accessor :fields, :order
+  
+  class << self
+    
+  attr_accessor :auto_init
+  
+  def maps
+    @@maps.keys
+  end
+  
+  # Returns true if a Mapper has been initialized for the given collection at the specified base URI.
+  def mapped?(uri, collection)
+    return @@maps.include?(self.signature(uri,collection))
+  end
+  
+  # Initializes Mappers for all collections at the specified base URI.
+  def init_all(base_uri)
+    uri = self.normalize(base_uri)
+    response = Nokogiri::XML(open(uri.merge('cgi-bin/oai.exe?verb=ListSets')))
+    sets = response.search('//xmlns:set/xmlns:setSpec/text()',response.namespaces).collect { |set| set.text }
+    sets.each { |set|
+      self.init_map(uri, set)
+    }
+  end
+  
+  # Initializes the Mapper for the given collection at the specified base URI.
+  def init_map(base_uri, collection)
+    uri = self.normalize(base_uri)
+
+    dc_map = self.from(uri, 'DC_MAPPING')
+    if dc_map.nil?
+      fields = open(uri.merge("dc.txt")) { |res| res.read }
+      dc_map = {}
+      fields.each_line { |field|
+        field_properties = field.chomp.split(/:/)
+        dc_field = self.normalize_field_name(field_properties[0])
+        field_code = field_properties[1]
+        dc_map[field_code] = dc_field
+      }
+      @@maps[self.signature(uri, 'DC_MAPPING')] = dc_map
+    end
+
+    fields = open(uri.merge("#{collection}/index/etc/config.txt")) { |res| res.read }
+    map = { :fields => Hash.new { |h,k| h[k] = [] }, :order => [] }
+    fields.each_line { |field|
+      field_properties = field.chomp.split(/:/)
+      field_label = field_properties.first
+      field_code = field_properties.last
+      map[:fields][dc_map[field_code]] << field_label
+      map[:order] << field_label unless field_properties[-3] == 'HIDE'
+    }
+    map[:fields]['dc.identifier'] << 'Permalink'
+    @@maps[self.signature(uri,collection)] = self.new(uri, collection, map[:fields], map[:order])
+  end
+  
+  # Assigns a map (either an initialized Map or a Hash/Array combination indicating the 
+  # field mapping and field order) to a given collection.
+  def assign_map(base_uri, collection, *args)
+    uri = self.normalize(base_uri)
+    if args[0].is_a?(self)
+      @@maps[self.signature(uri,collection)] = args[0]
+    else
+      @@maps[self.signature(uri,collection)] = self.new(uri, collection, *args)
+    end
+  end
+  
+  # Returns the appropriate Mapper for the given collection at the specified base URI. If it
+  # has not been initialized or the collection does not exist, returns nil.
+  def from(uri, collection)
+    if @@auto_init and (collection != 'DC_MAPPING')
+      unless self.mapped?(uri, collection)
+        self.init_map(uri, collection)
+      end
+    end
+    @@maps[self.signature(uri,collection)]
+  end
+  end
+
+  # Creates a map based on the hash of fields
+  def initialize(base_uri, collection, fields, order = nil)
+    @base_uri = base_uri
+    @collection = collection
+    @fields = fields
+    @order = order
+  end
+
+  def rename(old_field,new_field)
+    @fields.each_pair { |k,v| v.collect! { |name| name == old_field ? new_field : name } }
+    @order.collect! { |name| name == old_field ? new_field : name }
+  end
+  
+  # Returns a hash of field labels and data
+  def map(record)
+    data = record.metadata
+    result = {}
+    @fields.each_pair { |k,v|
+      v.each_with_index { |key,index|
+        if data[k]
+          value = data[k][index]
+          unless value.nil?
+            result[key] = value.split(/;\s*/)
+            if result[key].length == 1
+              result[key] = result[key].first
+            end
+          end
+        end
+      }
+    }
+    result
+  end
+
+  # Serialize the given Record to a Qualified Dublin Core XML string
+  def to_xml(record, opts = {})
+    save_options = { :encoding => 'UTF-8', :save_with => SaveOptions::AS_XML, :indent => 2 }.merge(opts)
+    data = self.map(record)
+    field_order = @order || []
+    builder = Nokogiri::XML::Builder.new do |doc|
+      doc.qualifieddc('xmlns:qdc' => "http://epubs.cclrc.ac.uk/xmlns/qdc/", 
+        'xmlns:dc' => "http://purl.org/dc/elements/1.1/", 
+        'xmlns:dcterms' => "http://purl.org/dc/terms/") {
+          field_order.each { |fieldname|
+            field_info = @fields.find { |k,v| v.include?(fieldname) }
+            unless field_info.nil? or field_info[0].nil?
+              (prefix,tag) = field_info[0].split(/\./)
+              index = field_info[1].index(fieldname)
+              value = data[fieldname]
+              if value.is_a?(Array)
+                value = value[index]
+              end
+              doc[prefix].send("#{tag}_".to_sym) {
+                doc.text(value)
+              }
+            end
+          }
+        }
+    end
+    builder.to_xml
+  end
+  
+  # Serialize the given Record to an HTML string
+  def to_html(record, vars = {})
+    erb = vars.delete(:template) || DEFAULT_TEMPLATE
+    data = self.map(record)
+    field_order = @order || []
+    template = ERB.new(erb,nil,'%')
+    template.result(binding)
+  end
+  
+  private
+  def self.signature(uri, collection)
+    "#{uri.to_s} :: #{collection}"
+  end
+
+  def self.normalize_field_name(fieldname)
+    parts = fieldname.downcase.gsub(/(\s+[a-z])/) { |ch| ch.upcase.strip }.split(/-/)
+    if parts.length == 1
+      "dc.#{parts[0]}"
+    else
+      "dcterms.#{parts[1]}"
+    end
+  end
+  
+end
+
+end

data/lib/contentdm/record.rb

@@ -0,0 +1,85 @@
+module ContentDm
+  
+class Record
+  
+  attr_reader :metadata, :source
+  
+  def initialize(data, source)
+    @metadata = data.dup
+    @source = source
+
+    # Account for bug in single-record output
+#    parts = self.permalink.split
+#    if parts.length > 1
+#      self.permalink = @source[:base_uri].merge(parts.last).to_s
+#    end
+    
+    (collection, record_id) = @metadata['dc.identifier'][-1].scan(/\?\/(.+),([0-9]+)$/).flatten
+    @source[:collection] = collection
+    @source[:id] = record_id.to_i
+    self.permalink = @source[:base_uri].merge("/u?/#{collection},#{record_id}").to_s
+  end
+  
+  def img_href(opts = {})
+    params = { 
+      'CISOROOT' => "/#{@source[:collection]}", 
+      'CISOPTR' => @source[:id],
+      'DMSCALE' => 100,
+      'DMWIDTH' => 0,
+      'DMHEIGHT' => 0,
+      'DMX' => 0,
+      'DMY' => 0,
+      'DMTEXT' => '',
+      'DMTHUMB' => '',
+      'DMROTATE' => 0
+    }
+    opts.each_pair { |k,v|
+      case k
+        when :width  then params['DMWIDTH'] = v
+        when :height then params['DMHEIGHT'] = v
+        when :scale  then params['DMSCALE'] = v
+        else              params[k] = v
+      end
+    }
+    query = params.collect { |k,v| "#{k}=#{::URI.encode(v.to_s)}" }.join('&')
+    @source[:base_uri].merge("cgi-bin/getimage.exe?#{query}")
+  end
+  
+  def thumbnail_href
+    params = { 
+      'CISOROOT' => "/#{@source[:collection]}", 
+      'CISOPTR' => @source[:id],
+    }
+    query = params.collect { |k,v| "#{k}=#{::URI.encode(v.to_s)}" }.join('&')
+    @source[:base_uri].merge("cgi-bin/thumbnail.exe?#{query}")
+  end
+  
+  def permalink
+    @metadata['dc.identifier'][-1]
+  end
+  
+  def permalink=(value)
+    @metadata['dc.identifier'][-1] = value
+  end
+  
+  def mapper
+    Mapper.from(@source[:base_uri],@source[:collection]) || GenericMapper.new
+  end
+  
+  # Serialize the Record to a Qualified Dublin Core XML string. If
+  # a Mapper has been initialized for the Record's owning collection,
+  # it will be used. Otherwise, the GenericMapper will be used.
+  def to_xml(opts = {})
+    mapper.to_xml(self, opts)
+  end
+  
+  # Serialize the Record to an HTML string.  If a Mapper has been
+  # initialized for the Record's owning collection, it will be 
+  # used. Otherwise, the GenericMapper will be used.
+  def to_html(opts = {})
+    mapper.to_html(self, opts)
+  end
+  
+end
+
+end

data/lib/contentdm/uri.rb

@@ -0,0 +1,11 @@
+module ContentDm
+  
+module URI
+  def normalize(uri)
+   local_uri = uri.is_a?(::URI) ? uri : ::URI.parse(uri)
+   local_uri.path.sub!(/\/+$/,'')
+   local_uri
+  end
+end
+
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification 
+name: contentdm
+version: !ruby/object:Gem::Version 
+  version: 0.1.20
+platform: ruby
+authors: 
+- Michael B. Klein
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-03 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Module providing access to structured metadata in CONTENTdm collections
+email: Michael.Klein@oregonstate.edu
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- lib/contentdm/harvester.rb
+- lib/contentdm/mapper.rb
+- lib/contentdm/record.rb
+- lib/contentdm/uri.rb
+- lib/contentdm.rb
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Access to structured metadata in CONTENTdm collections
+test_files: []
+