oai_talia 0.0.13

data/lib/oai/harvester/shell.rb

@@ -0,0 +1,338 @@
+module OAI
+  module Harvester
+    # = OAI::Harvester::Shell
+    # 
+    # A OAI-PMH client shell allowing OAI Harvesting to be configured in
+    # an interactive manner.  Typing 'oai' on the command line starts the
+    # shell.  The first time the shell is run it will prompt for the following
+    # configuration details:
+    # 1. A storage directory for all harvested records.  Harvests will be 
+    #    stored under this directory in a directory structure based on the 
+    #    date of the harvest.
+    # 2. A log file directory.
+    # 3. Email address(es) for sending daily harvesting activity reports.
+    # 4. Network address of the SMTP server for sending mail.
+    #
+    # After the initial configuration, new harvest sites can be added by using
+    # the 'new' command.  Sites are identified via nickname assigned by the
+    # user.  After choosing a nickname, provide the URL of a harvestable site,
+    # and the shell will prompt you for the rest of the configuration
+    # information.
+    #
+    # The shell automatically pulls down the list of sets in the repository, and
+    # the supported metadata prefixes.  Making it very simple to setup harvests.
+    #  
+    class Shell
+      include Readline
+    
+      def initialize(config)
+        @conf = config
+        @conf.sites ||= {} # Initialize sites hash there isn't one
+      end
+    
+      def start
+        unless @conf.storage
+          banner "Entering first-time setup"
+          config
+          setup_cron
+        end
+        puts "type 'help' for help"
+        while((input = readline("oai> ", true)) != 'exit')
+          begin
+            cmd = input.split
+            if 1 == cmd.size
+              self.send(cmd[0])
+            else
+              self.send(cmd.shift, cmd.join(" "))
+            end
+          rescue NoMethodError
+              puts "Not a recognized command. Type 'help' for clues."
+          rescue
+              puts "An error occurred:"
+              puts $!
+              puts $!.backtrace.join("\n")
+          end
+        end
+      end
+    
+      private
+    
+      def help
+        banner "Commands:"
+        puts "\tharvest site [date]     - Harvest site(s) manually"
+        puts "\tconfig                  - Configure harvester"
+        puts "\tlist <config>           - List known providers or configuration"
+        puts "\tinfo [site[, site]]     - Show information about a provider."
+        puts "\tnew                     - Add a new provider site to harvester"
+        puts "\tremove [site]           - Remove a provider site from harvester"
+        puts "\tedit [site]             - Change settings for a provider site"
+        puts "\texit                    - Exit the harvester shell.\n\n"
+      end
+    
+      def harvest(options)
+        site, *date = options.split(/\s/)
+        if @conf.sites.keys.include?(site)
+            banner "Harvesting '#{site}'"
+            if date && !date.empty?
+              begin
+                date = Time.parse(date.join(' ')).utc
+              rescue NoMethodError
+                puts "Couldn't parse the date supplied"
+                return
+              end
+            else 
+              date = nil
+            end
+            harvester = Harvest.new(@conf, @conf.storage, date)
+            harvester.start(site, true)
+            puts "done"
+        else
+          puts "Unknown repository: '#{args[0]}'"
+        end
+        puts # blank line
+      end
+    
+      def list(args = nil)
+        if 'config' == args
+          banner "Current Configuration"
+          list_config
+        else
+          banner "Configured Repositories"
+          @conf.sites.keys.each do |k|
+            puts k
+          end
+        end
+        puts # blank line
+      end
+    
+      def info(args)
+        banner "Provider Site Information"
+        sites = args.split(/[,\s|\s|,]/)
+        sites.each do |site|
+          print_site(site)
+        end
+        puts
+      rescue
+        puts args + " doesn't appear to be configured, use list to see configured repositories."
+      end
+    
+      def new
+        banner "Define New Harvesting Site"
+        name, site = form
+        @conf.sites[name] = site
+        @conf.save
+      end
+    
+      def edit(name)
+        banner "Edit Harvesting Site"
+        name, site = form(name)
+        @conf.sites[name] = site
+        @conf.save
+      end
+    
+      def remove(site)
+        if 'Y' == readline("Remove #{site}? (Y/N): ").upcase
+          @conf.sites.delete(site)
+          @conf.save
+          puts "#{site} removed"
+        end
+      end
+
+      # http://oai.getty.edu:80/oaicat/OAIHandler
+      def form(name = nil)
+        begin
+          if not name
+            name = prompt("nickname", nil)
+            while(@conf.sites.keys.include?(name))
+              show 0, "Nickname already in use, choose another."
+              name = prompt("nickname")
+            end
+          end
+          site = @conf.sites[name] || {}
+      
+          # URL
+          url = prompt("url", site['url'])
+          while(not (site['url'] = verify(url)))
+            puts "Trouble contacting provider, bad url?"
+            url = prompt("url", site['url'])
+          end
+      
+          # Metadata formats
+          formats = metadata(site['url'])
+          report "Repository supports [#{formats.join(', ')}] metadata formats."
+          prefix = prompt("prefix", site['prefix'])
+          while(not formats.include?(prefix))
+            prefix = prompt("prefix", site['prefix'])
+          end
+          site['prefix'] = prefix
+
+          # Sets
+          sets = ['all']
+          begin
+            sets.concat sets(site['url'])
+            site['set'] = 'all' unless site['set'] # default to all sets
+            report "Repository supports [#{sets.join(', ')}] metadata sets."
+            set = prompt("set", site['set'])
+            while(not sets.include?(site['set']))
+              set = prompt("set", site['set'])
+            end
+            site['set'] = set
+          rescue
+            site['set'] = 'all'
+          end
+
+          # Period
+          period = expand_period(prompt("period", "daily"))
+          while(not Config::PERIODS.include?(period))
+            puts "Must be daily, weekly, or monthly"
+            period = expand_period(prompt("period", "daily"))
+          end
+      
+          site['period'] = period
+      
+          return [name, site]
+        rescue 
+          puts "Problem adding/updating provider, aborting. (#{$!})"
+        end
+      end
+    
+      def config
+        begin
+          directory = prompt("storage directory", @conf.storage)
+          while not directory_acceptable(directory)
+            directory = prompt("storage directory: ", @conf.storage)
+          end
+
+          email = @conf.email.join(', ') rescue nil
+          @conf.email = parse_emails(prompt("email", email))
+        
+          @conf.mail_server = prompt("mail server", @conf.mail_server)
+
+          logfile = prompt("log file(s) directory", @conf.logfile)
+          while not directory_acceptable(logfile)
+            logfile = prompt("log file(s) directory", @conf.logfile)
+          end
+          @conf.storage = directory
+          @conf.logfile = logfile
+          @conf.save
+        rescue 
+          nil
+        end
+      end
+        
+      def display(key, value, split = 40)
+        (split - key.size).times { print " " } if key.size < split
+        puts "#{key}: #{value}"
+      end
+    
+      def banner(str)
+        puts "\n#{str}"
+        str.size.times { print "-" }
+        puts "\n"
+      end
+      
+      def report(str)
+        puts "\n#{str}\n"
+      end
+
+      def indent(number)
+        number.times do
+          print "\t"
+        end
+      end
+    
+      def prompt(text, default = nil, split = 20)
+        prompt_text = "#{text} [#{default}]: "
+        (split - prompt_text.size).times { print " " } if prompt_text.size < split
+        value = readline(prompt_text, true)
+        raise RuntimeError.new("Exit loop") unless value
+        return value.empty? ? default : value
+      end
+    
+      def verify(url)
+        begin
+          client = OAI::Client.new(url, :redirects => false)
+          identify = client.identify
+          puts "Repository name \"#{identify.repository_name}\""
+          return url
+        rescue
+          if $!.to_s =~ /^Permanently Redirected to \[(.*)\?.*\]/
+            report "Provider redirected to: #{$1}"
+            verify($1)
+          else
+            puts "Error selecting repository: #{$!}"
+          end
+        end
+      end
+    
+      def metadata(url)
+        formats = []
+        client = OAI::Client.new url
+        response = client.list_metadata_formats
+        response.to_a.each do |format|
+          formats << format.prefix
+        end
+        formats
+      end
+      
+      def sets(url)
+        sets = []
+        client = OAI::Client.new url
+        response = client.list_sets
+        response.to_a.each do |set|
+          sets << set.spec
+        end
+        sets
+      end
+
+      def directory_acceptable(dir)
+        if not (dir && File.exists?(dir) && File.writable?(dir))
+          puts "Directory doesn't exist, or isn't writtable."
+          return false
+        end
+        true
+      end
+    
+      def expand_period(str)
+        return str if Config::PERIODS.include?(str)
+        Config::PERIODS.each { |p| return p if p =~ /^#{str}/}
+        nil
+      end
+    
+      def parse_emails(emails)
+        return nil unless emails
+        addresses = emails.split(/[,\s|\s|,]/)
+      end
+        
+      def list_config
+        display("storage directory", @conf.storage, 20)
+        display("email", @conf.email.join(', '), 20) if @conf.email
+        display("mail server", @conf.mail_server, 20) if @conf.mail_server
+        display("log location", @conf.logfile, 20) if @conf.logfile
+      end
+    
+      def list_sites
+        banner "Sites"
+        @conf.sites.each_key { |site| print_site(site) }
+      end
+    
+      def print_site(site)
+        puts site
+        @conf.sites[site].each { |k,v| display(k, v, 15)}
+      end
+    
+      def setup_cron
+        banner "Scheduling Automatic Harvesting"
+        puts "To activate automatic harvesting you must add an entry to"
+        puts "your scheduler.  Linux/Mac OS X users should add the following"
+        puts "entry to their crontabs:\n\n"
+        puts "0 0 * * * #{$0} -D\n\n"
+        puts "Windows users should use WinAt to schedule"
+        puts "#{$0} to run every night.\n\n\n"
+      end
+    
+    end
+
+  end
+end        
+      

data/lib/oai/harvester.rb

@@ -0,0 +1,39 @@
+require 'zlib'
+require 'net/smtp'
+require 'yaml'
+require 'tempfile'
+require 'logger'
+require 'fileutils'
+require 'ostruct'
+require 'readline'
+require 'chronic'
+require 'socket'
+
+require 'oai/client'
+require 'oai/harvester/config'
+require 'oai/harvester/harvest'
+require 'oai/harvester/logging'
+require 'oai/harvester/mailer'
+require 'oai/harvester/shell'
+
+def harvestable_sites(conf)
+  sites = []
+  conf.sites.each do |k, v|
+    sites << k if needs_updating(v['period'], v['last'])
+  end if conf.sites
+  sites
+end
+
+def needs_updating(period, last)
+  return true if last.nil?
+  case period
+  when 'daily'
+    return true if Time.now - last > 86000
+  when 'weekly'
+    return true if Time.now - last > 604000
+  when 'monthly'
+    return true if Time.now - last > 2591000
+  end
+  return false
+end
+

data/lib/oai/provider/metadata_format/oai_dc.rb

@@ -0,0 +1,29 @@
+module OAI::Provider::Metadata
+  # = OAI::Metadata::DublinCore
+  # 
+  # Simple implementation of the Dublin Core metadata format.
+  class DublinCore < Format
+    
+    def initialize
+      @prefix = 'oai_dc'
+      @schema = 'http://www.openarchives.org/OAI/2.0/oai_dc.xsd'
+      @namespace = 'http://www.openarchives.org/OAI/2.0/oai_dc/'
+      @element_namespace = 'dc'
+      @fields = [ :title, :creator, :subject, :description, :publisher,
+                  :contributor, :date, :type, :format, :identifier,
+                  :source, :language, :relation, :coverage, :rights]
+    end
+
+    def header_specification
+      {
+        'xmlns:oai_dc' => "http://www.openarchives.org/OAI/2.0/oai_dc/",
+        'xmlns:dc' => "http://purl.org/dc/elements/1.1/",
+        'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
+        'xsi:schemaLocation' => 
+          %{http://www.openarchives.org/OAI/2.0/oai_dc/ 
+            http://www.openarchives.org/OAI/2.0/oai_dc.xsd}            
+      }
+    end
+
+  end
+end

data/lib/oai/provider/metadata_format/oai_europeana.rb

@@ -0,0 +1,38 @@
+module OAI::Provider::Metadata
+  # = OAI::Metadata::Europeana
+  # 
+  # Simple implementation of the Europeana metadata format.
+  class Europeana < Format
+    
+    def initialize
+      @prefix = 'ese'
+      @schema = 'http://www.openarchives.org/OAI/2.0/oai_dc.xsd'
+      @namespace = 'http://www.openarchives.org/OAI/2.0/oai_dc/'
+      @element_namespace = 'ese'
+      @fields = {:dc => [:title, :creator, :subject, :description, :publisher,
+                  :contributor, :date, :type, :format, :identifier,
+                  :source, :language, :relation, :coverage, :rights],
+                 :dcterms => :provenance,
+                 :ese => [:userTag, :unstored, :object, :language, :provider,
+                   :type, :uri, :year, :hasObject, :country]
+                  }
+    end
+
+    def header_specification
+      {
+        'xmlns:oai_dc'    =>  'http://www.openarchives.org/OAI/2.0/oai_dc/',
+        'xmlns:ese'       =>  'http://europeana.eu/terms/',
+        'xmlns:dc'        =>  'http://purl.org/dc/terms/',
+        'xmlns:dcterms'   =>  'http://purl.org/dc/terms/',
+        'xmlns:europeana' =>  'http://europeana.eu/terms/',
+        'xmlns:local'     =>  'http://metadata.cerl.org/oai/namespace/local/',
+        'xmlns:xsi'       =>  'http://www.w3.org/2001/XMLSchema-instance',
+        'xsi:schemaLocation' => 
+          %{http://purl.org/dc/elements/1.1/
+            http://dublincore.org/schemas/xmls/qdc/dc.xsd http://purl.org/dc/terms/
+            http://dublincore.org/schemas/xmls/qdc/dcterms.xsd}
+      }
+    end
+
+  end
+end

data/lib/oai/provider/metadata_format.rb

@@ -0,0 +1,143 @@
+require 'singleton'
+
+module OAI::Provider::Metadata
+  # == Metadata Base Class
+  #
+  # MetadataFormat is the base class from which all other format classes 
+  # should inherit.  Format classes provide mapping of record fields into XML.
+  #
+  # * prefix - contains the metadata_prefix used to select the format
+  # * schema - location of the xml schema
+  # * namespace - location of the namespace document
+  # * element_namespace - the namespace portion of the XML elements
+  # * fields - list of fields in this metadata format
+  #
+  # See OAI::Metadata::DublinCore for an example
+  #
+  class Format
+    include Singleton
+    
+    attr_accessor :prefix, :schema, :namespace, :element_namespace, :fields
+    
+    # Provided a model, and a record belonging to that model this method
+    # will return an xml represention of the record.  This is the method
+    # that should be extended if you need to create more complex xml
+    # representations.
+    def encode(model, record)
+      if record.respond_to?("to_#{prefix}")
+        record.send("to_#{prefix}")
+      else
+        xml = Builder::XmlMarkup.new
+        map = model.respond_to?("map_#{prefix}") ? model.send("map_#{prefix}") : {}
+        xml.tag!("#{prefix}:#{element_namespace}", header_specification) do
+          if fields.is_a?(Array)
+            fields.each do |field|
+              values = value_for(field, record, map)
+              if values.respond_to?(:each)
+                values.each do |value|
+                  xml.tag! "#{element_namespace}:#{field}", value
+                end
+              else
+                xml.tag! "#{element_namespace}:#{field}", values
+              end
+            end
+          elsif fields.is_a?(Hash)
+            fields.each_pair do |key, field_array|
+              field_array.to_a.each do |field|
+                values = value_for(field, record, map)
+                if values.respond_to?(:each)
+                  values.each do |value|
+                    xml.tag! "#{key}:#{field}", value
+                  end
+                else
+                  xml.tag! "#{key}:#{field}", values
+                end
+              end
+            end
+          end
+        end
+        xml.target!
+      end
+    end
+
+    private
+
+    # We try a bunch of different methods to get the data from the model.
+    #
+    # 1.  Check if the model defines a field mapping for the field of 
+    #     interest.
+    # 2.  Try calling the pluralized name method on the model.
+    # 3.  Try calling the singular name method on the model
+    def value_for(field, record, map)
+      method = map[field] ? map[field].to_s : field.to_s 
+     
+      if record.respond_to?(pluralize(method))
+        record.send pluralize(method)
+      elsif record.respond_to?(method)
+        # at this point, this function will throw a dep. error because of the call to type -- a reserved work
+        # in ruby
+        silence_warnings { record.send method }
+      else
+        []
+      end
+    end
+
+    # Subclasses must override
+    def header_specification
+      raise NotImplementedError.new
+    end
+    
+    # Shamelessly lifted form ActiveSupport.  Thanks Rails community!
+    def pluralize(word)
+      # Use ActiveSupports pluralization if it's available.
+      return word.pluralize if word.respond_to?(:pluralize)
+      
+      # Otherwise use our own simple pluralization rules.
+      result = word.to_s.dup
+      
+      # Uncountable words
+      return result if %w(equipment information rice money species series fish sheep).include?(result)
+      
+      # Irregular words
+      { 'person' => 'people', 'man' => 'men', 'child' => 'children', 'sex' => 'sexes', 
+        'move' => 'moves', 'cow' => 'kine' }.each { |k,v| return v if word == k }
+      
+      rules.each { |(rule, replacement)| break if result.gsub!(rule, replacement) }
+      result
+    end
+    
+    def rules
+      [
+        [/$/, 's'],
+        [/s$/i, 's'],
+        [/(ax|test)is$/i, '\1es'],
+        [/(octop|vir)us$/i, '\1i'],
+        [/(alias|status)$/i, '\1es'],
+        [/(bu)s$/i, '\1ses'],
+        [/(buffal|tomat)o$/i, '\1oes'],
+        [/([ti])um$/i, '\1a'],
+        [/sis$/i, 'ses'],
+        [/(?:([^f])fe|([lr])f)$/i, '\1\2ves'],
+        [/(hive)$/i, '\1s'],
+        [/([^aeiouy]|qu)y$/i, '\1ies'],
+        [/(x|ch|ss|sh)$/i, '\1es'],
+        [/(matr|vert|ind)(?:ix|ex)$/i, '\1ices'],
+        [/([m|l])ouse$/i, '\1ice'],
+        [/^(ox)$/i, '\1en'],
+        [/(quiz)$/i, '\1zes']
+      ]
+    end
+    
+    def silence_warnings
+      old_verbose, $VERBOSE = $VERBOSE, nil
+      yield
+    ensure
+      $VERBOSE = old_verbose
+    end
+    
+
+  end
+  
+end
+
+Dir.glob(File.dirname(__FILE__) + '/metadata_format/*.rb').each {|lib| require lib}