oai 0.0.3 → 0.0.4

data/lib/oai/harvester.rb

@@ -0,0 +1,38 @@
+require 'zlib'
+require 'net/smtp'
+require 'yaml'
+require 'tempfile'
+require 'logger'
+require 'fileutils'
+require 'ostruct'
+require 'readline'
+require 'chronic'
+require 'socket'
+
+require 'oai/harvester/config'
+require 'oai/harvester/harvest'
+require 'oai/harvester/logging'
+require 'oai/harvester/mailer'
+require 'oai/harvester/shell'
+
+def harvestable_sites(conf)
+  sites = []
+  conf.sites.each do |k, v|
+    sites << k if needs_updating(v['period'], v['last'])
+  end if conf.sites
+  sites
+end
+
+def needs_updating(period, last)
+  return true if last.nil?
+  case period
+  when 'daily'
+    return true if Time.now - last > 86000
+  when 'weekly'
+    return true if Time.now - last > 604000
+  when 'monthly'
+    return true if Time.now - last > 2591000
+  end
+  return false
+end
+

data/lib/oai/harvester/config.rb

@@ -0,0 +1,41 @@
+#
+#  Created by William Groppe on 2006-11-05.
+#  Copyright (c) 2006. All rights reserved.
+
+module OAI
+  module Harvester
+
+    LOW_RESOLUTION = "YYYY-MM-DD"
+  
+    class Config < OpenStruct
+    
+      PERIODS = %w(daily weekly monthly)
+      GLOBAL = "/etc/oai/harvester.yml"
+    
+      def self.load
+        config = find_config
+        File.exists?(config) ? new(YAML.load_file(config)) : new
+      end
+
+      def save
+        config = Config.find_config
+        open(config, 'w') do |out|
+          YAML.dump(@table, out)
+        end
+      end
+
+      private 
+      # Shamelessly lifted from Camping
+      def self.find_config
+        if home = ENV['HOME'] # POSIX
+          return GLOBAL if File.exists?(GLOBAL) && File.writable?(GLOBAL)
+          FileUtils.mkdir_p File.join(home, '.oai')
+          File.join(home, '.oai/harvester.yml')
+        elsif home = ENV['APPDATA'] # MSWIN
+          File.join(home, 'oai/harvester.yml')
+        end
+      end
+    
+    end
+  end
+end

data/lib/oai/harvester/harvest.rb

@@ -0,0 +1,144 @@
+#
+#  Created by William Groppe on 2006-11-03.
+
+module OAI
+  module Harvester
+  
+    class Harvest
+    
+      def initialize(config = nil, directory = nil, date = nil)
+        @config = config || Config.load
+        @directory = directory || @config.storage
+        @from = date
+        @from.freeze
+        @parser = defined?(XML::Document) ? 'libxml' : 'rexml'
+      end
+    
+      def start(sites = nil, interactive = false)
+        @interactive = interactive
+        sites = (@config.sites.keys rescue {}) unless sites
+        begin
+          sites.each do |site|
+            harvest(site)
+          end
+        ensure
+          @config.save
+        end
+      end
+    
+      private
+    
+      def harvest(site)
+        harvest_time = Time.now.utc
+        opts = build_options_hash(@config.sites[site])
+        opts[:until] = harvest_time.xmlschema
+      
+        # Allow a from date to be passed in
+        if(@from)
+          opts[:from] = @from
+        else 
+          opts[:from] = earliest(opts[:url])
+        end
+      
+        opts.delete(:set) if 'all' == opts[:set]
+
+        begin
+          # Connect, and download
+          file, records = call(opts.delete(:url), opts)
+      
+          # Move document to storage directory
+          dir = File.join(@directory, date_based_directory(harvest_time))
+          FileUtils.mkdir_p dir
+          FileUtils.mv(file.path, 
+            File.join(dir, "#{site}-#{filename(Time.parse(opts[:from]), 
+            harvest_time)}.xml.gz"))
+          @config.sites[site]['last'] = harvest_time
+        rescue
+          raise $! unless $!.respond_to?(:code)
+          raise $! if not @interactive || "noRecordsMatch" != $!.code
+          puts "No new records available"
+        end
+      end
+    
+      def call(url, opts)
+        # Preserve original options
+        options = opts.dup
+        
+        records = 0;
+        client = OAI::Client.new(url, :parser => @parser)
+        provider_config = client.identify
+        
+        if Harvester::LOW_RESOLUTION == provider_config.granularity
+          options[:from] = Time.parse(options[:from]).strftime("%Y-%m-%d")
+          options[:until] = Time.parse(options[:until]).strftime("%Y-%m-%d")
+        end
+        
+        file = Tempfile.new('oai_data')
+        gz = Zlib::GzipWriter.new(file)
+        gz << "<? xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
+        gz << "<records>"
+        begin
+          response = client.list_records(options)
+          get_records(response.doc).each do |rec|
+            gz << rec
+            records += 1
+          end
+          puts "#{records} records retrieved" if @interactive
+
+          # Get a full response by iterating with the resumption tokens.  
+          # Not very Ruby like.  Should fix OAI::Client to handle resumption
+          # tokens internally.
+          while(response.resumption_token and not response.resumption_token.empty?)
+            puts "\nresumption token recieved, continuing" if @interactive
+            response = client.list_records(:resumption_token => 
+              response.resumption_token)
+              get_records(response.doc).each do |rec|
+                gz << rec
+                records += 1
+              end
+            puts "#{records} records retrieved" if @interactive
+          end
+
+            gz << "</records>"
+            
+        ensure
+          gz.close
+          file.close
+        end
+
+        [file, records]
+      end
+    
+      def get_records(doc)
+        doc.find("/OAI-PMH/ListRecords/record").to_a
+      end
+    
+      def build_options_hash(site)
+        options = {:url => site['url']}
+        options[:set] = site['set'] if site['set']
+        options[:from] = site['last'].utc.xmlschema if site['last']
+        options[:metadata_prefix] = site['prefix'] if site['prefix']
+        options
+      end
+    
+      def date_based_directory(time)
+        "#{time.strftime(DIRECTORY_LAYOUT)}"
+      end
+
+      def filename(from_time, until_time)
+        format = "%Y-%m-%d"
+        "#{from_time.strftime(format)}_til_#{until_time.strftime(format)}"\
+        "_at_#{until_time.strftime('%H-%M-%S')}"
+      end
+    
+      # Get earliest timestamp from repository
+      def earliest(url)
+        client = OAI::Client.new url
+        identify = client.identify
+        Time.parse(identify.earliest_datestamp).utc.xmlschema
+      end
+    
+    end
+  
+  end
+end

data/lib/oai/harvester/logging.rb

@@ -0,0 +1,70 @@
+# Reopen Harvest and add logging
+module OAI
+  module Harvester
+
+    class Harvest
+      alias_method :orig_start, :start
+      alias_method :orig_harvest, :harvest
+      alias_method :orig_call, :call
+      alias_method :orig_init, :initialize
+    
+      def initialize(config = nil, directory = nil, date = nil)
+        orig_init(config, directory, date)
+        @summary = []
+        @logger = Logger.new(File.join(@config.logfile, "harvester.log"), 
+          shift_age = 'weekly') if @config.logfile
+        @logger.datetime_format = "%Y-%m-%d %H:%M"
+      
+        # Turn off logging if no logging directory is specified.
+        @logger.level = Logger::FATAL unless @config.logfile
+      end
+    
+      def start(sites = nil, interactive = false)
+        if not interactive
+          @logger.info { "Starting regular harvest" }
+          orig_start(sites)
+          begin
+            OAI::Harvester::
+              Mailer.send(@config.mail_server, @config.email, @summary)
+          rescue
+            @logger.error { "Error sending out summary email: #{$!}"}
+          end
+        else
+          @logger.info { "Starting interactive harvest"}
+          orig_start(sites, true)
+        end
+      end
+    
+      private
+    
+      def harvest(site)
+        begin
+          @logger.info { "Harvest of '#{site}' starting" }
+          @summary << "Harvest of '#{site}' attempted"
+          orig_harvest(site)
+        rescue OAI::Exception
+          if "noRecordsMatch" == $!.code
+            @logger.info "No new records available"
+            @summary << "'#{site}' had no new records."
+          else
+            @logger.error { "Harvesting of '#{site}' failed, message: #{$!}" }
+            @summary << "'#{site}' had an OAI Error! #{$!}"
+          end
+        rescue
+          @logger.error { "Harvesting of '#{site}' failed, message: #{$!}" }
+          @logger.error { "#{$!.backtrace.join('\n')}" }
+          @summary << "'#{site}' had an Error! #{$!}"
+        end
+      end
+    
+      def call(url, options)
+        @logger.info { "fetching: #{url} with options #{options.inspect}" }
+        file, records = orig_call(url, options)
+        @logger.info { "retrieved #{records} records" }
+        @summary << "Retrieved #{records} records."
+        return file, records
+      end
+    end
+    
+  end
+end

data/lib/oai/harvester/mailer.rb

@@ -0,0 +1,17 @@
+module OAI
+  module Harvester
+    
+    class Mailer
+    
+      def self.send(server = nil, email = nil, message = nil)
+        msg = %{Subject: Harvester Summary\n\n#{message.join("\n")}}
+        to = (email.map { |e| "'#{e}'"}).join(", ")
+        Net::SMTP.start(server) do |smtp|
+          smtp.send_message msg, "harvester@#{Socket.gethostname}", to
+        end
+      end
+    
+    end
+    
+  end
+end        

data/lib/oai/harvester/shell.rb

@@ -0,0 +1,334 @@
+module OAI
+  module Harvester
+    # = OAI::Harvester::Shell
+    # 
+    # A OAI-PMH client shell allowing OAI Harvesting to be configured in
+    # an interactive manner.  Typing 'oai' on the command line starts the
+    # shell.  The first time the shell is run it will prompt for the following
+    # configuration details:
+    # 1. A storage directory for all harvested records.  Harvests will be 
+    #    stored under this directory in a directory structure based on the 
+    #    date of the harvest.
+    # 2. A log file directory.
+    # 3. Email address(es) for sending daily harvesting activity reports.
+    # 4. Network address of the SMTP server for sending mail.
+    #
+    # After the initial configuration, new harvest sites can be added by using
+    # the 'new' command.  Sites are identified via nickname assigned by the
+    # user.  After choosing a nickname, provide the URL of a harvestable site,
+    # and the shell will prompt you for the rest of the configuration
+    # information.
+    #
+    # The shell automatically pulls down the list of sets in the repository, and
+    # the supported metadata prefixes.  Making it very simple to setup harvests.
+    #  
+    class Shell
+      include Readline
+    
+      def initialize(config)
+        @conf = config
+        @conf.sites ||= {} # Initialize sites hash there isn't one
+      end
+    
+      def start
+        unless @conf.storage
+          banner "Entering first-time setup"
+          config
+          setup_cron
+        end
+        puts "type 'help' for help"
+        while((input = readline("oai> ", true)) != 'exit')
+          begin
+            cmd = input.split
+            if 1 == cmd.size
+              self.send(cmd[0])
+            else
+              self.send(cmd.shift, cmd.join(" "))
+            end
+          rescue 
+            puts "Not a recognized command, or bad options.  Type 'help' for clues."
+            #puts $!
+            #puts $!.backtrace.join("\n")
+          end
+        end
+      end
+    
+      private
+    
+      def help
+        banner "Commands:"
+        puts "\tharvest site [date]     - Harvest site(s) manually"
+        puts "\tconfig                  - Configure harvester"
+        puts "\tlist <config>           - List known providers or configuration"
+        puts "\tinfo [site[, site]]     - Show information about a provider."
+        puts "\tnew                     - Add a new provider site to harvester"
+        puts "\tremove [site]           - Remove a provider site from harvester"
+        puts "\tedit [site]             - Change settings for a provider site"
+        puts "\texit                    - Exit the harvester shell.\n\n"
+      end
+    
+      def harvest(options)
+        site, *date = options.split(/\s/)
+        if @conf.sites.keys.include?(site)
+            banner "Harvesting '#{site}'"
+            if date && !date.empty?
+              begin
+                date = Chronic.parse(date.join(' ')).utc.xmlschema
+              rescue NoMethodError
+                puts "Couldn't parse the date supplied"
+                return
+              end
+            else 
+              date = nil
+            end
+            harvester = Harvest.new(@conf, @conf.storage, date)
+            harvester.start(site, true)
+            puts "done"
+        else
+          puts "Unknown repository: '#{args[0]}'"
+        end
+        puts # blank line
+      end
+    
+      def list(args = nil)
+        if 'config' == args
+          banner "Current Configuration"
+          list_config
+        else
+          banner "Configured Repositories"
+          @conf.sites.keys.each do |k|
+            puts k
+          end
+        end
+        puts # blank line
+      end
+    
+      def info(args)
+        banner "Provider Site Information"
+        sites = args.split(/[,\s|\s|,]/)
+        sites.each do |site|
+          print_site(site)
+        end
+        puts
+      end
+    
+      def new
+        banner "Define New Harvesting Site"
+        name, site = form
+        @conf.sites[name] = site
+        @conf.save
+      end
+    
+      def edit(name)
+        banner "Edit Harvesting Site"
+        name, site = form(name)
+        @conf.sites[name] = site
+        @conf.save
+      end
+    
+      def remove(site)
+        if 'Y' == readline("Remove #{site}? (Y/N): ").upcase
+          @conf.sites.delete(site)
+          @conf.save
+          puts "#{site} removed"
+        end
+      end
+
+      # http://oai.getty.edu:80/oaicat/OAIHandler
+      def form(name = nil)
+        begin
+          if not name
+            name = prompt("nickname", nil)
+            while(@conf.sites.keys.include?(name))
+              show 0, "Nickname already in use, choose another."
+              name = prompt("nickname")
+            end
+          end
+          site = @conf.sites[name] || {}
+      
+          # URL
+          url = prompt("url", site['url'])
+          while(not (site['url'] = verify(url)))
+            puts "Trouble contacting provider, bad url?"
+            url = prompt("url", site['url'])
+          end
+      
+          # Metadata formats
+          formats = metadata(site['url'])
+          report "Repository supports [#{formats.join(', ')}] metadata formats."
+          prefix = prompt("prefix", site['prefix'])
+          while(not formats.include?(prefix))
+            prefix = prompt("prefix", site['prefix'])
+          end
+          site['prefix'] = prefix
+
+          # Sets
+          sets = ['all']
+          begin
+            sets.concat sets(site['url'])
+            site['set'] = 'all' unless site['set'] # default to all sets
+            report "Repository supports [#{sets.join(', ')}] metadata sets."
+            set = prompt("set", site['set'])
+            while(not sets.include?(site['set']))
+              set = prompt("set", site['set'])
+            end
+            site['set'] = set
+          rescue
+            site['set'] = 'all'
+          end
+
+          # Period
+          period = expand_period(prompt("period", "daily"))
+          while(not Config::PERIODS.include?(period))
+            puts "Must be daily, weekly, or monthly"
+            period = expand_period(prompt("period", "daily"))
+          end
+      
+          site['period'] = period
+      
+          return [name, site]
+        rescue 
+          puts "Problem adding/updating provider, aborting. (#{$!})"
+        end
+      end
+    
+      def config
+        begin
+          directory = prompt("storage directory", @conf.storage)
+          while not directory_acceptable(directory)
+            directory = prompt("storage directory: ", @conf.storage)
+          end
+
+          email = @conf.email.join(', ') rescue nil
+          @conf.email = parse_emails(prompt("email", email))
+        
+          @conf.mail_server = prompt("mail server", @conf.mail_server)
+
+          logfile = prompt("log file(s) directory", @conf.logfile)
+          while not directory_acceptable(logfile)
+            logfile = prompt("log file(s) directory", @conf.logfile)
+          end
+          @conf.storage = directory
+          @conf.logfile = logfile
+          @conf.save
+        rescue 
+          nil
+        end
+      end
+        
+      def display(key, value, split = 40)
+        (split - key.size).times { print " " } if key.size < split
+        puts "#{key}: #{value}"
+      end
+    
+      def banner(str)
+        puts "\n#{str}"
+        str.size.times { print "-" }
+        puts "\n"
+      end
+      
+      def report(str)
+        puts "\n#{str}\n"
+      end
+
+      def indent(number)
+        number.times do
+          print "\t"
+        end
+      end
+    
+      def prompt(text, default = nil, split = 20)
+        prompt_text = "#{text} [#{default}]: "
+        (split - prompt_text.size).times { print " " } if prompt_text.size < split
+        value = readline(prompt_text, true)
+        raise RuntimeError.new("Exit loop") unless value
+        return value.empty? ? default : value
+      end
+    
+      def verify(url)
+        begin
+          client = OAI::Client.new(url, :redirects => false)
+          identify = client.identify
+          puts "Repository name \"#{identify.repository_name}\""
+          return url
+        rescue
+          if $!.to_s =~ /^Permanently Redirected to \[(.*)\?.*\]/
+            report "Provider redirected to: #{$1}"
+            verify($1)
+          else
+            puts "Error selecting repository: #{$!}"
+          end
+        end
+      end
+    
+      def metadata(url)
+        formats = []
+        client = OAI::Client.new url
+        response = client.list_metadata_formats
+        response.to_a.each do |format|
+          formats << format.prefix
+        end
+        formats
+      end
+      
+      def sets(url)
+        sets = []
+        client = OAI::Client.new url
+        response = client.list_sets
+        response.to_a.each do |set|
+          sets << set.spec
+        end
+        sets
+      end
+
+      def directory_acceptable(dir)
+        if not (dir && File.exists?(dir) && File.writable?(dir))
+          puts "Directory doesn't exist, or isn't writtable."
+          return false
+        end
+        true
+      end
+    
+      def expand_period(str)
+        return str if Config::PERIODS.include?(str)
+        Config::PERIODS.each { |p| return p if p =~ /^#{str}/}
+        nil
+      end
+    
+      def parse_emails(emails)
+        return nil unless emails
+        addresses = emails.split(/[,\s|\s|,]/)
+      end
+        
+      def list_config
+        display("storage directory", @conf.storage, 20)
+        display("email", @conf.email.join(', '), 20) if @conf.email
+        display("mail server", @conf.mail_server, 20) if @conf.mail_server
+        display("log location", @conf.logfile, 20) if @conf.logfile
+      end
+    
+      def list_sites
+        banner "Sites"
+        @conf.sites.each_key { |site| print_site(site) }
+      end
+    
+      def print_site(site)
+        puts site
+        @conf.sites[site].each { |k,v| display(k, v, 15)}
+      end
+    
+      def setup_cron
+        banner "Scheduling Automatic Harvesting"
+        puts "To activate automatic harvesting you must add an entry to"
+        puts "your scheduler.  Linux/Mac OS X users should add the following"
+        puts "entry to their crontabs:\n\n"
+        puts "0 0 * * * #{$0} -D\n\n"
+        puts "Windows users should use WinAt to schedule"
+        puts "#{$0} to run every night.\n\n\n"
+      end
+    
+    end
+
+  end
+end        
+