mrt-tind-harvester 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a8a12f69bf55200b7d6b02ee260281c3d1d7b050
+  data.tar.gz: 6ea06ac8dbff3b873ef69042512f3d81fb5b2e73
+SHA512:
+  metadata.gz: 7fdd4a906e4b22079c5a15f2eab8062dd551e6cc8e3ecf7d8909554b0bdec73ac84f616c12a862f03f854d57e3146880f59f1f82f5d3f7d38c2abce92222d7ee
+  data.tar.gz: dddc4f22729901b865d383dcfb63c8a0a04d9f4921a5c5a137ef6c619eff5886842747a6a5ffdae3e168e31bec189681d3e096fbe22b1cce52b7443d79ea0f5e

data/bin/mrt-tind-harvester

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+Dir.chdir(__dir__) { require 'bundler/setup' }
+
+require 'optparse'
+require 'optparse/time'
+require 'ostruct'
+require 'merritt/tind'
+
+USAGE = OpenStruct.new(
+  CONFIG: 'path to configuration file (required)'.freeze,
+  DRY_RUN: 'dry run (harvest, but do not submit or update last_harvest)'.freeze,
+  FROM: 'start date/time (inclusive) for selective harvesting'.freeze,
+  UNTIL: 'end date/time (inclusive) for selective harvesting'.freeze,
+  HELP: 'print help and exit'
+).freeze
+
+class TINDHarvesterApp
+  attr_reader :options
+
+  def initialize
+    @options = OpenStruct.new(dry_run: false, help: false)
+  end
+
+  def option_parser
+    @option_parser = OptionParser.new do |opts|
+      opts.on('-c CONFIG', '--config CONFIG', USAGE.CONFIG) { |config_file| options.config_file = config_file }
+      opts.on('-f', '--from DATETIME', Time, USAGE.FROM) { |from_time| options.from_time = to_utc_time(from_time) }
+      opts.on('-u', '--until DATETIME', Time, USAGE.UNTIL) { |until_time| options.until_time = to_utc_time(until_time) }
+      opts.on('-n', '--dry-run', USAGE.DRY_RUN) { options.dry_run = true }
+      opts.on('-h', '--help', USAGE.HELP) { options.help = true }
+    end
+  end
+
+  def do_harvest!
+    option_parser.parse!
+
+    print_help_and_exit(0) if options.help
+
+    harvester = Merritt::TIND::Harvester.from_file(config_file, dry_run: options.dry_run)
+    warn("Starting harvester; logging to #{harvester.log_path}")
+    harvester.process_feed!(from_time: options.from_time, until_time: options.until_time)
+  rescue StandardError => e
+    warn(e)
+    print_help_and_exit(1)
+  end
+
+  private
+
+  def config_file
+    config_file = options.config_file
+    return config_file if config_file && File.exist?(config_file)
+
+    exit_with_error('No configuration file specified') unless config_file
+    exit_with_error('The specified configuration file does not exist: ' + config_file)
+  end
+
+  def help
+    option_parser.to_s
+  end
+
+  def exit_with_error(msg)
+    warn(msg)
+    print_help_and_exit(1)
+  end
+
+  def print_help_and_exit(status)
+    warn(help)
+    exit(status)
+  end
+end
+
+app = TINDHarvesterApp.new
+app.do_harvest!

data/lib/merritt.rb

@@ -0,0 +1 @@
+Dir.glob(File.expand_path('merritt/*.rb', __dir__)).sort.each(&method(:require))

data/lib/merritt/tind.rb

@@ -0,0 +1 @@
+Dir.glob(File.expand_path('tind/*.rb', __dir__)).sort.each(&method(:require))

data/lib/merritt/tind/config.rb

@@ -0,0 +1,114 @@
+require 'pathname'
+require 'yaml'
+
+module Merritt
+  module TIND
+    class Config
+
+      attr_reader :config_h
+      attr_reader :config_path
+
+      def initialize(config_h = nil, config_yml: nil)
+        @config_h = config_h || {}
+        @config_path = Pathname.new(config_yml).realpath if config_yml
+      end
+
+      def oai_base_url
+        oai_config_h['base_url']
+      end
+
+      def oai_set
+        oai_config_h['set']
+      end
+
+      def stop_file_path
+        @stop_file_path ||= begin
+          stop_file = config_h['stop_file']
+          resolve_relative_path(stop_file)
+        end
+      end
+
+      def mrt_collection_ark
+        merritt_config_h['collection_ark']
+      end
+
+      def mrt_ingest_url
+        merritt_config_h['ingest_url']
+      end
+
+      def mrt_ingest_profile
+        merritt_config_h['ingest_profile']
+      end
+
+      def db_config_path
+        @db_config_path ||= begin
+          db = merritt_config_h['database']
+          resolve_relative_path(db)
+        end
+      end
+
+      def log_level
+        log_config_h['level']
+      end
+
+      def log_path
+        @log_path ||= begin
+          lp = log_config_h['file']
+          resolve_relative_path(lp)
+        end
+      end
+
+      def last_harvest_path
+        @last_harvest_path ||= begin
+          lh = config_h['last_harvest']
+          resolve_relative_path(lh)
+        end
+      end
+
+      private
+
+      def oai_config_h
+        config_h['oai'] || {}
+      end
+
+      def merritt_config_h
+        config_h['merritt'] || {}
+      end
+
+      def log_config_h
+        config_h['log'] || {}
+      end
+
+      def resolve_relative_path(filename)
+        return nil unless filename
+
+        pathname = Pathname.new(filename)
+        return pathname if pathname.absolute?
+        return pathname unless config_path
+
+        (config_path.parent + pathname).cleanpath
+      end
+
+      class << self
+
+        def from_file(config_yml)
+          # A missing config.yml is not normal
+          raise ArgumentError, "Can't read config from nil file" unless config_yml
+          raise ArgumentError, "Specified config file #{config_yml} does not exist" unless File.exist?(config_yml)
+
+          config_h = YAML.load_file(config_yml)
+          env_config = config_h[environment]
+          raise ArgumentError, "No configuration for environment '#{environment}' found in #{config_yml}" if env_config.nil? || env_config.empty?
+
+          Config.new(env_config, config_yml: config_yml)
+        end
+
+        def environment
+          %w[HARVESTER_ENV RAILS_ENV RACK_ENV].each { |v| return ENV[v] if ENV[v] }
+          'development'
+        end
+      end
+
+    end
+  end
+end

data/lib/merritt/tind/feed.rb

@@ -0,0 +1,29 @@
+require 'oai/client'
+
+module Merritt
+  module TIND
+    class Feed
+      include Enumerable
+
+      def initialize(resp)
+        @resp = ensure_full_response(resp)
+      end
+
+      def each
+        return enum_for(:each) unless block_given?
+
+        @resp.each { |oai_record| yield Record.from_oai(oai_record) }
+      end
+
+      private
+
+      def ensure_full_response(resp)
+        return resp unless resp.respond_to?(:resumption_token) # already wrapped
+        return resp unless resp.resumption_token # nothing to paginate
+
+        resp.full
+      end
+
+    end
+  end
+end

data/lib/merritt/tind/feed_processor.rb

@@ -0,0 +1,61 @@
+module Merritt
+  module TIND
+    class FeedProcessor
+
+      attr_reader :feed
+      attr_reader :harvester
+      attr_reader :server
+
+      def initialize(feed:, server:, harvester:)
+        @feed = feed
+        @server = server
+        @harvester = harvester
+      end
+
+      def process_feed!
+        feed.each { |r| process_record(r, server) }
+
+        log.debug("Updating #{config.last_harvest_path}:\n#{last_harvest_next.to_yaml.gsub(/^/, "\t")}")
+        update_last_harvest!
+      end
+
+      private
+
+      def config
+        harvester.config
+      end
+
+      def log
+        harvester.log
+      end
+
+      def dry_run?
+        harvester.dry_run?
+      end
+
+      def last_harvest_next
+        @last_harvest_next ||= begin
+          last_harvest = harvester.last_harvest
+          last_harvest ? last_harvest.clone : LastHarvest.new
+        end
+      end
+
+      def update_last_harvest!
+        if dry_run?
+          log.info("Dry run: #{config.last_harvest_path} not updated")
+        else
+          last_harvest_next.write_to(config.last_harvest_path)
+        end
+      end
+
+      def process_record(r, server)
+        RecordProcessor.new(r, harvester, server).process_record!
+        @last_harvest_next = last_harvest_next.update(success: r)
+      rescue StandardError => e
+        # TODO: can we identify failures after submission?
+        log.warn(e)
+        @last_harvest_next = last_harvest_next.update(failure: r)
+      end
+    end
+  end
+end

data/lib/merritt/tind/files.rb

@@ -0,0 +1,66 @@
+require 'timeout'
+
+module Merritt
+  module TIND
+    module Files
+      DEFAULT_TIMEOUT_SECS = 5
+      DEFAULT_SLEEP_INTERVAL_SECS = 0.1
+
+      class << self
+
+        def with_lock(filename)
+          f = acquire_lock(filename)
+          yield f
+        ensure
+          f.flock(File::LOCK_UN) if f
+        end
+
+        def rotate_and_lock(filename)
+          with_lock(filename) do |f|
+            if File.size?(filename)
+              rotating(filename) { |f1| yield f1 }
+            else
+              yield f
+            end
+          end
+        end
+
+        private
+
+        def rotating(filename)
+          rotate_to = rotated_name(filename)
+
+          File.rename(filename, rotate_to)
+          with_lock(filename) { |f| yield f }
+        end
+
+        def rotated_name(filename)
+          loop do
+            renamed_file = filename + '-' + Time.now.utc.iso8601(3)
+            return renamed_file unless File.exist?(renamed_file)
+
+            sleep(DEFAULT_SLEEP_INTERVAL_SECS)
+          end
+        end
+
+        def acquire_lock(filename)
+          Timeout.timeout(DEFAULT_TIMEOUT_SECS) do
+            loop do
+              f = File.open(filename, 'a+')
+              f.flock(File::LOCK_EX)
+              return f if File.identical?(filename, f)
+
+              # we do cover this, but it's called in a subprocess
+              # so SimpleCov can't tell we've called it
+              # :nocov:
+              f.flock(File::LOCK_UN)
+              sleep(DEFAULT_SLEEP_INTERVAL_SECS)
+              # :nocov:
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/merritt/tind/harvester.rb

@@ -0,0 +1,138 @@
+require 'faraday_middleware'
+require 'oai/client'
+
+module Merritt
+  module TIND
+    class Harvester
+
+      attr_reader :config
+
+      def initialize(config, dry_run: false)
+        @config = config
+        @dry_run = dry_run
+
+        set_str = config.oai_set ? "'#{config.oai_set}'" : '<nil>'
+        log.info("Initializing harvester for base URL #{oai_base_uri}, set #{set_str} => collection #{config.mrt_collection_ark}")
+      end
+
+      def process_feed!(from_time: nil, until_time: nil)
+        return if stop_file_present?
+
+        opts = to_oai_opts(from_time, until_time)
+        log.info("harvesting #{query_uri(opts)}")
+        resp = oai_client.list_records(opts)
+        feed = Feed.new(resp)
+        return process_feed(feed, nil) if dry_run?
+
+        with_server { |server| process_feed(feed, server) }
+      end
+
+      def with_server
+        server = Mrt::Ingest::OneTimeServer.new
+        server.start_server
+        yield server
+      ensure
+        server.join_server
+      end
+
+      def dry_run?
+        @dry_run
+      end
+
+      def stop_file_present?
+        stop_file_path = config.stop_file_path
+        stop_file_present = stop_file_path && stop_file_path.exist?
+        log.info("Stop file present: #{config.stop_file_path}") if stop_file_present
+        stop_file_present
+      end
+
+      def last_harvest
+        # read this from the file every time
+        LastHarvest.from_file(config.last_harvest_path)
+      end
+
+      def oai_client
+        @oai_client ||= Harvester.oai_client_for(oai_base_uri)
+      end
+
+      def oai_base_uri
+        @oai_base_uri ||= URI.parse(config.oai_base_url)
+      end
+
+      def mrt_collection_ark
+        config.mrt_collection_ark
+      end
+
+      def mrt_ingest_profile
+        config.mrt_ingest_profile
+      end
+
+      def mrt_inv_db
+        @mrt_inv_db ||= InventoryDB.from_file(config.db_config_path)
+      end
+
+      def mrt_ingest_client
+        # TODO: secure way to get username and password?
+        @mrt_ingest_client ||= Mrt::Ingest::Client.new(config.mrt_ingest_url)
+      end
+
+      def log
+        @log ||= Logging.new_logger(log_path, config.log_level)
+      end
+
+      def determine_from_time(from_time = nil)
+        return from_time if from_time
+
+        lh = last_harvest
+        oldest_failed = lh.oldest_failed_datestamp
+        return oldest_failed if oldest_failed
+
+        lh.newest_success_datestamp
+      end
+
+      def log_path
+        config.log_path
+      end
+
+      private
+
+      def process_feed(feed, server)
+        return unless feed
+
+        feed_processor = FeedProcessor.new(feed: feed, server: server, harvester: self)
+        feed_processor.process_feed!
+      end
+
+      def query_uri(opts)
+        query = '?ListRecords'
+        opts.each { |k, v| query << "&#{k}=#{v}" } if opts
+        oai_base_uri.merge(query)
+      end
+
+      def to_oai_opts(from_time, until_time)
+        from_time = determine_from_time(from_time)
+        from_iso8601, until_iso8601 = Times.iso8601_range(from_time, until_time)
+        { from: from_iso8601, until: until_iso8601, set: config.oai_set }.compact
+      end
+
+      class << self
+
+        def from_file(config_yml, dry_run: false)
+          config = Config.from_file(config_yml)
+          Harvester.new(config, dry_run: dry_run)
+        end
+
+        def oai_client_for(base_uri)
+          # Workaround for https://github.com/code4lib/ruby-oai/issues/45
+          http_client = Faraday.new(base_uri) do |conn|
+            conn.request(:retry, max: 5, retry_statuses: 503)
+            conn.response(:follow_redirects, limit: 5)
+            conn.adapter(:net_http)
+          end
+          OAI::Client.new(base_uri.to_s, http: http_client)
+        end
+      end
+
+    end
+  end
+end

data/lib/merritt/tind/inventory_db.rb

@@ -0,0 +1,54 @@
+require 'mysql2'
+
+module Merritt
+  module TIND
+    class InventoryDB
+
+      attr_reader :db_connection
+
+      def initialize(db_config_h)
+        @db_connection = Mysql2::Client.new(db_config_h)
+      end
+
+      class << self
+        def from_file(db_config_path)
+          raise "Can't connect to nil database" unless db_config_path
+          raise ArgumentError, "Specified database config #{db_config_path} does not exist" unless File.exist?(db_config_path)
+
+          db_config = YAML.load_file(db_config_path)
+          env_db_config = db_config[Config.environment]
+          InventoryDB.new(env_db_config)
+        end
+      end
+
+      def find_existing_object(local_id, collection_ark)
+        result = existing_object_stmt.execute(local_id, collection_ark).first
+        return nil unless result
+
+        OpenStruct.new(result)
+      end
+
+      private
+
+      # TODO: is this right or should we be using erc_where? or both?
+      EXISTING_OBJECT_SQL = <<~SQL.freeze
+        SELECT o.*
+          FROM inv_objects AS o
+               JOIN inv_collections_inv_objects AS co
+                 ON co.inv_object_id = o.id
+               JOIN inv_collections AS c
+                 ON c.id = co.inv_collection_id
+               JOIN inv_localids AS li
+                 ON li.inv_object_ark = o.ark
+         WHERE li.local_id = ?
+           AND c.ark = ?
+        LIMIT 1
+      SQL
+
+      def existing_object_stmt
+        @existing_object_stmt ||= db_connection.prepare(EXISTING_OBJECT_SQL)
+      end
+
+    end
+  end
+end

data/lib/merritt/tind/last_harvest.rb

@@ -0,0 +1,84 @@
+require 'yaml'
+
+module Merritt
+  module TIND
+
+    class LastHarvest
+
+      OLDEST_FAILED = 'oldest_failed'.freeze
+      NEWEST_SUCCESS = 'newest_success'.freeze
+
+      attr_reader :oldest_failed
+      attr_reader :newest_success
+
+      # @param oldest_failed [Record, nil] the oldest record that failed to submit
+      # @param newest_success [Record, nil] the newest record successfully submitted
+      def initialize(oldest_failed: nil, newest_success: nil)
+        @oldest_failed = oldest_failed
+        @newest_success = newest_success
+      end
+
+      def to_h
+        {
+          OLDEST_FAILED => (oldest_failed && oldest_failed.to_h),
+          NEWEST_SUCCESS => (newest_success && newest_success.to_h)
+        }
+      end
+
+      def to_yaml
+        to_h.to_yaml
+      end
+
+      def write_to(last_harvest_yml)
+        Files.rotate_and_lock(last_harvest_yml) do |f|
+          f.write(to_yaml)
+        end
+      end
+
+      def oldest_failed_datestamp
+        oldest_failed && oldest_failed.datestamp
+      end
+
+      def newest_success_datestamp
+        newest_success && newest_success.datestamp
+      end
+
+      def update(success: nil, failure: nil)
+        LastHarvest.new(
+          newest_success: Record.later(success, newest_success),
+          oldest_failed: Record.earlier(failure, oldest_failed)
+        )
+      end
+
+      private
+
+      def initialize_dup(source)
+        @newest_success = source.newest_success && source.newest_success.dup
+        @oldest_failed = source.oldest_failed && source.oldest_failed.dup
+      end
+
+      def initialize_clone(source)
+        @newest_success = source.newest_success && source.newest_success.clone
+        @oldest_failed = source.oldest_failed && source.oldest_failed.clone
+      end
+
+      class << self
+        def from_file(last_harvest_yml)
+          return from_hash(YAML.load_file(last_harvest_yml)) if last_harvest_yml && File.exist?(last_harvest_yml)
+
+          # A missing last_yarvest.yml is normal
+          LastHarvest.new
+        end
+
+        def from_hash(h)
+          LastHarvest.new(
+            oldest_failed: Record.from_hash(h[OLDEST_FAILED]),
+            newest_success: Record.from_hash(h[NEWEST_SUCCESS])
+          )
+        end
+
+      end
+
+    end
+  end
+end

data/lib/merritt/tind/logging.rb

@@ -0,0 +1,42 @@
+require 'logger'
+require 'time'
+
+module Merritt
+  module TIND
+    module Logging
+      NUM_LOG_FILES = 10
+      DEFAULT_LOG_LEVEL = Logger::DEBUG
+
+      class << self
+        def fmt_log(severity, datetime, _, msg)
+          "#{datetime.iso8601}\t#{severity}\t#{msg}\n"
+        end
+
+        def new_logger(log_dev = nil, log_level = nil)
+          log_dev ||= STDERR
+          log_level ||= Logger::DEBUG
+
+          created_log_dir = ensure_log_dir(log_dev)
+          logger = Logger.new(log_dev, NUM_LOG_FILES, level: log_level, formatter: Logging.method(:fmt_log))
+          created_log_dir.each { |d| logger.info("Created log directory #{d}") } if created_log_dir
+          logger
+        end
+
+        private
+
+        def io_like?(log_dev)
+          # This is how Ruby's Logger identifies an IO-like log device
+          log_dev.respond_to?(:write) && log_dev.respond_to?(:close)
+        end
+
+        def ensure_log_dir(log_dev)
+          return if io_like?(log_dev)
+
+          # assume it's a string or a pathname
+          log_dir = Pathname.new(log_dev).parent
+          FileUtils.mkdir_p(log_dir) unless log_dir.exist?
+        end
+      end
+    end
+  end
+end

data/lib/merritt/tind/module_info.rb

@@ -0,0 +1,7 @@
+module Merritt
+  module TIND
+    NAME = 'mrt-tind-harvester'.freeze
+    VERSION = '0.0.1'.freeze
+    COPYRIGHT = 'Copyright (c) 2019 The Regents of the University of California'.freeze
+  end
+end

data/lib/merritt/tind/record.rb

@@ -0,0 +1,109 @@
+require 'oai/client'
+require 'time'
+
+module Merritt
+  module TIND
+    class Record
+      IDENTIFIER = 'identifier'.freeze
+      DATESTAMP = 'datestamp'.freeze
+
+      attr_reader :identifier
+      attr_reader :datestamp
+      attr_reader :metadata
+
+      def initialize(identifier:, datestamp:, oai_metadata: nil)
+        @identifier = identifier
+        @datestamp = datestamp
+        @metadata = oai_metadata
+      end
+
+      def erc
+        # TODO: something smarter when we know the real requirements
+        {
+          'what' => identifier,
+          'where' => local_id,
+          'when' => dc_dates.first || datestamp,
+          'when/created' => dc_dates.first || datestamp,
+          'when/modified' => datestamp
+        }
+      end
+
+      def dc_identifiers
+        @dc_identifiers ||= REXML::XPath.match(metadata, './/dc:identifier').map(&:text)
+      end
+
+      def dc_dates
+        @dc_dates ||= begin
+          REXML::XPath.match(metadata, './/dc:date')
+            .map(&:text)
+            .map { |t| Time.parse(t) }
+        end
+      end
+
+      def dc_titles
+        @dc_titles ||= REXML::XPath.match(metadata, './/dc:title').map(&:text)
+      end
+
+      def dc_creators
+        @dc_creators ||= REXML::XPath.match(metadata, './/dc:creator').map(&:text)
+      end
+
+      def content_uri
+        @content_uri ||= begin
+          # TODO: something smarter when we know the real requirements
+          content_url = dc_identifiers.find do |dc_id|
+            dc_id.start_with?('http') && dc_id.end_with?('jpg')
+          end
+          content_url && URI.parse(content_url)
+        end
+      end
+
+      def local_id
+        # TODO: something smarter when we know the real requirements
+        dc_identifiers.first || identifier
+      end
+
+      def to_h
+        { IDENTIFIER => identifier, DATESTAMP => datestamp }
+      end
+
+      class << self
+
+        def later(r1, r2)
+          return r1 if r2.nil?
+          return r2 if r1.nil?
+          return r1 if (r1.datestamp <=> r2.datestamp) > 0
+
+          r2
+        end
+
+        def earlier(r1, r2)
+          return r1 if r2.nil?
+          return r2 if r1.nil?
+          return r1 if (r1.datestamp <=> r2.datestamp) < 0
+
+          r2
+        end
+
+        def from_hash(h)
+          return unless h
+
+          Record.new(identifier: h[IDENTIFIER], datestamp: h[DATESTAMP])
+        end
+
+        # Constructs a new {Record} wrapping the specified record.
+        #
+        # @param oai_record [OAI::Record] An OAI record as returned by `OAI::Client`
+        def from_oai(oai_record)
+          raise ArgumentError, "can't parse nil record" unless oai_record
+
+          header = oai_record.header
+          identifier = header.identifier
+          datestamp = header.datestamp && Time.parse(header.datestamp)
+          Record.new(identifier: identifier, datestamp: datestamp, oai_metadata: oai_record.metadata)
+        end
+
+      end
+    end
+  end
+end

data/lib/merritt/tind/record_processor.rb

@@ -0,0 +1,90 @@
+require 'mrt/ingest'
+require 'ostruct'
+
+module Merritt
+  module TIND
+    class RecordProcessor
+
+      USER_AGENT = 'Merritt/TIND Harvester'.freeze
+
+      attr_reader :record
+      attr_reader :harvester
+      attr_reader :server
+
+      def initialize(record, harvester, server)
+        @record = record
+        @harvester = harvester
+        @server = server
+      end
+
+      def process_record!
+        return true if already_up_to_date?
+
+        log.info("Processing record: #{local_id} (content: #{content_uri}")
+        return true if harvester.dry_run?
+
+        submit_to_ingest!
+      end
+
+      private
+
+      def submit_to_ingest!
+        ingest_object.add_component(content_uri)
+        response = ingest_object.start_ingest(ingest_client, ingest_profile, USER_AGENT)
+        log.info("Batch #{response.batch_id} queued at #{response.submission_date}")
+        true # TODO: is there anything in the response that might cause us to return false?
+      end
+
+      def already_up_to_date?
+        @already_up_to_date ||= existing_object && existing_object.modified >= record.datestamp
+      end
+
+      def existing_object
+        @existing_object = (find_existing_object || false) if @existing_object.nil?
+        @existing_object
+      end
+
+      def find_existing_object
+        inv_db.find_existing_object(local_id, collection_ark)
+      end
+
+      def inv_db
+        harvester.mrt_inv_db
+      end
+
+      def local_id
+        record.local_id
+      end
+
+      def content_uri
+        record.content_uri
+      end
+
+      def collection_ark
+        harvester.mrt_collection_ark
+      end
+
+      def ingest_client
+        harvester.mrt_ingest_client
+      end
+
+      def ingest_profile
+        harvester.mrt_ingest_profile
+      end
+
+      def log
+        harvester.log
+      end
+
+      def ingest_object
+        @ingest_object ||= begin
+          Mrt::Ingest::IObject.new(
+            erc: record.erc,
+            server: server,
+            local_identifier: record.local_id
+          )
+        end
+      end
+    end
+  end
+end

data/lib/merritt/tind/times.rb

@@ -0,0 +1,35 @@
+require 'time'
+
+module Merritt
+  module TIND
+    module Times
+      class << self
+        def iso8601_range(from_time, until_time)
+          from_time, until_time = valid_range(from_time, until_time)
+          [
+            from_time && from_time.iso8601,
+            until_time && until_time.iso8601
+          ]
+        end
+
+        private
+
+        def valid_range(from_time, until_time)
+          from_time, until_time = [from_time, until_time].map(&method(:utc_or_nil))
+          if from_time && until_time
+            raise RangeError, "from_time #{from_time} must be <= until_time #{until_time}" if from_time > until_time
+          end
+
+          [from_time, until_time]
+        end
+
+        def utc_or_nil(time)
+          return time.utc if time.respond_to?(:utc)
+          return unless time
+
+          raise ArgumentError, "time #{time} does not appear to be a Time"
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,298 @@
+--- !ruby/object:Gem::Specification
+name: mrt-tind-harvester
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- David Moles
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-05-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mrt-ingest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: oai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: capistrano
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: capistrano-bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.68'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.68'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.33'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.33'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: standalone_migrations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+description: Harvests TIND OAI-PMH feed to identify files for ingest into Merritt
+email:
+- david.moles@ucop.edu
+executables:
+- mrt-tind-harvester
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/mrt-tind-harvester
+- lib/merritt.rb
+- lib/merritt/tind.rb
+- lib/merritt/tind/config.rb
+- lib/merritt/tind/feed.rb
+- lib/merritt/tind/feed_processor.rb
+- lib/merritt/tind/files.rb
+- lib/merritt/tind/harvester.rb
+- lib/merritt/tind/inventory_db.rb
+- lib/merritt/tind/last_harvest.rb
+- lib/merritt/tind/logging.rb
+- lib/merritt/tind/module_info.rb
+- lib/merritt/tind/record.rb
+- lib/merritt/tind/record_processor.rb
+- lib/merritt/tind/times.rb
+homepage: https://github.com/CDLUC3/mrt-tind-harvester
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.1
+signing_key: 
+specification_version: 4
+summary: TIND harvester for Merritt
+test_files: []