pushmi_pullyu 0.2.0

data/lib/pushmi_pullyu/aip.rb

@@ -0,0 +1,22 @@
+require 'fileutils'
+
+module PushmiPullyu::AIP
+  class NoidInvalid < StandardError; end
+
+  def create(noid)
+    raise NoidInvalid if noid.blank? || noid.include?('/')
+
+    aip_directory = "#{PushmiPullyu.options[:workdir]}/#{noid}"
+    aip_filename = "#{aip_directory}.tar"
+
+    PushmiPullyu::AIP::Downloader.new(noid, aip_directory).run
+    PushmiPullyu::AIP::Creator.new(noid, aip_directory, aip_filename).run
+
+    yield aip_filename
+
+    FileUtils.rm_rf(aip_filename) if File.exist?(aip_filename)
+    FileUtils.rm_rf(aip_directory) if File.exist?(aip_directory)
+  end
+
+  module_function :create
+end

data/lib/pushmi_pullyu/aip/creator.rb

@@ -0,0 +1,44 @@
+require 'minitar'
+require 'bagit'
+require 'fileutils'
+
+class PushmiPullyu::AIP::Creator
+
+  class BagInvalid < StandardError; end
+
+  # Assumption: the AIP has already been downloaded
+
+  def initialize(noid, aip_directory, aip_filename)
+    @noid = noid
+    @aip_directory = aip_directory
+    @aip_filename = aip_filename
+  end
+
+  def run
+    bag_aip
+    tar_bag
+  end
+
+  private
+
+  def bag_aip
+    bag = BagIt::Bag.new(@aip_directory)
+    bag.manifest!
+    raise BagInvalid unless bag.valid?
+  end
+
+  def tar_bag
+    # We want to change the directory to the work directory path so we get the tar file to be exactly
+    # the contents of the noid directory and not the entire work directory structure. For example the noid.tar
+    # contains just the noid directory instead of having the noid.tar contain the tmp directory
+    # which contains the workdir directory and then finally the noid directory
+
+    # Before we change directorys, we need to calculate the absolute filepath of our aip filename
+    tar_aip_filename = File.expand_path(@aip_filename)
+
+    Dir.chdir(PushmiPullyu.options[:workdir]) do
+      Minitar.pack(@noid, File.open(tar_aip_filename, 'wb'))
+    end
+  end
+
+end

data/lib/pushmi_pullyu/aip/downloader.rb

@@ -0,0 +1,177 @@
+require 'fileutils'
+require 'ostruct'
+require 'rdf'
+require 'rdf/n3'
+
+# Download all of the metadata/datastreams and associated data
+# related to an object
+class PushmiPullyu::AIP::Downloader
+
+  class NoContentFilename < StandardError; end
+
+  def initialize(noid, aip_directory)
+    @noid = noid
+    @aip_directory = aip_directory
+  end
+
+  def run
+    make_directories
+
+    PushmiPullyu.logger.info("#{@noid}: Retreiving data from Fedora ...")
+
+    [:main_object, :fixity, :content_datastream_metadata, :versions, :thumbnail,
+     :characterization, :fedora3foxml, :fedora3foxml_metadata].each do |item|
+      path_spec = aip_paths[item]
+      download_and_log(path_spec, PushmiPullyu::AIP::FedoraFetcher.new(@noid))
+    end
+
+    # Need content filename from metadata
+    path_spec = OpenStruct.new(
+      remote: '/content',
+      local: content_filename, # lookup filename derived from metadata
+      optional: false
+    )
+    download_and_log(path_spec, PushmiPullyu::AIP::FedoraFetcher.new(@noid))
+
+    download_permissions
+  end
+
+  private
+
+  def download_and_log(path_spec, fedora_fetcher)
+    output_file = path_spec.local
+
+    log_fetching(fedora_fetcher.object_url(path_spec.remote), output_file)
+
+    is_rdf = (output_file !~ /\.n3$/)
+
+    is_success = fedora_fetcher.download_object(output_file,
+                                                url_extra: path_spec.remote,
+                                                optional: path_spec.optional,
+                                                is_rdf: is_rdf)
+    log_saved(is_success, output_file)
+  end
+
+  def download_permissions
+    PushmiPullyu.logger.info("#{@noid}: looking up permissions from Solr ...")
+    results = PushmiPullyu::AIP::SolrFetcher.new(@noid).fetch_permission_object_ids
+    if results.empty?
+      PushmiPullyu.logger.info("#{@noid}: permissions not found")
+    else
+      results.each do |permission_id|
+        PushmiPullyu.logger.info("#{@noid}: permission object #{permission_id} found")
+        download_permission(permission_id)
+      end
+    end
+  end
+
+  def download_permission(permission_id)
+    path_spec = OpenStruct.new(
+      remote: nil,
+      local: "#{aip_dirs.metadata}/permission_#{permission_id}.n3",
+      optional: false
+    )
+    download_and_log(path_spec, PushmiPullyu::AIP::FedoraFetcher.new(permission_id))
+  end
+
+  ### Logging
+
+  def log_fetching(url, output_file)
+    message = "#{@noid}: #{output_file} -- fetching from #{url} ..."
+    PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
+  end
+
+  def log_saved(is_success, output_file)
+    message = "#{@noid}: #{output_file} -- #{is_success ? 'saved' : 'not_found'}"
+    PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
+  end
+
+  ### Directories
+
+  def aip_dirs
+    @aip_dirs ||= OpenStruct.new(
+      objects: "#{@aip_directory}/data/objects",
+      metadata: "#{@aip_directory}/data/objects/metadata",
+      logs: "#{@aip_directory}/data/logs",
+      thumbnails: "#{@aip_directory}/data/thumbnails"
+    )
+  end
+
+  def make_directories
+    clean_directories
+    PushmiPullyu.logger.debug("#{@noid}: Creating directories ...")
+    aip_dirs.to_h.values.each do |path|
+      FileUtils.mkdir_p(path)
+    end
+    PushmiPullyu.logger.debug("#{@noid}: Creating directories done")
+  end
+
+  def clean_directories
+    return unless File.exist?(@aip_directory)
+    PushmiPullyu.logger.debug("#{@noid}: Nuking directories ...")
+    FileUtils.rm_rf(@aip_directory)
+  end
+
+  ### Files
+
+  def aip_paths
+    @aip_paths ||= OpenStruct.new(
+      main_object: OpenStruct.new(
+        remote: nil, # Base path
+        local: "#{aip_dirs.metadata}/object_metadata.n3",
+        optional: false
+      ),
+      fixity: OpenStruct.new(
+        remote: '/content/fcr:fixity',
+        local: "#{aip_dirs.logs}/content_fixity_report.n3",
+        optional: false
+      ),
+      content_datastream_metadata: OpenStruct.new(
+        remote: '/content/fcr:metadata',
+        local: "#{aip_dirs.metadata}/content_fcr_metadata.n3",
+        optional: false
+      ),
+      versions: OpenStruct.new(
+        remote: '/content/fcr:versions',
+        local: "#{aip_dirs.metadata}/content_versions.n3",
+        optional: false
+      ),
+
+      # Optional downloads
+      thumbnail: OpenStruct.new(
+        remote: '/thumbnail',
+        local: "#{aip_dirs.thumbnails}/thumbnail",
+        optional: true
+      ),
+      characterization: OpenStruct.new(
+        remote: '/characterization',
+        local: "#{aip_dirs.logs}/content_characterization.n3",
+        optional: true
+      ),
+      fedora3foxml: OpenStruct.new(
+        remote: '/fedora3foxml',
+        local: "#{aip_dirs.metadata}/fedora3foxml.xml",
+        optional: true
+      ),
+      fedora3foxml_metadata: OpenStruct.new(
+        remote: '/fedora3foxml/fcr:metadata',
+        local: "#{aip_dirs.metadata}/fedora3foxml.n3",
+        optional: true
+      )
+    ).freeze
+  end
+
+  # Extract filename from main object metadata
+  def content_filename
+    filename_predicate = RDF::URI('info:fedora/fedora-system:def/model#downloadFilename')
+
+    graph = RDF::Graph.load(aip_paths.main_object.local)
+
+    graph.query(predicate: filename_predicate) do |results|
+      return "#{aip_dirs.objects}/#{results.object}"
+    end
+
+    raise NoContentFilename
+  end
+
+end

data/lib/pushmi_pullyu/aip/fedora_fetcher.rb

@@ -0,0 +1,59 @@
+require 'net/http'
+
+class PushmiPullyu::AIP::FedoraFetcher
+
+  class FedoraFetchError < StandardError; end
+
+  RDF_FORMAT = 'text/rdf+n3'.freeze
+
+  def initialize(noid)
+    @noid = noid
+  end
+
+  def object_url(url_extra = nil)
+    url = "#{PushmiPullyu.options[:fedora][:url]}#{base_path}/#{pairtree}"
+    url += url_extra if url_extra
+    url
+  end
+
+  # Return true on success, raise an error otherwise
+  # (or use 'optional' to return false on 404)
+  def download_object(download_path, url_extra: nil,
+                      optional: false, is_rdf: false)
+
+    uri = URI(object_url(url_extra))
+
+    request = Net::HTTP::Get.new(uri)
+    request.basic_auth(PushmiPullyu.options[:fedora][:user],
+                       PushmiPullyu.options[:fedora][:password])
+
+    request['Accept'] = RDF_FORMAT if is_rdf
+
+    response = Net::HTTP.start(uri.hostname, uri.port) do |http|
+      http.request(request)
+    end
+
+    if response.is_a?(Net::HTTPSuccess)
+      file = File.open(download_path, 'wb')
+      file.write(response.body)
+      file.close
+      return true
+    elsif response.is_a?(Net::HTTPNotFound)
+      raise FedoraFetchError unless optional
+      return false
+    else
+      raise FedoraFetchError
+    end
+  end
+
+  private
+
+  def pairtree
+    "#{@noid[0..1]}/#{@noid[2..3]}/#{@noid[4..5]}/#{@noid[6..7]}/#{@noid}"
+  end
+
+  def base_path
+    PushmiPullyu.options[:fedora][:base_path]
+  end
+
+end

data/lib/pushmi_pullyu/aip/solr_fetcher.rb

@@ -0,0 +1,33 @@
+require 'json'
+require 'net/http'
+
+class PushmiPullyu::AIP::SolrFetcher
+
+  class SolrFetchError < StandardError; end
+
+  def initialize(noid)
+    @noid = noid
+  end
+
+  def fetch_permission_object_ids
+    hash = JSON.parse(run_query_json)
+
+    return [] if hash['response']['docs'].empty?
+
+    hash['response']['docs'].map { |hit| hit['id'] }
+  end
+
+  private
+
+  # Return fetched results, else raise an error
+  def run_query_json
+    response = Net::HTTP.get_response(
+      URI("#{PushmiPullyu.options[:solr][:url]}/select?q=accessTo_ssim:#{@noid}&fl=id&wt=json")
+    )
+
+    return response.body if response.is_a?(Net::HTTPSuccess)
+
+    raise SolrFetchError
+  end
+
+end

data/lib/pushmi_pullyu/cli.rb

@@ -0,0 +1,255 @@
+require 'erb'
+require 'fileutils'
+require 'optparse'
+require 'rollbar'
+require 'singleton'
+require 'yaml'
+
+# CLI runner
+class PushmiPullyu::CLI
+
+  include Singleton
+  include PushmiPullyu::Logging
+
+  COMMANDS = ['start', 'stop', 'restart', 'reload', 'run', 'zap', 'status'].freeze
+
+  def initialize
+    PushmiPullyu.server_running = true # set to false by interrupt signal trap
+    PushmiPullyu.reset_logger = false # set to true by SIGHUP trap
+  end
+
+  def parse(argv = ARGV)
+    opts = parse_options(argv)
+    opts[:daemonize] = true if COMMANDS.include? argv[0]
+    opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
+
+    PushmiPullyu.options = opts
+  end
+
+  def run
+    configure_rollbar
+    begin
+      if options[:daemonize]
+        start_server_as_daemon
+      else
+        # If we're running in the foreground sync the output.
+        $stdout.sync = $stderr.sync = true
+        start_server
+      end
+    # rubocop:disable Lint/RescueException
+    rescue Exception => e
+      Rollbar.error(e)
+      raise e
+    end
+  end
+
+  def start_server
+    setup_signal_traps
+
+    setup_log
+    print_banner
+
+    run_tick_loop
+  end
+
+  private
+
+  def configure_rollbar
+    Rollbar.configure do |config|
+      config.enabled = false unless options[:rollbar_token].present?
+      config.access_token = options[:rollbar_token]
+    end
+  end
+
+  def options
+    PushmiPullyu.options
+  end
+
+  def parse_config(config_file)
+    opts = {}
+    if File.exist?(config_file)
+      opts = YAML.safe_load(ERB.new(IO.read(config_file)).result).deep_symbolize_keys || opts
+    end
+
+    opts
+  end
+
+  # Parse the options.
+  def parse_options(argv)
+    opts = {}
+
+    @parsed_opts = OptionParser.new do |o|
+      o.banner = 'Usage: pushmi_pullyu [options] [start|stop|restart|run]'
+      o.separator ''
+      o.separator 'Specific options:'
+
+      o.on('-a', '--minimum-age AGE',
+           Float, 'Minimum amount of time an item must spend in the queue, in seconds.') do |minimum_age|
+        opts[:minimum_age] = minimum_age
+      end
+
+      o.on('-d', '--debug', 'Enable debug logging') do
+        opts[:debug] = true
+      end
+
+      o.on('-r', '--rollbar-token TOKEN', 'Enable error reporting to Rollbar') do |token|
+        opts[:rollbar_token] = token if token.present?
+      end
+
+      o.on '-C', '--config PATH', 'Path for YAML config file' do |config_file|
+        opts[:config_file] = config_file
+      end
+
+      o.on('-L', '--logdir PATH', 'Path for directory to store log files') do |logdir|
+        opts[:logdir] = logdir
+      end
+
+      o.on('-D', '--piddir PATH', 'Path for directory to store pid files') do |piddir|
+        opts[:piddir] = piddir
+      end
+
+      o.on('-W', '--workdir PATH', 'Path for directory where AIP creation work takes place in') do |workdir|
+        opts[:workdir] = workdir
+      end
+
+      o.on('-N', '--process_name NAME', 'Name of the application process') do |process_name|
+        opts[:process_name] = process_name
+      end
+
+      o.on('-m', '--monitor', 'Start monitor process for a deamon') do
+        opts[:monitor] = true
+      end
+
+      o.on('-q', '--queue NAME', 'Name of the queue to read from') do |queue|
+        opts[:queue_name] = queue
+      end
+
+      o.separator ''
+      o.separator 'Common options:'
+
+      o.on_tail('-v', '--version', 'Show version') do
+        puts "PushmiPullyu version: #{PushmiPullyu::VERSION}"
+        exit
+      end
+
+      o.on_tail('-h', '--help', 'Show this message') do
+        puts o
+        exit
+      end
+    end.parse!(argv)
+
+    ['config/pushmi_pullyu.yml', 'config/pushmi_pullyu.yml.erb'].each do |filename|
+      opts[:config_file] ||= filename if File.exist?(filename)
+    end
+
+    opts
+  end
+
+  def print_banner
+    logger.info "Loading PushmiPullyu #{PushmiPullyu::VERSION}"
+    logger.info "Running in #{RUBY_DESCRIPTION}"
+    logger.info 'Starting processing, hit Ctrl-C to stop' unless options[:daemonize]
+  end
+
+  def rotate_logs
+    PushmiPullyu::Logging.reopen
+    Daemonize.redirect_io(PushmiPullyu.application_log_file) if options[:daemonize]
+    PushmiPullyu.reset_logger = false
+  end
+
+  def run_preservation_cycle
+    item = queue.wait_next_item
+
+    # add additional information about the error context to errors that occur while processing this item.
+    Rollbar.scoped(noid: item) do
+      begin
+        # Download AIP from Fedora, bag and tar AIP directory and cleanup after block code
+        PushmiPullyu::AIP.create(item) do |aip_filename|
+          # Push tarred AIP to swift API
+          deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
+          # Log successful preservation event to the log files
+          PushmiPullyu::Logging.log_preservation_event(deposited_file)
+        end
+      rescue => e
+        Rollbar.error(e)
+        logger.error(e)
+        # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
+        # move on to the next item.
+      end
+    end
+  end
+
+  def run_tick_loop
+    while PushmiPullyu.server_running?
+      run_preservation_cycle
+      rotate_logs if PushmiPullyu.reset_logger?
+    end
+  end
+
+  def setup_log
+    if options[:daemonize]
+      PushmiPullyu::Logging.initialize_logger(PushmiPullyu.application_log_file)
+    else
+      logger.formatter = PushmiPullyu::Logging::SimpleFormatter.new
+    end
+    logger.level = ::Logger::DEBUG if options[:debug]
+  end
+
+  def setup_signal_traps
+    Signal.trap('INT') { shutdown }
+    Signal.trap('TERM') { shutdown }
+    Signal.trap('HUP') { PushmiPullyu.reset_logger = true }
+  end
+
+  def queue
+    @queue ||= PushmiPullyu::PreservationQueue.new(connection: {
+                                                     host: options[:redis][:host],
+                                                     port: options[:redis][:port]
+                                                   },
+                                                   queue_name: options[:queue_name],
+                                                   age_at_least: options[:minimum_age])
+  end
+
+  def swift
+    @swift ||= PushmiPullyu::SwiftDepositer.new(username: options[:swift][:username],
+                                                password: options[:swift][:password],
+                                                tenant: options[:swift][:tenant],
+                                                endpoint: options[:swift][:endpoint],
+                                                auth_version: options[:swift][:auth_version])
+  end
+
+  # On first call of shutdown, this will gracefully close the main run loop
+  # which let's the program exit itself. Calling shutdown again will force shutdown the program
+  def shutdown
+    if !PushmiPullyu.server_running?
+      exit!(1)
+    else
+      # using stderr instead of logger as it uses an underlying mutex which is not allowed inside trap contexts.
+      $stderr.puts 'Exiting...  Interrupt again to force quit.'
+      PushmiPullyu.server_running = false
+    end
+  end
+
+  def start_server_as_daemon
+    require 'daemons'
+
+    pwd = Dir.pwd # Current directory is changed during daemonization, so store it
+
+    opts = {
+      ARGV: @parsed_opts,
+      dir: options[:piddir],
+      dir_mode: :normal,
+      monitor: options[:monitor],
+      log_output: true,
+      log_dir: File.expand_path(options[:logdir]),
+      logfilename: File.basename(PushmiPullyu.application_log_file),
+      output_logfilename: File.basename(PushmiPullyu.application_log_file)
+    }
+
+    Daemons.run_proc(options[:process_name], opts) do |*_argv|
+      Dir.chdir(pwd)
+      start_server
+    end
+  end
+
+end