pushmi_pullyu 0.2.0

data/lib/pushmi_pullyu/logging.rb

@@ -0,0 +1,70 @@
+require 'logger'
+
+# PushmiPullyu::Logging is a standard Ruby logger wrapper
+module PushmiPullyu::Logging
+  # Simple formatter which only displays the message.
+  # Taken from ActiveSupport
+  class SimpleFormatter < Logger::Formatter
+
+    # This method is invoked when a log event occurs
+    def call(_severity, _timestamp, _program_name, msg)
+      "#{msg.is_a?(String) ? msg : msg.inspect}\n"
+    end
+
+  end
+
+  def self.initialize_logger(log_target = STDOUT)
+    @logger = Logger.new(log_target)
+    @logger.level = Logger::INFO
+    @logger
+  end
+
+  def self.logger
+    @logger ||= initialize_logger
+  end
+
+  def self.log_aip_activity(aip_directory, message)
+    log_file = "#{aip_directory}/data/logs/aipcreation.log"
+    aip_logger = Logger.new(log_file)
+    aip_logger.level = logger.level
+
+    # Log to both the application log, and the log file that gets archived in the AIP
+    logger.info(message)
+    aip_logger.info(message)
+
+    aip_logger.close
+  end
+
+  def self.log_preservation_event(deposited_file)
+    preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
+
+    message = "#{deposited_file.name} was successfully deposited into Swift Storage! \n"\
+    "Here are the details of this preservation event: \n"\
+    "\t NOID: '#{deposited_file.name}' \n"\
+    "\t Timestamp of Completion: '#{deposited_file.last_modified}' \n"\
+    "\t AIP Checksum: '#{deposited_file.etag}' \n"\
+    "\t Metadata: #{deposited_file.metadata} \n"
+
+    # Log to both the application log, and the preservation log file
+    logger.info(message)
+    preservation_logger.info(message)
+
+    preservation_logger.close
+  end
+
+  def self.logger=(log)
+    @logger = log
+  end
+
+  def self.reopen
+    if @logger
+      @logger.reopen
+    else
+      @logger = initialize_logger
+    end
+  end
+
+  def logger
+    PushmiPullyu::Logging.logger
+  end
+end

data/lib/pushmi_pullyu/preservation_queue.rb

@@ -0,0 +1,76 @@
+require 'redis'
+require 'connection_pool'
+
+# Quick and dirty take on sorted sets
+
+# 1) Create a sorted set in Redis (https://redis.io/topics/data-types). Call it preservation_queue
+#
+# 2) In GenericFile add an after_save that:
+#    - determines a monotonically increasing "score". Obvious scores would be either the time in seconds/milliseconds
+#      or using something like redis INCR to create an atomic, increasing counter. It doesn't matter if 2 different
+#      noids ever have the same score, it only that scores generally increase over time.
+#    - zadd preservation_queue score "noid" adds the noid and gives it the score from above.
+#
+# 3) Pushmi-pullyu pops elements out of the sorted set, lowest score to highest.
+#
+# A sorted set will only ever contain a noid once, with whatever score it was last given. Because preservation_queue
+# is sorted lowest score to highest, and because scores increase over time, a cascade of jobs/updates will cause a noid
+# to keep "moving back" in the queue until it becomes the least recently updated noid in the queue, at which point it
+# will be popped and preserved. Any further updates will trigger a new AIP build.
+class PushmiPullyu::PreservationQueue
+
+  class ConnectionError < StandardError; end
+
+  def initialize(connection: {}, pool_opts: { size: 1, timeout: 5 }, poll_interval: 10, age_at_least: 0,
+                 queue_name: 'dev:pmpy_queue')
+    # we use a connection pool even though we're not (currently) threading
+    # as it transparently provides for repairing connections if they are closed after long periods of inactivity
+    @redis = ConnectionPool.new(pool_opts) do
+      connection.reverse_merge!(host: 'localhost', port: 6379)
+      Redis.new(connection)
+    end
+
+    raise ConnectionError unless connected?
+
+    @poll_interval = poll_interval
+    @age_at_least = age_at_least
+    @queue_name = queue_name
+  end
+
+  def next_item
+    raise ConnectionError unless connected?
+
+    @redis.with do |conn|
+      conn.watch(@queue_name) do |rd| # transactional mutation of the set is dependent on the set key
+        element, score = rd.zrange(@queue_name, 0, 0, with_scores: true).first
+
+        if element && ((Time.now.to_f - @age_at_least) >= score)
+          rd.multi do |tx|
+            tx.zrem(@queue_name, element) # remove the top element transactionally
+          end
+          return element
+        else
+          rd.unwatch # cancel the transaction since there was nothing in the queue
+          return nil
+        end
+      end
+    end
+  end
+
+  def wait_next_item
+    while PushmiPullyu.continue_polling?
+      element = next_item
+      return element if element.present?
+      sleep @poll_interval
+    end
+  end
+
+  protected
+
+  def connected?
+    @redis.with do |conn|
+      conn.ping == 'PONG'
+    end
+  end
+
+end

data/lib/pushmi_pullyu/swift_depositer.rb

@@ -0,0 +1,51 @@
+require 'digest/md5'
+require 'openstack'
+
+class PushmiPullyu::SwiftDepositer
+
+  attr_reader :swift_connection
+
+  def initialize(connection)
+    @swift_connection = OpenStack::Connection.create(
+      username: connection[:username],
+      api_key: connection[:password],
+      auth_method: 'password',
+      auth_url: "#{connection[:endpoint]}/auth/#{connection[:auth_version]}",
+      authtenant_name: connection[:tenant],
+      service_type: 'object-store'
+    )
+  end
+
+  def deposit_file(file_name, swift_container)
+    file_base_name = File.basename(file_name, '.*')
+
+    checksum = Digest::MD5.file(file_name).hexdigest
+
+    era_container = swift_connection.container(swift_container)
+
+    deposited_file = if era_container.object_exists?(file_base_name)
+                       era_container.object(file_base_name)
+                     else
+                       era_container.create_object(file_base_name)
+                     end
+
+    # Add swift metadata with in accordance to AIP spec:
+    # https://docs.google.com/document/d/154BqhDPAdGW-I9enrqLpBYbhkF9exX9lV3kMaijuwPg/edit#
+    metadata = {
+      project: 'ERA',
+      project_id: file_base_name,
+      promise: 'bronze',
+      aip_version: '1.0'
+    }
+
+    # ruby-openstack wants all keys of the metadata to be named like "X-Object-Meta-{{Key}}", so update them
+    metadata.transform_keys! { |key| "X-Object-Meta-#{key}" }
+
+    deposited_file.write(File.open(file_name),
+                         { 'etag' => checksum,
+                           'content-type' => 'application/x-tar' }.merge(metadata))
+
+    deposited_file
+  end
+
+end

data/lib/pushmi_pullyu/version.rb

@@ -0,0 +1,3 @@
+module PushmiPullyu
+  VERSION = '0.2.0'.freeze
+end

data/pushmi_pullyu.gemspec

@@ -0,0 +1,48 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pushmi_pullyu/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'pushmi_pullyu'
+  spec.version       = PushmiPullyu::VERSION
+  spec.authors       = ['Shane Murnaghan']
+  spec.email         = ['murnagha@ualberta.ca']
+
+  spec.summary       = 'Ruby application to manage flow of content from Fedora into Swift for preservation'
+  spec.homepage      = 'https://github.com/ualbertalib/pushmi_pullyu'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.3.1'
+
+  spec.add_runtime_dependency 'activesupport', '~> 5.0'
+  spec.add_runtime_dependency 'bagit', '~> 0.4'
+  spec.add_runtime_dependency 'connection_pool', '~> 2.2'
+  spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
+  spec.add_runtime_dependency 'minitar', '~> 0.6'
+  spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
+  spec.add_runtime_dependency 'rdf', '~> 1.99'
+  spec.add_runtime_dependency 'rdf-n3', '~> 1.99'
+  spec.add_runtime_dependency 'redis', '~> 3.3'
+  spec.add_runtime_dependency 'rollbar', '~> 2.14'
+
+  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'coveralls', '~> 0.8'
+  spec.add_development_dependency 'danger', '~> 5.2'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.45'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1.10'
+  spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
+  spec.add_development_dependency 'timecop', '~> 0.8'
+  spec.add_development_dependency 'vcr', '~> 3.0'
+  spec.add_development_dependency 'webmock', '~> 2.1'
+end

metadata

@@ -0,0 +1,391 @@
+--- !ruby/object:Gem::Specification
+name: pushmi_pullyu
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Shane Murnaghan
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-06-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: bagit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+- !ruby/object:Gem::Dependency
+  name: minitar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: openstack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.10
+- !ruby/object:Gem::Dependency
+  name: rdf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.99'
+- !ruby/object:Gem::Dependency
+  name: rdf-n3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.99'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: rollbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: danger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.45'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.45'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.4
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: 
+email:
+- murnagha@ualberta.ca
+executables:
+- pushmi_pullyu
+extensions: []
+extra_rdoc_files: []
+files:
+- ".coveralls.yml"
+- ".editorconfig"
+- ".gitignore"
+- ".hound.yml"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Dangerfile
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- config/.gitkeep
+- docs/images/pushmi-pullyu.png
+- docs/images/system-infrastructure-diagram.png
+- examples/pushmi_pullyu.yml
+- exe/pushmi_pullyu
+- lib/pushmi_pullyu.rb
+- lib/pushmi_pullyu/aip.rb
+- lib/pushmi_pullyu/aip/creator.rb
+- lib/pushmi_pullyu/aip/downloader.rb
+- lib/pushmi_pullyu/aip/fedora_fetcher.rb
+- lib/pushmi_pullyu/aip/solr_fetcher.rb
+- lib/pushmi_pullyu/cli.rb
+- lib/pushmi_pullyu/logging.rb
+- lib/pushmi_pullyu/preservation_queue.rb
+- lib/pushmi_pullyu/swift_depositer.rb
+- lib/pushmi_pullyu/version.rb
+- log/.gitkeep
+- pushmi_pullyu.gemspec
+- tmp/pids/.gitkeep
+- tmp/work/.gitkeep
+homepage: https://github.com/ualbertalib/pushmi_pullyu
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Ruby application to manage flow of content from Fedora into Swift for preservation
+test_files: []