pushmi_pullyu 1.0.5 → 2.0.2

data/lib/pushmi_pullyu/cli.rb

@@ -4,6 +4,7 @@ require 'optparse'
 require 'rollbar'
 require 'singleton'
 require 'yaml'
+require 'json'
 
 # CLI runner
 class PushmiPullyu::CLI
@@ -59,13 +60,14 @@ class PushmiPullyu::CLI
     Rollbar.configure do |config|
       config.enabled = false unless options[:rollbar][:token].present?
       config.access_token = options[:rollbar][:token]
-
+      config.use_exception_level_filters_default = true
+      config.exception_level_filters['IOError'] = 'ignore'
       # add a filter after Rollbar has built the error payload but before it is delivered to the API,
       # in order to strip sensitive information out of certain error messages
       exception_message_transformer = proc do |payload|
-        clean_message = payload[:exception].message.sub(/http:\/\/.+:.+@(.+)\/fedora\/rest\/prod\/(.*)/,
-                                                        "http://\1/fedora/rest/prod/\2")
-        payload[:exception] = payload[:exception].exception(clean_message)
+        clean_message = payload[:exception][:message].sub(/http:\/\/.+:.+@(.+)\/aip\/v1\/(.*)/,
+                                                          "http://\1/aip/v1/\2")
+        payload[:exception][:message] = clean_message
         payload[:message] = clean_message
       end
 
@@ -87,7 +89,7 @@ class PushmiPullyu::CLI
 
   def parse_config(config_file)
     if File.exist?(config_file)
-      YAML.safe_load(ERB.new(IO.read(config_file)).result).deep_symbolize_keys || {}
+      YAML.safe_load(ERB.new(File.read(config_file)).result).deep_symbolize_keys || {}
     else
       {}
     end
@@ -180,26 +182,29 @@ class PushmiPullyu::CLI
   end
 
   def run_preservation_cycle
-    item = queue.wait_next_item
-    return unless item.present?
+    entity_json = JSON.parse(queue.wait_next_item)
+    entity = {
+      type: entity_json['type'],
+      uuid: entity_json['uuid']
+    }
+    return unless entity[:type].present? && entity[:uuid].present?
 
     # add additional information about the error context to errors that occur while processing this item.
-    Rollbar.scoped(noid: item) do
-      begin
-        # Download AIP from Fedora, bag and tar AIP directory and cleanup after block code
-        PushmiPullyu::AIP.create(item) do |aip_filename, aip_directory|
-          # Push tarred AIP to swift API
-          deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
-          # Log successful preservation event to the log files
-          PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
-        end
-      # rubocop:disable Lint/RescueException
-      rescue Exception => e
-        Rollbar.error(e)
-        logger.error(e)
-        # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
-        # move on to the next item.
+    Rollbar.scoped(entity_uuid: entity[:uuid]) do
+      # Download AIP from Jupiter, bag and tar AIP directory and cleanup after
+      # block code
+      PushmiPullyu::AIP.create(entity) do |aip_filename, aip_directory|
+        # Push tarred AIP to swift API
+        deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
+        # Log successful preservation event to the log files
+        PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
       end
+      # rubocop:disable Lint/RescueException
+    rescue Exception => e
+      Rollbar.error(e)
+      logger.error(e)
+      # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
+      # move on to the next item.
       # rubocop:enable Lint/RescueException
     end
   end
@@ -244,12 +249,12 @@ class PushmiPullyu::CLI
   # On first call of shutdown, this will gracefully close the main run loop
   # which let's the program exit itself. Calling shutdown again will force shutdown the program
   def shutdown
-    if !PushmiPullyu.server_running?
-      exit!(1)
-    else
+    if PushmiPullyu.server_running?
       # using stderr instead of logger as it uses an underlying mutex which is not allowed inside trap contexts.
       warn 'Exiting...  Interrupt again to force quit.'
       PushmiPullyu.server_running = false
+    else
+      exit!(1)
     end
   end
 

data/lib/pushmi_pullyu/logging.rb

@@ -21,7 +21,7 @@ module PushmiPullyu::Logging
 
     attr_writer :logger
 
-    def initialize_logger(log_target = STDOUT)
+    def initialize_logger(log_target = $stdout)
       @logger = Logger.new(log_target)
       @logger.level = Logger::INFO
       @logger
@@ -47,11 +47,11 @@ module PushmiPullyu::Logging
       preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
 
       message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
-      "Here are the details of this preservation event:\n"\
-      "\tNOID: '#{deposited_file.name}'\n"\
-      "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
-      "\tAIP Checksum: '#{deposited_file.etag}'\n"\
-      "\tMetadata: #{deposited_file.metadata}\n"\
+                "Here are the details of this preservation event:\n"\
+                "\tUUID: '#{deposited_file.name}'\n"\
+                "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
+                "\tAIP Checksum: '#{deposited_file.etag}'\n"\
+                "\tMetadata: #{deposited_file.metadata}\n"\
 
       file_details = file_log_details(aip_directory)
 

data/lib/pushmi_pullyu/swift_depositer.rb

@@ -50,7 +50,7 @@ class PushmiPullyu::SwiftDepositer
     else
       # for creating new: construct hash with symbols as keys, add metadata as a hash within the header hash
       headers = { etag: checksum,
-                  content_type:  'application/x-tar',
+                  content_type: 'application/x-tar',
                   metadata: metadata }
       deposited_file = era_container.create_object(file_base_name, headers, File.open(file_name))
     end

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '1.0.5'.freeze
+  VERSION = '2.0.2'.freeze
 end

data/lib/pushmi_pullyu.rb

@@ -1,22 +1,15 @@
-# require 'pushmi_pullyu/version' must be first as it declares the PushmiPullyu module.
-# (This fixes a weird NameError bug when using the nested compact syntax for
-# defining modules/classes like `module PushmiPullyu::Logging`)
-require 'pushmi_pullyu/version'
+# require 'pushmi_pullyu/version' must be first as it declares the PushmiPullyu
+# module. (This fixes a weird NameError bug when using the nested compact syntax
+# for defining modules/classes like `module PushmiPullyu::Logging`)
 
+require 'pushmi_pullyu/version'
 require 'pushmi_pullyu/logging'
-
 require 'pushmi_pullyu/aip'
 require 'pushmi_pullyu/aip/creator'
 require 'pushmi_pullyu/aip/downloader'
-require 'pushmi_pullyu/aip/fedora_fetcher'
-require 'pushmi_pullyu/aip/file_list_creator'
-require 'pushmi_pullyu/aip/owner_email_editor'
-require 'active_record'
-require 'pushmi_pullyu/aip/user'
 require 'pushmi_pullyu/cli'
 require 'pushmi_pullyu/preservation_queue'
 require 'pushmi_pullyu/swift_depositer'
-
 require 'active_support'
 require 'active_support/core_ext'
 
@@ -36,12 +29,6 @@ module PushmiPullyu
     redis: {
       url: 'redis://localhost:6379'
     },
-    fedora: {
-      url: 'http://localhost:8080/fcrepo/rest',
-      user: 'fedoraAdmin',
-      password: 'fedoraAdmin',
-      base_path: '/dev'
-    },
     swift: {
       tenant: 'tester',
       username: 'test:tester',
@@ -58,6 +45,12 @@ module PushmiPullyu
       pool: ENV['RAILS_MAX_THREADS'] || 5,
       url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
       database: 'jupiter_development'
+    },
+    jupiter: {
+      user: ENV['JUPITER_USER'],
+      api_key: ENV['JUPITER_API_KEY'],
+      jupiter_url: ENV['JUPITER_URL'] || 'http://localhost:3000/',
+      aip_api_path: ENV['JUPITER_AIP_API_PATH'] || 'aip/v1'
     }
   }.freeze
 

data/pushmi_pullyu.gemspec

@@ -5,10 +5,10 @@ require 'pushmi_pullyu/version'
 Gem::Specification.new do |spec|
   spec.name          = 'pushmi_pullyu'
   spec.version       = PushmiPullyu::VERSION
-  spec.authors       = ['Shane Murnaghan']
-  spec.email         = ['murnagha@ualberta.ca']
+  spec.authors       = ['Shane Murnaghan', 'Omar Rodriguez-Arenas']
+  spec.email         = ['murnagha@ualberta.ca', 'orodrigu@ualberta.ca']
 
-  spec.summary       = 'Ruby application to manage flow of content from Fedora into Swift for preservation'
+  spec.summary       = 'Ruby application to manage flow of content from Jupiter into Swift for preservation'
   spec.homepage      = 'https://github.com/ualbertalib/pushmi_pullyu'
   spec.license       = 'MIT'
 
@@ -19,31 +19,35 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.3.1'
+  spec.required_ruby_version = '>= 2.5'
 
-  spec.add_runtime_dependency 'activerecord', '>= 5.1.5', '< 5.3.0'
-  spec.add_runtime_dependency 'activesupport', '~> 5.0'
+  spec.add_runtime_dependency 'activesupport', '>= 5', '< 7'
   spec.add_runtime_dependency 'bagit', '~> 0.4'
   spec.add_runtime_dependency 'connection_pool', '~> 2.2'
   spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
-  spec.add_runtime_dependency 'minitar', '~> 0.6'
+  spec.add_runtime_dependency 'minitar', '~> 0.7'
   spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
   spec.add_runtime_dependency 'pg', '>= 1.0', '< 1.2'
   spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
   spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
   spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
   spec.add_runtime_dependency 'rest-client', '>= 1.8', '< 3.0'
-  spec.add_runtime_dependency 'rollbar', '~> 2.14'
+  spec.add_runtime_dependency 'rollbar', '>= 2.18', '< 4.0'
+  spec.add_runtime_dependency 'uuid', '~> 2.3.9'
 
-  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'coveralls', '~> 0.8'
-  spec.add_development_dependency 'danger', '~> 5.2'
+  spec.add_development_dependency 'danger', '~> 8.0'
   spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
-  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'pry-byebug', '~> 3.6'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.51'
-  spec.add_development_dependency 'rubocop-rspec', '~> 1.10'
+  spec.add_development_dependency 'rubocop', '~> 1.23'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.6'
   spec.add_development_dependency 'timecop', '~> 0.8'
-  spec.add_development_dependency 'vcr', '~> 4.0'
+  spec.add_development_dependency 'vcr', '~> 5.0'
   spec.add_development_dependency 'webmock', '~> 3.3'
+  spec.metadata = {
+    'rubygems_mfa_required' => 'true'
+  }
 end

metadata

@@ -1,49 +1,36 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 2.0.2
 platform: ruby
 authors:
 - Shane Murnaghan
+- Omar Rodriguez-Arenas
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-18 00:00:00.000000000 Z
+date: 2021-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.5
+        version: '5'
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.3.0
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.5
+        version: '5'
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.3.0
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: bagit
   requirement: !ruby/object:Gem::Requirement
@@ -98,14 +85,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: openstack
   requirement: !ruby/object:Gem::Requirement
@@ -228,32 +215,52 @@ dependencies:
         version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rollbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.18'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.18'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: uuid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: 2.3.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: 2.3.9
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -274,14 +281,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '8.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '8.0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -302,20 +309,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.4
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -336,28 +357,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.51'
+        version: '1.23'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.51'
+        version: '1.23'
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -378,14 +399,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -403,6 +424,7 @@ dependencies:
 description: 
 email:
 - murnagha@ualberta.ca
+- orodrigu@ualberta.ca
 executables:
 - pushmi_pullyu
 extensions: []
@@ -410,11 +432,15 @@ extra_rdoc_files: []
 files:
 - ".coveralls.yml"
 - ".editorconfig"
+- ".github/CODEOWNERS"
+- ".github/PULL_REQUEST_TEMPLATE"
+- ".github/dependabot.yml"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".hound.yml"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
+- CHANGELOG.md
 - Dangerfile
 - Gemfile
 - LICENSE.txt
@@ -431,10 +457,6 @@ files:
 - lib/pushmi_pullyu/aip.rb
 - lib/pushmi_pullyu/aip/creator.rb
 - lib/pushmi_pullyu/aip/downloader.rb
-- lib/pushmi_pullyu/aip/fedora_fetcher.rb
-- lib/pushmi_pullyu/aip/file_list_creator.rb
-- lib/pushmi_pullyu/aip/owner_email_editor.rb
-- lib/pushmi_pullyu/aip/user.rb
 - lib/pushmi_pullyu/cli.rb
 - lib/pushmi_pullyu/logging.rb
 - lib/pushmi_pullyu/preservation_queue.rb
@@ -447,7 +469,8 @@ files:
 homepage: https://github.com/ualbertalib/pushmi_pullyu
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -456,16 +479,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.1
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
-summary: Ruby application to manage flow of content from Fedora into Swift for preservation
+summary: Ruby application to manage flow of content from Jupiter into Swift for preservation
 test_files: []

data/.travis.yml

@@ -1,16 +0,0 @@
-language: ruby
-sudo: false
-cache: bundler
-services:
-  - redis-server
-
-before_install:
-  - gem install bundler
-  - gem update bundler
-
-rvm:
-  - 2.3.3
-  - 2.4.0
-
-before_script:
-  - bundle exec danger

data/lib/pushmi_pullyu/aip/fedora_fetcher.rb

@@ -1,66 +0,0 @@
-require 'net/http'
-
-class PushmiPullyu::AIP::FedoraFetcher
-
-  class FedoraFetchError < StandardError; end
-
-  RDF_FORMAT = 'text/rdf+n3'.freeze
-
-  def initialize(noid)
-    @noid = noid
-  end
-
-  def object_url(url_extra = nil)
-    url = "#{PushmiPullyu.options[:fedora][:url]}#{base_path}/#{pairtree}"
-    url += url_extra if url_extra
-    url
-  end
-
-  # Return true on success, raise an error otherwise
-  # (or use 'optional' to return false on 404)
-  def download_object(download_path, url_extra: nil,
-                      optional: false, is_rdf: false,
-                      should_add_user_email: false)
-
-    uri = URI(object_url(url_extra))
-
-    request = Net::HTTP::Get.new(uri)
-    request.basic_auth(PushmiPullyu.options[:fedora][:user],
-                       PushmiPullyu.options[:fedora][:password])
-
-    request['Accept'] = RDF_FORMAT if is_rdf
-
-    response = Net::HTTP.start(uri.hostname, uri.port) do |http|
-      http.request(request)
-    end
-
-    if response.is_a?(Net::HTTPSuccess)
-      body = if should_add_user_email
-               PushmiPullyu::AIP::OwnerEmailEditor.new(response.body).run
-             else
-               response.body
-             end
-      file = File.open(download_path, 'wb')
-      file.write(body)
-      file.close
-      return true
-    elsif response.is_a?(Net::HTTPNotFound)
-      raise FedoraFetchError unless optional
-
-      return false
-    else
-      raise FedoraFetchError
-    end
-  end
-
-  private
-
-  def pairtree
-    "#{@noid[0..1]}/#{@noid[2..3]}/#{@noid[4..5]}/#{@noid[6..7]}/#{@noid}"
-  end
-
-  def base_path
-    PushmiPullyu.options[:fedora][:base_path]
-  end
-
-end