pushmi_pullyu 1.0.3 → 2.0.1

data/lib/pushmi_pullyu/cli.rb

@@ -4,6 +4,7 @@ require 'optparse'
 require 'rollbar'
 require 'singleton'
 require 'yaml'
+require 'json'
 
 # CLI runner
 class PushmiPullyu::CLI
@@ -48,7 +49,6 @@ class PushmiPullyu::CLI
     setup_signal_traps
 
     setup_log
-    setup_db_connection
     print_banner
 
     run_tick_loop
@@ -60,6 +60,19 @@ class PushmiPullyu::CLI
     Rollbar.configure do |config|
       config.enabled = false unless options[:rollbar][:token].present?
       config.access_token = options[:rollbar][:token]
+      config.use_exception_level_filters_default = true
+      config.exception_level_filters['IOError'] = 'ignore'
+      # add a filter after Rollbar has built the error payload but before it is delivered to the API,
+      # in order to strip sensitive information out of certain error messages
+      exception_message_transformer = proc do |payload|
+        clean_message = payload[:exception][:message].sub(/http:\/\/.+:.+@(.+)\/aip\/v1\/(.*)/,
+                                                          "http://\1/aip/v1/\2")
+        payload[:exception][:message] = clean_message
+        payload[:message] = clean_message
+      end
+
+      config.transform << exception_message_transformer
+
       if options[:rollbar][:proxy_host].present?
         config.proxy = {}
         config.proxy[:host] = options[:rollbar][:proxy_host]
@@ -169,26 +182,29 @@ class PushmiPullyu::CLI
   end
 
   def run_preservation_cycle
-    item = queue.wait_next_item
-    return unless item.present?
+    entity_json = JSON.parse(queue.wait_next_item)
+    entity = {
+      type: entity_json['type'],
+      uuid: entity_json['uuid']
+    }
+    return unless entity[:type].present? && entity[:uuid].present?
 
     # add additional information about the error context to errors that occur while processing this item.
-    Rollbar.scoped(noid: item) do
-      begin
-        # Download AIP from Fedora, bag and tar AIP directory and cleanup after block code
-        PushmiPullyu::AIP.create(item) do |aip_filename, aip_directory|
-          # Push tarred AIP to swift API
-          deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
-          # Log successful preservation event to the log files
-          PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
-        end
-      # rubocop:disable Lint/RescueException
-      rescue Exception => e
-        Rollbar.error(e)
-        logger.error(e)
-        # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
-        # move on to the next item.
+    Rollbar.scoped(entity_uuid: entity[:uuid]) do
+      # Download AIP from Jupiter, bag and tar AIP directory and cleanup after
+      # block code
+      PushmiPullyu::AIP.create(entity) do |aip_filename, aip_directory|
+        # Push tarred AIP to swift API
+        deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
+        # Log successful preservation event to the log files
+        PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
       end
+      # rubocop:disable Lint/RescueException
+    rescue Exception => e
+      Rollbar.error(e)
+      logger.error(e)
+      # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
+      # move on to the next item.
       # rubocop:enable Lint/RescueException
     end
   end
@@ -264,12 +280,4 @@ class PushmiPullyu::CLI
     end
   end
 
-  def setup_db_connection
-    ActiveRecord::Base.establish_connection(database_configuration)
-  end
-
-  def database_configuration
-    PushmiPullyu.options[:database][:url]
-  end
-
 end

data/lib/pushmi_pullyu/logging.rb

@@ -48,7 +48,7 @@ module PushmiPullyu::Logging
 
       message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
       "Here are the details of this preservation event:\n"\
-      "\tNOID: '#{deposited_file.name}'\n"\
+      "\tUUID: '#{deposited_file.name}'\n"\
       "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
       "\tAIP Checksum: '#{deposited_file.etag}'\n"\
       "\tMetadata: #{deposited_file.metadata}\n"\

data/lib/pushmi_pullyu/preservation_queue.rb

@@ -63,6 +63,7 @@ class PushmiPullyu::PreservationQueue
     while PushmiPullyu.continue_polling?
       element = next_item
       return element if element.present?
+
       sleep @poll_interval
     end
   end

data/lib/pushmi_pullyu/swift_depositer.rb

@@ -50,7 +50,7 @@ class PushmiPullyu::SwiftDepositer
     else
       # for creating new: construct hash with symbols as keys, add metadata as a hash within the header hash
       headers = { etag: checksum,
-                  content_type:  'application/x-tar',
+                  content_type: 'application/x-tar',
                   metadata: metadata }
       deposited_file = era_container.create_object(file_base_name, headers, File.open(file_name))
     end

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '1.0.3'.freeze
+  VERSION = '2.0.1'.freeze
 end

data/pushmi_pullyu.gemspec

@@ -5,10 +5,10 @@ require 'pushmi_pullyu/version'
 Gem::Specification.new do |spec|
   spec.name          = 'pushmi_pullyu'
   spec.version       = PushmiPullyu::VERSION
-  spec.authors       = ['Shane Murnaghan']
-  spec.email         = ['murnagha@ualberta.ca']
+  spec.authors       = ['Shane Murnaghan', 'Omar Rodriguez-Arenas']
+  spec.email         = ['murnagha@ualberta.ca', 'orodrigu@ualberta.ca']
 
-  spec.summary       = 'Ruby application to manage flow of content from Fedora into Swift for preservation'
+  spec.summary       = 'Ruby application to manage flow of content from Jupiter into Swift for preservation'
   spec.homepage      = 'https://github.com/ualbertalib/pushmi_pullyu'
   spec.license       = 'MIT'
 
@@ -19,31 +19,32 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.3.1'
+  spec.required_ruby_version = '>= 2.5'
 
-  spec.add_runtime_dependency 'activerecord', '>= 5.1.5', '< 5.3.0'
-  spec.add_runtime_dependency 'activesupport', '~> 5.0'
+  spec.add_runtime_dependency 'activesupport', '>= 5', '< 7'
   spec.add_runtime_dependency 'bagit', '~> 0.4'
   spec.add_runtime_dependency 'connection_pool', '~> 2.2'
   spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
-  spec.add_runtime_dependency 'minitar', '~> 0.6'
+  spec.add_runtime_dependency 'minitar', '~> 0.7'
   spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
-  spec.add_runtime_dependency 'pg', '~> 1.0.0'
+  spec.add_runtime_dependency 'pg', '>= 1.0', '< 1.2'
   spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
   spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
   spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
   spec.add_runtime_dependency 'rest-client', '>= 1.8', '< 3.0'
-  spec.add_runtime_dependency 'rollbar', '~> 2.14'
+  spec.add_runtime_dependency 'rollbar', '>= 2.18', '< 4.0'
+  spec.add_runtime_dependency 'uuid', '~> 2.3.9'
 
-  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'coveralls', '~> 0.8'
-  spec.add_development_dependency 'danger', '~> 5.2'
+  spec.add_development_dependency 'danger', '~> 8.0'
   spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
-  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'pry-byebug', '~> 3.6'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '~> 0.51'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.10'
   spec.add_development_dependency 'timecop', '~> 0.8'
-  spec.add_development_dependency 'vcr', '~> 4.0'
+  spec.add_development_dependency 'vcr', '~> 5.0'
   spec.add_development_dependency 'webmock', '~> 3.3'
 end

metadata

@@ -1,49 +1,36 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 2.0.1
 platform: ruby
 authors:
 - Shane Murnaghan
+- Omar Rodriguez-Arenas
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-23 00:00:00.000000000 Z
+date: 2021-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.5
+        version: '5'
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.3.0
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.5
+        version: '5'
     - - "<"
       - !ruby/object:Gem::Version
-        version: 5.3.0
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: bagit
   requirement: !ruby/object:Gem::Requirement
@@ -98,14 +85,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: openstack
   requirement: !ruby/object:Gem::Requirement
@@ -130,16 +117,22 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rdf
   requirement: !ruby/object:Gem::Requirement
@@ -222,32 +215,52 @@ dependencies:
         version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rollbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.18'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.18'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: uuid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: 2.3.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.14'
+        version: 2.3.9
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -268,14 +281,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '8.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '8.0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -296,20 +309,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.4
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -372,14 +399,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -397,6 +424,7 @@ dependencies:
 description: 
 email:
 - murnagha@ualberta.ca
+- orodrigu@ualberta.ca
 executables:
 - pushmi_pullyu
 extensions: []
@@ -404,11 +432,15 @@ extra_rdoc_files: []
 files:
 - ".coveralls.yml"
 - ".editorconfig"
+- ".github/CODEOWNERS"
+- ".github/PULL_REQUEST_TEMPLATE"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".hound.yml"
 - ".rspec"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - Dangerfile
 - Gemfile
 - LICENSE.txt
@@ -425,10 +457,6 @@ files:
 - lib/pushmi_pullyu/aip.rb
 - lib/pushmi_pullyu/aip/creator.rb
 - lib/pushmi_pullyu/aip/downloader.rb
-- lib/pushmi_pullyu/aip/fedora_fetcher.rb
-- lib/pushmi_pullyu/aip/file_list_creator.rb
-- lib/pushmi_pullyu/aip/owner_email_editor.rb
-- lib/pushmi_pullyu/aip/user.rb
 - lib/pushmi_pullyu/cli.rb
 - lib/pushmi_pullyu/logging.rb
 - lib/pushmi_pullyu/preservation_queue.rb
@@ -450,16 +478,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.1
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
-summary: Ruby application to manage flow of content from Fedora into Swift for preservation
+summary: Ruby application to manage flow of content from Jupiter into Swift for preservation
 test_files: []