pushmi_pullyu 2.0.0 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8596ed87a287d7708b92c6a3538dcff1f976ddd1b2dc0171b30011bcfe62276b
-  data.tar.gz: 9babcb9576246ee00e581056361731b324e27d02c49814af9b26346fbd269120
+  metadata.gz: d64e5d619eb9dee63c358167cd6778034eb72fa2686fbe95fa187cfdfe7525b3
+  data.tar.gz: 333805bd92b1d5e308949d583e46056ffb1aeac9900d9c23ca59f2b88a7a4d01
 SHA512:
-  metadata.gz: 0d031eef30ea6828c6fac6c094ab040e914fce8094298249f56ebf2b44d8820c22fd3de6ff98abc546e482f278351b6838419d9b09a90cdd9be6b50576896afc
-  data.tar.gz: 858a1d2f2a91a7ee9f9c0257391aa601316afa8fb07c09e883b817ec7b90879f5fdeda9eca34b375aeea5c1ed2c9b06b13c1212a03e30c538d2942a2e4976dca
+  metadata.gz: b8b162ca028ca37362d0bcd68737189f6af656c747bfc18a1593e35635bddf606d649e8d02299d092b1b569cab7e8a03950855fe2f7da4f5877a2714c99af8cc
+  data.tar.gz: 56c8653b1d868993e31607de7cd1d811752ff4de1ab63813dec2f745321f2b7d6a54107eb304f0f93191d6372bf22e55977512155edca08ee25ad96c23fc0099

data/.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # https://help.github.com/articles/about-codeowners/
-* @ConnorSheremeta @lagoan @mbarnett @murny @pgwillia @piyapongch
+* @ConnorSheremeta @lagoan @henryzhang87 @pgwillia @piyapongch

data/.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "10:00"
+  open-pull-requests-limit: 10
+  ignore:
+  - dependency-name: vcr
+    versions:
+    - "> 5.0"

data/.github/workflows/ruby.yml

@@ -25,7 +25,7 @@ jobs:
       run: bundle exec rubocop --parallel
     - name: Run Danger
       env:
-        DANGER_GITHUB_API_TOKEN: ${{ secrets.DANGER_GITHUB_API_TOKEN }}
+        DANGER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: bundle exec danger
     - name: Run tests
       run: bundle exec rake spec

data/.rubocop.yml

@@ -13,6 +13,7 @@ AllCops:
     - 'Dangerfile'
   ExtraDetails: true
   TargetRubyVersion: 2.5
+  NewCops: enable
 
 # readability is Actually Good
 Layout/EmptyLinesAroundClassBody:
@@ -35,7 +36,7 @@ Metrics/ClassLength:
 Metrics/CyclomaticComplexity:
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Enabled: true
   Max: 120 # default is 80
 
@@ -96,14 +97,14 @@ RSpec/ExampleLength:
 RSpec/MultipleExpectations:
   Enabled: false
 
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
 RSpec/DescribedClass:
   EnforcedStyle: explicit
 
-Lint/RaiseException:
-  Enabled: true
-
-Lint/StructNewOverride:
-  Enabled: true
+Naming/VariableNumber:
+  Enabled: false
 
-RSpec/MultipleMemoizedHelpers:
-  Enabled: false
+Style/OpenStructUse:
+  Enabled: false

data/CHANGELOG.md

@@ -8,9 +8,26 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
 
 ## [Unreleased]
 
+## [2.0.3] - 2022-04-28
+
+- Changed Danger token in Github Actions
+- remove pg dependency
+- unblock CI (rubocop and coveralls)
+- Log to preservation_events.json as well in an easy to use json format.
+
+## [2.0.2] - 2021-11-22
+
+- Fix authentication bug when using ssl 
+- bump rubocop and rubocop-rspec
+
+## [2.0.1] - 2021-02-02
+
+- Fix dependency declaration for UUID gem
+
 ## [2.0.0] - 2020-12-14
 
 ### Removed
 - Data output for original_file information
 
+- 
 ## [1.0.6] - 2018-11-29

data/README.md

@@ -6,7 +6,6 @@
 
 [![Gem Version](https://badge.fury.io/rb/pushmi_pullyu.svg)](https://rubygems.org/gems/pushmi_pullyu)
 [![Github Build Status](https://github.com/ualbertalib/pushmi_pullyu/workflows/CI/badge.svg)](https://github.com/ualbertalib/pushmi_pullyu/actions)
-[![Coverage Status](https://coveralls.io/repos/github/ualbertalib/pushmi_pullyu/badge.svg?branch=master)](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
 
 PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
 
@@ -104,6 +103,12 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Testing
 
+Pre-requisites:
+
+Will need jupiter running locally and Openstack Swift running on port 8080.
+- [Instructions for jupiter](https://github.com/ualbertalib/jupiter#docker)
+- Use `docker-compose up -d` to start Swift (additional instructions in docker-compose.yml)
+
 To run the test suite:
 
 ```bash

data/docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  swift:
+    # useful if you need to re-run the tests/spec
+    # https://docs.openstack.org/ocata/cli-reference/swift.html
+    # manual steps to set environment variables and create container
+    # swift auth -v -U test:tester -K testing -A http://localhost:8080/auth/v1.0
+    # swift auth
+    # swift post ERA
+    image: openstackswift/saio
+    ports:
+      - '8080:8080'

data/examples/pushmi_pullyu.yml

@@ -21,9 +21,6 @@ minimum_age: 0
 redis:
   url: redis://localhost:6379
 
-database:
-  url: postgresql://jupiter:mysecretpassword@127.0.0.1/jupiter_development
-
 #parameters project_name and project_domain_name are required only for keystone v3 authentication
 swift:
   tenant: tester

data/lib/pushmi_pullyu/aip/downloader.rb

@@ -80,7 +80,8 @@ class PushmiPullyu::AIP::Downloader
   def authenticate_http_calls
     @uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
     @http = Net::HTTP.new(@uri.host, @uri.port)
-    request = Net::HTTP::Post.new(@uri.request_uri + 'auth/system')
+    @http.use_ssl = true if @uri.instance_of? URI::HTTPS
+    request = Net::HTTP::Post.new("#{@uri.request_uri}auth/system")
     request.set_form_data(
       email: PushmiPullyu.options[:jupiter][:user],
       api_key: PushmiPullyu.options[:jupiter][:api_key]
@@ -102,9 +103,7 @@ class PushmiPullyu::AIP::Downloader
 
     response = @http.request(request)
     is_success = if response.is_a?(Net::HTTPSuccess)
-                   File.open(local, 'wb') do |file|
-                     file.write(response.body)
-                   end
+                   File.binwrite(local, response.body)
                    # Response was a success and the file was saved to local
                    File.exist? local
                  end

data/lib/pushmi_pullyu/cli.rb

@@ -89,7 +89,7 @@ class PushmiPullyu::CLI
 
   def parse_config(config_file)
     if File.exist?(config_file)
-      YAML.safe_load(ERB.new(IO.read(config_file)).result).deep_symbolize_keys || {}
+      YAML.safe_load(ERB.new(File.read(config_file)).result).deep_symbolize_keys || {}
     else
       {}
     end
@@ -249,12 +249,12 @@ class PushmiPullyu::CLI
   # On first call of shutdown, this will gracefully close the main run loop
   # which let's the program exit itself. Calling shutdown again will force shutdown the program
   def shutdown
-    if !PushmiPullyu.server_running?
-      exit!(1)
-    else
+    if PushmiPullyu.server_running?
       # using stderr instead of logger as it uses an underlying mutex which is not allowed inside trap contexts.
       warn 'Exiting...  Interrupt again to force quit.'
       PushmiPullyu.server_running = false
+    else
+      exit!(1)
     end
   end
 

data/lib/pushmi_pullyu/logging.rb

@@ -21,7 +21,7 @@ module PushmiPullyu::Logging
 
     attr_writer :logger
 
-    def initialize_logger(log_target = STDOUT)
+    def initialize_logger(log_target = $stdout)
       @logger = Logger.new(log_target)
       @logger.level = Logger::INFO
       @logger
@@ -45,13 +45,14 @@ module PushmiPullyu::Logging
 
     def log_preservation_event(deposited_file, aip_directory)
       preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
+      preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
 
       message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
-      "Here are the details of this preservation event:\n"\
-      "\tUUID: '#{deposited_file.name}'\n"\
-      "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
-      "\tAIP Checksum: '#{deposited_file.etag}'\n"\
-      "\tMetadata: #{deposited_file.metadata}\n"\
+                "Here are the details of this preservation event:\n"\
+                "\tUUID: '#{deposited_file.name}'\n"\
+                "\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
+                "\tAIP Checksum: '#{deposited_file.etag}'\n"\
+                "\tMetadata: #{deposited_file.metadata}\n"\
 
       file_details = file_log_details(aip_directory)
 
@@ -72,6 +73,65 @@ module PushmiPullyu::Logging
       preservation_logger.info(message)
 
       preservation_logger.close
+
+      message_json_str = preservation_event_to_json(deposited_file, aip_directory)
+      preservation_json_logger.info("#{message_json_str},")
+      preservation_json_logger.close
+    end
+
+    ###
+    # Provides an alternative logging method in json format for the convenience of
+    # parsing in the process of auditing against OpenStack Swift preservation.
+    #
+    # output format:
+    #  I, [2022-04-06T11:07:21.983875 #20791]  INFO -- : \
+    # {
+    #  "do_uuid": "83b5d21f-a60a-43ba-945a-f03deec64a1d",
+    #  "aip_deposited_at": "Thu, 07 Apr 2022 16:37:00 GMT",
+    #  "aip_md5sum": "fe5832a510799b04c1c503e46dc3b589",
+    #  "aip_sha256": "",
+    #  "aip_metadata": "{\"project-id\":\"83b5d21f-a60a-43ba-945a-f03deec64a1d\",
+    #                  \"aip-version\":\"1.0\",
+    #                  \"project\":\"ERA\",
+    #                  \"promise\":\"bronze\"}",
+    #  "aip_file_details": [
+    #   {
+    #     "fileset_uuid": "b2c6ac0f-f2ed-489e-bbae-bd26465207aa",
+    #     "file_name": "Spallacci_Amanda_202103_PhD.pdf",
+    #     "file_type": "pdf",
+    #     "file_size": "2051363"
+    #    }
+    #   ]
+    # }
+    #
+    # note:
+    #   to parse, the prefix "I, ... INFO --:" in each line needs to be
+    #   stripped using a bash command such as "sed"
+    def preservation_event_to_json(deposited_file, aip_directory)
+      message = {}
+
+      message['do_uuid'] = deposited_file.name.to_s
+      message['aip_deposited_at'] = deposited_file.last_modified.to_s
+      message['aip_md5sum'] = deposited_file.etag.to_s
+      message['aip_sha256'] = ''
+      message['aip_metadata'] = deposited_file.metadata.to_json.to_s
+
+      file_details = file_log_details(aip_directory)
+
+      tmp_details = []
+      if file_details.present?
+        file_details.each do |file_detail|
+          tmp_hash = {}
+          tmp_hash['fileset_uuid'] = file_detail[:fileset_name].to_s
+          tmp_hash['file_name'] = file_detail[:file_name].to_s
+          tmp_hash['file_type'] = file_detail[:file_extension].to_s
+          tmp_hash['file_size'] = file_detail[:file_size].to_s
+          tmp_details << tmp_hash
+        end
+      end
+
+      message['aip_file_details'] = tmp_details
+      message.to_json
     end
 
     def reopen

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '2.0.0'.freeze
+  VERSION = '2.0.3'.freeze
 end

data/lib/pushmi_pullyu.rb

@@ -40,18 +40,14 @@ module PushmiPullyu
     },
     rollbar: {
     },
-    database: {
-      encoding: 'utf8',
-      pool: ENV['RAILS_MAX_THREADS'] || 5,
-      url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
-      database: 'jupiter_development'
-    },
+    # rubocop disable: Style/FetchEnvVar
     jupiter: {
-      user: ENV['JUPITER_USER'],
-      api_key: ENV['JUPITER_API_KEY'],
-      jupiter_url: ENV['JUPITER_URL'] || 'http://localhost:3000/',
-      aip_api_path: ENV['JUPITER_AIP_API_PATH'] || 'aip/v1'
+      user: ENV.fetch('JUPITER_USER', nil),
+      api_key: ENV.fetch('JUPITER_API_KEY', nil),
+      jupiter_url: ENV.fetch('JUPITER_URL', nil) || 'http://era.lvh.me:3000/',
+      aip_api_path: ENV.fetch('JUPITER_AIP_API_PATH', nil) || 'aip/v1'
     }
+    # rubocop enable: Style/FetchEnvVar
   }.freeze
 
   def self.options

data/pushmi_pullyu.gemspec

@@ -21,30 +21,31 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.5'
 
-  spec.add_runtime_dependency 'activesupport', '>= 5', '< 7'
+  spec.add_runtime_dependency 'activesupport', '>= 5', '< 8'
   spec.add_runtime_dependency 'bagit', '~> 0.4'
   spec.add_runtime_dependency 'connection_pool', '~> 2.2'
   spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
   spec.add_runtime_dependency 'minitar', '~> 0.7'
   spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
-  spec.add_runtime_dependency 'pg', '>= 1.0', '< 1.2'
   spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
   spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
   spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
   spec.add_runtime_dependency 'rest-client', '>= 1.8', '< 3.0'
   spec.add_runtime_dependency 'rollbar', '>= 2.18', '< 4.0'
+  spec.add_runtime_dependency 'uuid', '~> 2.3.9'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'coveralls', '~> 0.8'
   spec.add_development_dependency 'danger', '~> 8.0'
   spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
   spec.add_development_dependency 'pry-byebug', '~> 3.6'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.51'
-  spec.add_development_dependency 'rubocop-rspec', '~> 1.10'
+  spec.add_development_dependency 'rubocop', '~> 1.23'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.6'
   spec.add_development_dependency 'timecop', '~> 0.8'
-  spec.add_development_dependency 'uuid', '~> 2.3.9'
   spec.add_development_dependency 'vcr', '~> 5.0'
   spec.add_development_dependency 'webmock', '~> 3.3'
+  spec.metadata = {
+    'rubygems_mfa_required' => 'true'
+  }
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.3
 platform: ruby
 authors:
 - Shane Murnaghan
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-29 00:00:00.000000000 Z
+date: 2022-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -20,7 +20,7 @@ dependencies:
         version: '5'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7'
+        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
         version: '5'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7'
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: bagit
   requirement: !ruby/object:Gem::Requirement
@@ -113,26 +113,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.3.10
-- !ruby/object:Gem::Dependency
-  name: pg
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rdf
   requirement: !ruby/object:Gem::Requirement
@@ -234,33 +214,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: uuid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
+        version: 2.3.9
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.3.9
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: danger
   requirement: !ruby/object:Gem::Requirement
@@ -343,28 +323,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.51'
+        version: '1.23'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.51'
+        version: '1.23'
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -379,20 +359,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
-- !ruby/object:Gem::Dependency
-  name: uuid
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.3.9
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.3.9
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -434,12 +400,12 @@ files:
 - ".editorconfig"
 - ".github/CODEOWNERS"
 - ".github/PULL_REQUEST_TEMPLATE"
+- ".github/dependabot.yml"
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".hound.yml"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - CHANGELOG.md
 - Dangerfile
 - Gemfile
@@ -449,6 +415,7 @@ files:
 - bin/console
 - bin/setup
 - config/.gitkeep
+- docker-compose.yml
 - docs/images/pushmi-pullyu.png
 - docs/images/system-infrastructure-diagram.png
 - examples/pushmi_pullyu.yml
@@ -469,7 +436,8 @@ files:
 homepage: https://github.com/ualbertalib/pushmi_pullyu
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -485,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Ruby application to manage flow of content from Jupiter into Swift for preservation

data/.travis.yml

@@ -1,16 +0,0 @@
-language: ruby
-sudo: false
-cache: bundler
-services:
-  - redis-server
-
-before_install:
-  - gem install bundler
-  - gem update bundler
-
-rvm:
-  - 2.5.0
-  - 2.6.0
-
-before_script:
-  - bundle exec danger