pushmi_pullyu 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0e6a8b121842bd88b72a166f8ff85b9b7d78e8dd294c248c573af3981582a81
-  data.tar.gz: 29e1631a92cd88dcb827769d6d6ef1e89172cc099fa88eaa1193406f4df604ba
+  metadata.gz: d64e5d619eb9dee63c358167cd6778034eb72fa2686fbe95fa187cfdfe7525b3
+  data.tar.gz: 333805bd92b1d5e308949d583e46056ffb1aeac9900d9c23ca59f2b88a7a4d01
 SHA512:
-  metadata.gz: 4daaab3f235775f372c1141f1eabc0f8f122272115105d07f08797d492d2475c003e7145ae9aa39df7ce7a0592da6f0eb3ff3bc2bd65e71758299dbaaf6ba535
-  data.tar.gz: f96752a30fc91992224d2a76c39106a1992aa4eb2a2eec4b9f09b30b503a403b74353b6b0e6eecec0e85911760c02797f3fc9a1bb51dc35e32fc6dc299925450
+  metadata.gz: b8b162ca028ca37362d0bcd68737189f6af656c747bfc18a1593e35635bddf606d649e8d02299d092b1b569cab7e8a03950855fe2f7da4f5877a2714c99af8cc
+  data.tar.gz: 56c8653b1d868993e31607de7cd1d811752ff4de1ab63813dec2f745321f2b7d6a54107eb304f0f93191d6372bf22e55977512155edca08ee25ad96c23fc0099

data/.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # https://help.github.com/articles/about-codeowners/
-* @ConnorSheremeta @lagoan @mbarnett @murny @pgwillia @piyapongch
+* @ConnorSheremeta @lagoan @henryzhang87 @pgwillia @piyapongch

data/.github/workflows/ruby.yml

@@ -25,7 +25,7 @@ jobs:
       run: bundle exec rubocop --parallel
     - name: Run Danger
       env:
-        DANGER_GITHUB_API_TOKEN: ${{ secrets.DANGER_GITHUB_API_TOKEN }}
+        DANGER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: bundle exec danger
     - name: Run tests
       run: bundle exec rake spec

data/.rubocop.yml

@@ -36,7 +36,7 @@ Metrics/ClassLength:
 Metrics/CyclomaticComplexity:
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Enabled: true
   Max: 120 # default is 80
 
@@ -107,4 +107,4 @@ Naming/VariableNumber:
   Enabled: false
 
 Style/OpenStructUse:
-  Enabled: false
+  Enabled: false

data/CHANGELOG.md

@@ -8,6 +8,13 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
 
 ## [Unreleased]
 
+## [2.0.3] - 2022-04-28
+
+- Changed Danger token in Github Actions
+- remove pg dependency
+- unblock CI (rubocop and coveralls)
+- Log to preservation_events.json as well in an easy to use json format.
+
 ## [2.0.2] - 2021-11-22
 
 - Fix authentication bug when using ssl 
@@ -22,4 +29,5 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
 ### Removed
 - Data output for original_file information
 
+- 
 ## [1.0.6] - 2018-11-29

data/README.md

@@ -6,7 +6,6 @@
 
 [![Gem Version](https://badge.fury.io/rb/pushmi_pullyu.svg)](https://rubygems.org/gems/pushmi_pullyu)
 [![Github Build Status](https://github.com/ualbertalib/pushmi_pullyu/workflows/CI/badge.svg)](https://github.com/ualbertalib/pushmi_pullyu/actions)
-[![Coverage Status](https://coveralls.io/repos/github/ualbertalib/pushmi_pullyu/badge.svg?branch=master)](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
 
 PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
 
@@ -104,6 +103,12 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Testing
 
+Pre-requisites:
+
+Will need jupiter running locally and Openstack Swift running on port 8080.
+- [Instructions for jupiter](https://github.com/ualbertalib/jupiter#docker)
+- Use `docker-compose up -d` to start Swift (additional instructions in docker-compose.yml)
+
 To run the test suite:
 
 ```bash

data/docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  swift:
+    # useful if you need to re-run the tests/spec
+    # https://docs.openstack.org/ocata/cli-reference/swift.html
+    # manual steps to set environment variables and create container
+    # swift auth -v -U test:tester -K testing -A http://localhost:8080/auth/v1.0
+    # swift auth
+    # swift post ERA
+    image: openstackswift/saio
+    ports:
+      - '8080:8080'

data/examples/pushmi_pullyu.yml

@@ -21,9 +21,6 @@ minimum_age: 0
 redis:
   url: redis://localhost:6379
 
-database:
-  url: postgresql://jupiter:mysecretpassword@127.0.0.1/jupiter_development
-
 #parameters project_name and project_domain_name are required only for keystone v3 authentication
 swift:
   tenant: tester

data/lib/pushmi_pullyu/aip/downloader.rb

@@ -103,9 +103,7 @@ class PushmiPullyu::AIP::Downloader
 
     response = @http.request(request)
     is_success = if response.is_a?(Net::HTTPSuccess)
-                   File.open(local, 'wb') do |file|
-                     file.write(response.body)
-                   end
+                   File.binwrite(local, response.body)
                    # Response was a success and the file was saved to local
                    File.exist? local
                  end

data/lib/pushmi_pullyu/logging.rb

@@ -45,6 +45,7 @@ module PushmiPullyu::Logging
 
     def log_preservation_event(deposited_file, aip_directory)
       preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
+      preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
 
       message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
                 "Here are the details of this preservation event:\n"\
@@ -72,6 +73,65 @@ module PushmiPullyu::Logging
       preservation_logger.info(message)
 
       preservation_logger.close
+
+      message_json_str = preservation_event_to_json(deposited_file, aip_directory)
+      preservation_json_logger.info("#{message_json_str},")
+      preservation_json_logger.close
+    end
+
+    ###
+    # Provides an alternative logging method in json format for the convenience of
+    # parsing in the process of auditing against OpenStack Swift preservation.
+    #
+    # output format:
+    #  I, [2022-04-06T11:07:21.983875 #20791]  INFO -- : \
+    # {
+    #  "do_uuid": "83b5d21f-a60a-43ba-945a-f03deec64a1d",
+    #  "aip_deposited_at": "Thu, 07 Apr 2022 16:37:00 GMT",
+    #  "aip_md5sum": "fe5832a510799b04c1c503e46dc3b589",
+    #  "aip_sha256": "",
+    #  "aip_metadata": "{\"project-id\":\"83b5d21f-a60a-43ba-945a-f03deec64a1d\",
+    #                  \"aip-version\":\"1.0\",
+    #                  \"project\":\"ERA\",
+    #                  \"promise\":\"bronze\"}",
+    #  "aip_file_details": [
+    #   {
+    #     "fileset_uuid": "b2c6ac0f-f2ed-489e-bbae-bd26465207aa",
+    #     "file_name": "Spallacci_Amanda_202103_PhD.pdf",
+    #     "file_type": "pdf",
+    #     "file_size": "2051363"
+    #    }
+    #   ]
+    # }
+    #
+    # note:
+    #   to parse, the prefix "I, ... INFO --:" in each line needs to be
+    #   stripped using a bash command such as "sed"
+    def preservation_event_to_json(deposited_file, aip_directory)
+      message = {}
+
+      message['do_uuid'] = deposited_file.name.to_s
+      message['aip_deposited_at'] = deposited_file.last_modified.to_s
+      message['aip_md5sum'] = deposited_file.etag.to_s
+      message['aip_sha256'] = ''
+      message['aip_metadata'] = deposited_file.metadata.to_json.to_s
+
+      file_details = file_log_details(aip_directory)
+
+      tmp_details = []
+      if file_details.present?
+        file_details.each do |file_detail|
+          tmp_hash = {}
+          tmp_hash['fileset_uuid'] = file_detail[:fileset_name].to_s
+          tmp_hash['file_name'] = file_detail[:file_name].to_s
+          tmp_hash['file_type'] = file_detail[:file_extension].to_s
+          tmp_hash['file_size'] = file_detail[:file_size].to_s
+          tmp_details << tmp_hash
+        end
+      end
+
+      message['aip_file_details'] = tmp_details
+      message.to_json
     end
 
     def reopen

data/lib/pushmi_pullyu/version.rb

@@ -1,3 +1,3 @@
 module PushmiPullyu
-  VERSION = '2.0.2'.freeze
+  VERSION = '2.0.3'.freeze
 end

data/lib/pushmi_pullyu.rb

@@ -40,18 +40,14 @@ module PushmiPullyu
     },
     rollbar: {
     },
-    database: {
-      encoding: 'utf8',
-      pool: ENV['RAILS_MAX_THREADS'] || 5,
-      url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
-      database: 'jupiter_development'
-    },
+    # rubocop disable: Style/FetchEnvVar
     jupiter: {
-      user: ENV['JUPITER_USER'],
-      api_key: ENV['JUPITER_API_KEY'],
-      jupiter_url: ENV['JUPITER_URL'] || 'http://localhost:3000/',
-      aip_api_path: ENV['JUPITER_AIP_API_PATH'] || 'aip/v1'
+      user: ENV.fetch('JUPITER_USER', nil),
+      api_key: ENV.fetch('JUPITER_API_KEY', nil),
+      jupiter_url: ENV.fetch('JUPITER_URL', nil) || 'http://era.lvh.me:3000/',
+      aip_api_path: ENV.fetch('JUPITER_AIP_API_PATH', nil) || 'aip/v1'
     }
+    # rubocop enable: Style/FetchEnvVar
   }.freeze
 
   def self.options

data/pushmi_pullyu.gemspec

@@ -21,13 +21,12 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.5'
 
-  spec.add_runtime_dependency 'activesupport', '>= 5', '< 7'
+  spec.add_runtime_dependency 'activesupport', '>= 5', '< 8'
   spec.add_runtime_dependency 'bagit', '~> 0.4'
   spec.add_runtime_dependency 'connection_pool', '~> 2.2'
   spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
   spec.add_runtime_dependency 'minitar', '~> 0.7'
   spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
-  spec.add_runtime_dependency 'pg', '>= 1.0', '< 1.2'
   spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
   spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
   spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
@@ -36,7 +35,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'uuid', '~> 2.3.9'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'coveralls', '~> 0.8'
   spec.add_development_dependency 'danger', '~> 8.0'
   spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
   spec.add_development_dependency 'pry-byebug', '~> 3.6'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pushmi_pullyu
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Shane Murnaghan
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-11-23 00:00:00.000000000 Z
+date: 2022-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -20,7 +20,7 @@ dependencies:
         version: '5'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7'
+        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
         version: '5'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7'
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: bagit
   requirement: !ruby/object:Gem::Requirement
@@ -113,26 +113,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.3.10
-- !ruby/object:Gem::Dependency
-  name: pg
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rdf
   requirement: !ruby/object:Gem::Requirement
@@ -261,20 +241,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: coveralls
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: danger
   requirement: !ruby/object:Gem::Requirement
@@ -449,6 +415,7 @@ files:
 - bin/console
 - bin/setup
 - config/.gitkeep
+- docker-compose.yml
 - docs/images/pushmi-pullyu.png
 - docs/images/system-infrastructure-diagram.png
 - examples/pushmi_pullyu.yml