pushmi_pullyu 2.0.2 → 2.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. checksums.yaml +4 -4
  2. data/.github/CODEOWNERS +1 -1
  3. data/.github/workflows/ruby.yml +1 -1
  4. data/.rubocop.yml +2 -2
  5. data/CHANGELOG.md +8 -0
  6. data/README.md +6 -1
  7. data/docker-compose.yml +11 -0
  8. data/examples/pushmi_pullyu.yml +0 -3
  9. data/lib/pushmi_pullyu/aip/downloader.rb +1 -3
  10. data/lib/pushmi_pullyu/logging.rb +60 -0
  11. data/lib/pushmi_pullyu/version.rb +1 -1
  12. data/lib/pushmi_pullyu.rb +6 -10
  13. data/pushmi_pullyu.gemspec +1 -3
  14. metadata +5 -38
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e0e6a8b121842bd88b72a166f8ff85b9b7d78e8dd294c248c573af3981582a81
4
- data.tar.gz: 29e1631a92cd88dcb827769d6d6ef1e89172cc099fa88eaa1193406f4df604ba
3
+ metadata.gz: d64e5d619eb9dee63c358167cd6778034eb72fa2686fbe95fa187cfdfe7525b3
4
+ data.tar.gz: 333805bd92b1d5e308949d583e46056ffb1aeac9900d9c23ca59f2b88a7a4d01
5
5
  SHA512:
6
- metadata.gz: 4daaab3f235775f372c1141f1eabc0f8f122272115105d07f08797d492d2475c003e7145ae9aa39df7ce7a0592da6f0eb3ff3bc2bd65e71758299dbaaf6ba535
7
- data.tar.gz: f96752a30fc91992224d2a76c39106a1992aa4eb2a2eec4b9f09b30b503a403b74353b6b0e6eecec0e85911760c02797f3fc9a1bb51dc35e32fc6dc299925450
6
+ metadata.gz: b8b162ca028ca37362d0bcd68737189f6af656c747bfc18a1593e35635bddf606d649e8d02299d092b1b569cab7e8a03950855fe2f7da4f5877a2714c99af8cc
7
+ data.tar.gz: 56c8653b1d868993e31607de7cd1d811752ff4de1ab63813dec2f745321f2b7d6a54107eb304f0f93191d6372bf22e55977512155edca08ee25ad96c23fc0099
data/.github/CODEOWNERS CHANGED
@@ -1,2 +1,2 @@
1
1
  # https://help.github.com/articles/about-codeowners/
2
- * @ConnorSheremeta @lagoan @mbarnett @murny @pgwillia @piyapongch
2
+ * @ConnorSheremeta @lagoan @henryzhang87 @pgwillia @piyapongch
data/.github/workflows/ruby.yml CHANGED
@@ -25,7 +25,7 @@ jobs:
25
25
  run: bundle exec rubocop --parallel
26
26
  - name: Run Danger
27
27
  env:
28
- DANGER_GITHUB_API_TOKEN: ${{ secrets.DANGER_GITHUB_API_TOKEN }}
28
+ DANGER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
29
29
  run: bundle exec danger
30
30
  - name: Run tests
31
31
  run: bundle exec rake spec
data/.rubocop.yml CHANGED
@@ -36,7 +36,7 @@ Metrics/ClassLength:
36
36
  Metrics/CyclomaticComplexity:
37
37
  Enabled: false
38
38
 
39
- Metrics/LineLength:
39
+ Layout/LineLength:
40
40
  Enabled: true
41
41
  Max: 120 # default is 80
42
42
 
@@ -107,4 +107,4 @@ Naming/VariableNumber:
107
107
  Enabled: false
108
108
 
109
109
  Style/OpenStructUse:
110
- Enabled: false
110
+ Enabled: false
data/CHANGELOG.md CHANGED
@@ -8,6 +8,13 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
8
8
 
9
9
  ## [Unreleased]
10
10
 
11
+ ## [2.0.3] - 2022-04-28
12
+
13
+ - Changed Danger token in Github Actions
14
+ - remove pg dependency
15
+ - unblock CI (rubocop and coveralls)
16
+ - Log to preservation_events.json as well in an easy to use json format.
17
+
11
18
  ## [2.0.2] - 2021-11-22
12
19
 
13
20
  - Fix authentication bug when using ssl
@@ -22,4 +29,5 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
22
29
  ### Removed
23
30
  - Data output for original_file information
24
31
 
32
+ -
25
33
  ## [1.0.6] - 2018-11-29
data/README.md CHANGED
@@ -6,7 +6,6 @@
6
6
 
7
7
  [![Gem Version](https://badge.fury.io/rb/pushmi_pullyu.svg)](https://rubygems.org/gems/pushmi_pullyu)
8
8
  [![Github Build Status](https://github.com/ualbertalib/pushmi_pullyu/workflows/CI/badge.svg)](https://github.com/ualbertalib/pushmi_pullyu/actions)
9
- [![Coverage Status](https://coveralls.io/repos/github/ualbertalib/pushmi_pullyu/badge.svg?branch=master)](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
10
9
 
11
10
  PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
12
11
 
@@ -104,6 +103,12 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
104
103
 
105
104
  ## Testing
106
105
 
106
+ Pre-requisites:
107
+
108
+ Will need jupiter running locally and Openstack Swift running on port 8080.
109
+ - [Instructions for jupiter](https://github.com/ualbertalib/jupiter#docker)
110
+ - Use `docker-compose up -d` to start Swift (additional instructions in docker-compose.yml)
111
+
107
112
  To run the test suite:
108
113
 
109
114
  ```bash
data/docker-compose.yml ADDED
@@ -0,0 +1,11 @@
1
+ services:
2
+ swift:
3
+ # useful if you need to re-run the tests/spec
4
+ # https://docs.openstack.org/ocata/cli-reference/swift.html
5
+ # manual steps to set environment variables and create container
6
+ # swift auth -v -U test:tester -K testing -A http://localhost:8080/auth/v1.0
7
+ # swift auth
8
+ # swift post ERA
9
+ image: openstackswift/saio
10
+ ports:
11
+ - '8080:8080'
data/examples/pushmi_pullyu.yml CHANGED
@@ -21,9 +21,6 @@ minimum_age: 0
21
21
  redis:
22
22
  url: redis://localhost:6379
23
23
 
24
- database:
25
- url: postgresql://jupiter:mysecretpassword@127.0.0.1/jupiter_development
26
-
27
24
  #parameters project_name and project_domain_name are required only for keystone v3 authentication
28
25
  swift:
29
26
  tenant: tester
data/lib/pushmi_pullyu/aip/downloader.rb CHANGED
@@ -103,9 +103,7 @@ class PushmiPullyu::AIP::Downloader
103
103
 
104
104
  response = @http.request(request)
105
105
  is_success = if response.is_a?(Net::HTTPSuccess)
106
- File.open(local, 'wb') do |file|
107
- file.write(response.body)
108
- end
106
+ File.binwrite(local, response.body)
109
107
  # Response was a success and the file was saved to local
110
108
  File.exist? local
111
109
  end
data/lib/pushmi_pullyu/logging.rb CHANGED
@@ -45,6 +45,7 @@ module PushmiPullyu::Logging
45
45
 
46
46
  def log_preservation_event(deposited_file, aip_directory)
47
47
  preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
48
+ preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
48
49
 
49
50
  message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
50
51
  "Here are the details of this preservation event:\n"\
@@ -72,6 +73,65 @@ module PushmiPullyu::Logging
72
73
  preservation_logger.info(message)
73
74
 
74
75
  preservation_logger.close
76
+
77
+ message_json_str = preservation_event_to_json(deposited_file, aip_directory)
78
+ preservation_json_logger.info("#{message_json_str},")
79
+ preservation_json_logger.close
80
+ end
81
+
82
+ ###
83
+ # Provides an alternative logging method in json format for the convenience of
84
+ # parsing in the process of auditing against OpenStack Swift preservation.
85
+ #
86
+ # output format:
87
+ # I, [2022-04-06T11:07:21.983875 #20791] INFO -- : \
88
+ # {
89
+ # "do_uuid": "83b5d21f-a60a-43ba-945a-f03deec64a1d",
90
+ # "aip_deposited_at": "Thu, 07 Apr 2022 16:37:00 GMT",
91
+ # "aip_md5sum": "fe5832a510799b04c1c503e46dc3b589",
92
+ # "aip_sha256": "",
93
+ # "aip_metadata": "{\"project-id\":\"83b5d21f-a60a-43ba-945a-f03deec64a1d\",
94
+ # \"aip-version\":\"1.0\",
95
+ # \"project\":\"ERA\",
96
+ # \"promise\":\"bronze\"}",
97
+ # "aip_file_details": [
98
+ # {
99
+ # "fileset_uuid": "b2c6ac0f-f2ed-489e-bbae-bd26465207aa",
100
+ # "file_name": "Spallacci_Amanda_202103_PhD.pdf",
101
+ # "file_type": "pdf",
102
+ # "file_size": "2051363"
103
+ # }
104
+ # ]
105
+ # }
106
+ #
107
+ # note:
108
+ # to parse, the prefix "I, ... INFO --:" in each line needs to be
109
+ # stripped using a bash command such as "sed"
110
+ def preservation_event_to_json(deposited_file, aip_directory)
111
+ message = {}
112
+
113
+ message['do_uuid'] = deposited_file.name.to_s
114
+ message['aip_deposited_at'] = deposited_file.last_modified.to_s
115
+ message['aip_md5sum'] = deposited_file.etag.to_s
116
+ message['aip_sha256'] = ''
117
+ message['aip_metadata'] = deposited_file.metadata.to_json.to_s
118
+
119
+ file_details = file_log_details(aip_directory)
120
+
121
+ tmp_details = []
122
+ if file_details.present?
123
+ file_details.each do |file_detail|
124
+ tmp_hash = {}
125
+ tmp_hash['fileset_uuid'] = file_detail[:fileset_name].to_s
126
+ tmp_hash['file_name'] = file_detail[:file_name].to_s
127
+ tmp_hash['file_type'] = file_detail[:file_extension].to_s
128
+ tmp_hash['file_size'] = file_detail[:file_size].to_s
129
+ tmp_details << tmp_hash
130
+ end
131
+ end
132
+
133
+ message['aip_file_details'] = tmp_details
134
+ message.to_json
75
135
  end
76
136
 
77
137
  def reopen
data/lib/pushmi_pullyu/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module PushmiPullyu
2
- VERSION = '2.0.2'.freeze
2
+ VERSION = '2.0.3'.freeze
3
3
  end
data/lib/pushmi_pullyu.rb CHANGED
@@ -40,18 +40,14 @@ module PushmiPullyu
40
40
  },
41
41
  rollbar: {
42
42
  },
43
- database: {
44
- encoding: 'utf8',
45
- pool: ENV['RAILS_MAX_THREADS'] || 5,
46
- url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
47
- database: 'jupiter_development'
48
- },
43
+ # rubocop disable: Style/FetchEnvVar
49
44
  jupiter: {
50
- user: ENV['JUPITER_USER'],
51
- api_key: ENV['JUPITER_API_KEY'],
52
- jupiter_url: ENV['JUPITER_URL'] || 'http://localhost:3000/',
53
- aip_api_path: ENV['JUPITER_AIP_API_PATH'] || 'aip/v1'
45
+ user: ENV.fetch('JUPITER_USER', nil),
46
+ api_key: ENV.fetch('JUPITER_API_KEY', nil),
47
+ jupiter_url: ENV.fetch('JUPITER_URL', nil) || 'http://era.lvh.me:3000/',
48
+ aip_api_path: ENV.fetch('JUPITER_AIP_API_PATH', nil) || 'aip/v1'
54
49
  }
50
+ # rubocop enable: Style/FetchEnvVar
55
51
  }.freeze
56
52
 
57
53
  def self.options
data/pushmi_pullyu.gemspec CHANGED
@@ -21,13 +21,12 @@ Gem::Specification.new do |spec|
21
21
 
22
22
  spec.required_ruby_version = '>= 2.5'
23
23
 
24
- spec.add_runtime_dependency 'activesupport', '>= 5', '< 7'
24
+ spec.add_runtime_dependency 'activesupport', '>= 5', '< 8'
25
25
  spec.add_runtime_dependency 'bagit', '~> 0.4'
26
26
  spec.add_runtime_dependency 'connection_pool', '~> 2.2'
27
27
  spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
28
28
  spec.add_runtime_dependency 'minitar', '~> 0.7'
29
29
  spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
30
- spec.add_runtime_dependency 'pg', '>= 1.0', '< 1.2'
31
30
  spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
32
31
  spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
33
32
  spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
@@ -36,7 +35,6 @@ Gem::Specification.new do |spec|
36
35
  spec.add_runtime_dependency 'uuid', '~> 2.3.9'
37
36
 
38
37
  spec.add_development_dependency 'bundler', '~> 2.0'
39
- spec.add_development_dependency 'coveralls', '~> 0.8'
40
38
  spec.add_development_dependency 'danger', '~> 8.0'
41
39
  spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
42
40
  spec.add_development_dependency 'pry-byebug', '~> 3.6'
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pushmi_pullyu
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.2
4
+ version: 2.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shane Murnaghan
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: exe
11
11
  cert_chain: []
12
- date: 2021-11-23 00:00:00.000000000 Z
12
+ date: 2022-04-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activesupport
@@ -20,7 +20,7 @@ dependencies:
20
20
  version: '5'
21
21
  - - "<"
22
22
  - !ruby/object:Gem::Version
23
- version: '7'
23
+ version: '8'
24
24
  type: :runtime
25
25
  prerelease: false
26
26
  version_requirements: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
30
30
  version: '5'
31
31
  - - "<"
32
32
  - !ruby/object:Gem::Version
33
- version: '7'
33
+ version: '8'
34
34
  - !ruby/object:Gem::Dependency
35
35
  name: bagit
36
36
  requirement: !ruby/object:Gem::Requirement
@@ -113,26 +113,6 @@ dependencies:
113
113
  - - ">="
114
114
  - !ruby/object:Gem::Version
115
115
  version: 3.3.10
116
- - !ruby/object:Gem::Dependency
117
- name: pg
118
- requirement: !ruby/object:Gem::Requirement
119
- requirements:
120
- - - ">="
121
- - !ruby/object:Gem::Version
122
- version: '1.0'
123
- - - "<"
124
- - !ruby/object:Gem::Version
125
- version: '1.2'
126
- type: :runtime
127
- prerelease: false
128
- version_requirements: !ruby/object:Gem::Requirement
129
- requirements:
130
- - - ">="
131
- - !ruby/object:Gem::Version
132
- version: '1.0'
133
- - - "<"
134
- - !ruby/object:Gem::Version
135
- version: '1.2'
136
116
  - !ruby/object:Gem::Dependency
137
117
  name: rdf
138
118
  requirement: !ruby/object:Gem::Requirement
@@ -261,20 +241,6 @@ dependencies:
261
241
  - - "~>"
262
242
  - !ruby/object:Gem::Version
263
243
  version: '2.0'
264
- - !ruby/object:Gem::Dependency
265
- name: coveralls
266
- requirement: !ruby/object:Gem::Requirement
267
- requirements:
268
- - - "~>"
269
- - !ruby/object:Gem::Version
270
- version: '0.8'
271
- type: :development
272
- prerelease: false
273
- version_requirements: !ruby/object:Gem::Requirement
274
- requirements:
275
- - - "~>"
276
- - !ruby/object:Gem::Version
277
- version: '0.8'
278
244
  - !ruby/object:Gem::Dependency
279
245
  name: danger
280
246
  requirement: !ruby/object:Gem::Requirement
@@ -449,6 +415,7 @@ files:
449
415
  - bin/console
450
416
  - bin/setup
451
417
  - config/.gitkeep
418
+ - docker-compose.yml
452
419
  - docs/images/pushmi-pullyu.png
453
420
  - docs/images/system-infrastructure-diagram.png
454
421
  - examples/pushmi_pullyu.yml