pushmi_pullyu 2.0.6 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 128bfb644445954d7b9c5eb3700dd1f3e4d3c92ea1e8448ab98054d750b2d53d
4
- data.tar.gz: 2fc44c64692367c9c4b254ec32b8241c7ecdc8fb1a33384561cbfcd34096a085
3
+ metadata.gz: da4560ad8303a06ea455d4da2915bd9be6a98edb70a8ee91bc12cb25d977f3b5
4
+ data.tar.gz: db98dc991f3149b0d217f6ae61f9c07c34ecc8ab7c878d4e60ecdbc08aed95bd
5
5
  SHA512:
6
- metadata.gz: daaaf67bda17bdea14310b41a87a1c51190f41337d8f0d467887dea5ed9cc700b2b05822bddd6038a4c7746927439d2576fd8a7c3fb2a878d7f59a8ac84e45be
7
- data.tar.gz: 33197ef8eb83869cea35ea5af258d83c16c56e77cbb0716fc61a6fa344b3d5cdf75e25a0fd72d6e2ab71148ddae9648351d29c5f2b53c9569af9f04869538ba0
6
+ metadata.gz: 5d565b6ecf4fcd0cab2690f04aaf619dc70d83848a8fc48960de393fac94b83ff18f9751528dbbb184c4f411d270113ec90c807146c8c14e35010c8a40434734
7
+ data.tar.gz: 478f4047534baaf2931733f0f48d199dd51088262e0076f5b52ad1f63f74fb5fee6dc5265ec8c8b5064f8b87d1ecaebf94513a5d9ecf299c3455df4526fbc9cb
data/.rubocop.yml CHANGED
@@ -20,7 +20,6 @@ Layout/EmptyLinesAroundClassBody:
20
20
  EnforcedStyle: empty_lines_except_namespace
21
21
 
22
22
  Layout/IndentationConsistency:
23
- Enabled: true
24
23
  EnforcedStyle: normal
25
24
 
26
25
  # A calculated magnitude based on number of assignments,
@@ -61,15 +60,6 @@ Style/ClassAndModuleChildren:
61
60
  Style/Documentation:
62
61
  Enabled: false
63
62
 
64
- Style/HashEachMethods:
65
- Enabled: true
66
-
67
- Style/HashTransformKeys:
68
- Enabled: true
69
-
70
- Style/HashTransformValues:
71
- Enabled: true
72
-
73
63
  Naming/FileName:
74
64
  Exclude:
75
65
  - Dangerfile
data/CHANGELOG.md CHANGED
@@ -6,8 +6,17 @@ PushmiPullyu is a Ruby application, whose primary job is to manage the flow of c
6
6
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
7
7
  and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
8
8
 
9
- ## [Unreleased]
9
+ ## [2.1.0]
10
+ - Add more logging information [#433](https://github.com/ualbertalib/pushmi_pullyu/issues/433)
11
+ - Add V3 authentication [#349](https://github.com/ualbertalib/pushmi_pullyu/issues/349)
10
12
 
13
+ - Add logic to perform authentication against the V3 Auth protocol
14
+
15
+ ## [2.0.7] - 2023-09-13
16
+
17
+ - Fix nil exception [#314](https://github.com/ualbertalib/pushmi_pullyu/issues/314)
18
+ - Bump rubocop from 1.28.1 to 1.54.1 [PR#348](https://github.com/ualbertalib/pushmi_pullyu/pull/348)
19
+ - Add missing logging for standard exception [#314](https://github.com/ualbertalib/pushmi_pullyu/issues/314)
11
20
  ## [2.0.6] - 2023-03-17
12
21
 
13
22
  - Fix URI concatenation for jupiter's base url. [#309](https://github.com/ualbertalib/pushmi_pullyu/issues/309)
@@ -51,5 +60,4 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
51
60
  ### Removed
52
61
  - Data output for original_file information
53
62
 
54
- -
55
63
  ## [1.0.6] - 2018-11-29
data/Gemfile CHANGED
@@ -2,3 +2,17 @@ source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in pushmi_pullyu.gemspec
4
4
  gemspec
5
+
6
+ group :development, :test do
7
+ gem 'bundler', '~> 2.0'
8
+ gem 'danger', '~> 9.3'
9
+ gem 'pry', '~> 0.10', '>= 0.10.4'
10
+ gem 'pry-byebug', '~> 3.6'
11
+ gem 'rake', '~> 13.0'
12
+ gem 'rspec', '~> 3.13'
13
+ gem 'rubocop', '~> 1.64'
14
+ gem 'rubocop-rspec', '~> 2.24'
15
+ gem 'timecop', '~> 0.9'
16
+ gem 'vcr', '~> 5.0'
17
+ gem 'webmock', '~> 3.19'
18
+ end
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- pushmi_pullyu (2.0.6)
4
+ pushmi_pullyu (2.1.0)
5
5
  activesupport (>= 5, < 8)
6
6
  bagit (~> 0.4)
7
7
  connection_pool (~> 2.2)
@@ -10,7 +10,7 @@ PATH
10
10
  openstack (~> 3.3, >= 3.3.10)
11
11
  rdf (>= 1.99, < 4.0)
12
12
  rdf-n3 (>= 1.99, < 4.0)
13
- redis (>= 3.3, < 5.0)
13
+ redis (>= 3.3, < 6.0)
14
14
  rest-client (>= 1.8, < 3.0)
15
15
  rollbar (>= 2.18, < 4.0)
16
16
  uuid (~> 2.3.9)
@@ -18,80 +18,73 @@ PATH
18
18
  GEM
19
19
  remote: https://rubygems.org/
20
20
  specs:
21
- activesupport (7.0.4.2)
21
+ activesupport (7.1.1)
22
+ base64
23
+ bigdecimal
22
24
  concurrent-ruby (~> 1.0, >= 1.0.2)
25
+ connection_pool (>= 2.2.5)
26
+ drb
23
27
  i18n (>= 1.6, < 2)
24
28
  minitest (>= 5.1)
29
+ mutex_m
25
30
  tzinfo (~> 2.0)
26
- addressable (2.8.1)
31
+ addressable (2.8.5)
27
32
  public_suffix (>= 2.0.2, < 6.0)
28
- amazing_print (1.4.0)
29
33
  ast (2.4.2)
30
- bagit (0.4.5)
34
+ bagit (0.4.6)
31
35
  docopt (~> 0.5.0)
32
36
  validatable (~> 1.6)
37
+ base64 (0.2.0)
38
+ bigdecimal (3.1.4)
33
39
  builder (3.2.4)
34
40
  byebug (11.1.3)
35
- claide (1.0.3)
41
+ claide (1.1.0)
36
42
  claide-plugins (0.9.2)
37
43
  cork
38
44
  nap
39
45
  open4 (~> 1.3)
40
46
  coderay (1.1.3)
41
47
  colored2 (3.1.2)
42
- concurrent-ruby (1.2.0)
43
- connection_pool (2.3.0)
48
+ concurrent-ruby (1.2.2)
49
+ connection_pool (2.4.1)
44
50
  cork (0.3.0)
45
51
  colored2 (~> 3.1)
46
52
  crack (0.4.5)
47
53
  rexml
48
54
  daemons (1.4.1)
49
- danger (8.4.2)
55
+ danger (9.3.2)
50
56
  claide (~> 1.0)
51
57
  claide-plugins (>= 0.9.2)
52
58
  colored2 (~> 3.1)
53
59
  cork (~> 0.1)
54
- faraday (>= 0.9.0, < 2.0)
60
+ faraday (>= 0.9.0, < 3.0)
55
61
  faraday-http-cache (~> 2.0)
56
- git (~> 1.7)
62
+ git (~> 1.13)
57
63
  kramdown (~> 2.3)
58
64
  kramdown-parser-gfm (~> 1.0)
59
65
  no_proxy_fix
60
- octokit (~> 4.7)
66
+ octokit (~> 6.0)
61
67
  terminal-table (>= 1, < 4)
62
- diff-lcs (1.5.0)
68
+ diff-lcs (1.5.1)
63
69
  docopt (0.5.0)
64
70
  domain_name (0.5.20190701)
65
71
  unf (>= 0.0.5, < 1.0.0)
66
- ebnf (2.3.1)
67
- amazing_print (~> 1.4)
72
+ drb (2.1.1)
73
+ ruby2_keywords
74
+ ebnf (2.3.5)
68
75
  htmlentities (~> 4.3)
69
76
  rdf (~> 3.2)
70
77
  scanf (~> 1.0)
71
78
  sxp (~> 1.2)
72
- unicode-types (~> 1.7)
73
- faraday (1.8.0)
74
- faraday-em_http (~> 1.0)
75
- faraday-em_synchrony (~> 1.0)
76
- faraday-excon (~> 1.1)
77
- faraday-httpclient (~> 1.0.1)
78
- faraday-net_http (~> 1.0)
79
- faraday-net_http_persistent (~> 1.1)
80
- faraday-patron (~> 1.0)
81
- faraday-rack (~> 1.0)
82
- multipart-post (>= 1.2, < 3)
79
+ unicode-types (~> 1.8)
80
+ faraday (2.7.11)
81
+ base64
82
+ faraday-net_http (>= 2.0, < 3.1)
83
83
  ruby2_keywords (>= 0.0.4)
84
- faraday-em_http (1.0.0)
85
- faraday-em_synchrony (1.0.0)
86
- faraday-excon (1.1.0)
87
- faraday-http-cache (2.2.0)
84
+ faraday-http-cache (2.5.0)
88
85
  faraday (>= 0.8)
89
- faraday-httpclient (1.0.1)
90
- faraday-net_http (1.0.1)
91
- faraday-net_http_persistent (1.2.0)
92
- faraday-patron (1.0.0)
93
- faraday-rack (1.0.0)
94
- git (1.13.0)
86
+ faraday-net_http (3.0.2)
87
+ git (1.18.0)
95
88
  addressable (~> 2.8)
96
89
  rchardet (~> 1.8)
97
90
  hashdiff (1.0.1)
@@ -99,50 +92,53 @@ GEM
99
92
  http-accept (1.7.0)
100
93
  http-cookie (1.0.5)
101
94
  domain_name (~> 0.5)
102
- i18n (1.12.0)
95
+ i18n (1.14.1)
103
96
  concurrent-ruby (~> 1.0)
104
- json (2.6.3)
105
- kramdown (2.3.1)
97
+ json (2.7.2)
98
+ kramdown (2.4.0)
106
99
  rexml
107
100
  kramdown-parser-gfm (1.1.0)
108
101
  kramdown (~> 2.0)
102
+ language_server-protocol (3.17.0.3)
109
103
  link_header (0.0.8)
110
104
  logger (1.5.3)
111
105
  macaddr (1.7.2)
112
106
  systemu (~> 2.6.5)
113
107
  matrix (0.4.2)
114
108
  method_source (1.0.0)
115
- mime-types (3.4.1)
109
+ mime-types (3.5.1)
116
110
  mime-types-data (~> 3.2015)
117
- mime-types-data (3.2022.0105)
111
+ mime-types-data (3.2023.1003)
118
112
  minitar (0.9)
119
- minitest (5.17.0)
120
- multipart-post (2.1.1)
113
+ minitest (5.20.0)
114
+ mutex_m (0.1.2)
121
115
  nap (1.1.0)
122
- net-http-persistent (4.0.1)
116
+ net-http-persistent (4.0.2)
123
117
  connection_pool (~> 2.2)
124
118
  netrc (0.11.0)
125
119
  no_proxy_fix (0.1.2)
126
- octokit (4.21.0)
127
- faraday (>= 0.9)
128
- sawyer (~> 0.8.0, >= 0.5.3)
120
+ octokit (6.1.1)
121
+ faraday (>= 1, < 3)
122
+ sawyer (~> 0.9)
129
123
  open4 (1.3.4)
130
124
  openstack (3.3.21)
131
125
  json
132
- parallel (1.22.1)
133
- parser (3.1.2.1)
126
+ parallel (1.24.0)
127
+ parser (3.3.1.0)
134
128
  ast (~> 2.4.1)
135
- pry (0.14.1)
129
+ racc
130
+ pry (0.14.2)
136
131
  coderay (~> 1.1)
137
132
  method_source (~> 1.0)
138
133
  pry-byebug (3.10.1)
139
134
  byebug (~> 11.0)
140
135
  pry (>= 0.13, < 0.15)
141
- public_suffix (5.0.0)
136
+ public_suffix (5.0.3)
137
+ racc (1.8.0)
142
138
  rainbow (3.1.1)
143
139
  rake (13.0.6)
144
140
  rchardet (1.8.0)
145
- rdf (3.2.9)
141
+ rdf (3.2.11)
146
142
  link_header (~> 0.0, >= 0.0.8)
147
143
  rdf-aggregate-repo (3.2.1)
148
144
  rdf (~> 3.2)
@@ -154,78 +150,91 @@ GEM
154
150
  rdf-xsd (3.2.1)
155
151
  rdf (~> 3.2)
156
152
  rexml (~> 3.2)
157
- redis (4.8.1)
158
- regexp_parser (2.6.0)
153
+ redis (5.0.8)
154
+ redis-client (>= 0.17.0)
155
+ redis-client (0.17.1)
156
+ connection_pool
157
+ regexp_parser (2.9.2)
159
158
  rest-client (2.1.0)
160
159
  http-accept (>= 1.7.0, < 2.0)
161
160
  http-cookie (>= 1.0.2, < 2.0)
162
161
  mime-types (>= 1.16, < 4.0)
163
162
  netrc (~> 0.8)
164
- rexml (3.2.5)
165
- rollbar (3.4.0)
166
- rspec (3.12.0)
167
- rspec-core (~> 3.12.0)
168
- rspec-expectations (~> 3.12.0)
169
- rspec-mocks (~> 3.12.0)
170
- rspec-core (3.12.0)
171
- rspec-support (~> 3.12.0)
172
- rspec-expectations (3.12.0)
163
+ rexml (3.2.8)
164
+ strscan (>= 3.0.9)
165
+ rollbar (3.4.1)
166
+ rspec (3.13.0)
167
+ rspec-core (~> 3.13.0)
168
+ rspec-expectations (~> 3.13.0)
169
+ rspec-mocks (~> 3.13.0)
170
+ rspec-core (3.13.0)
171
+ rspec-support (~> 3.13.0)
172
+ rspec-expectations (3.13.0)
173
173
  diff-lcs (>= 1.2.0, < 2.0)
174
- rspec-support (~> 3.12.0)
175
- rspec-mocks (3.12.0)
174
+ rspec-support (~> 3.13.0)
175
+ rspec-mocks (3.13.0)
176
176
  diff-lcs (>= 1.2.0, < 2.0)
177
- rspec-support (~> 3.12.0)
178
- rspec-support (3.12.0)
179
- rubocop (1.28.1)
177
+ rspec-support (~> 3.13.0)
178
+ rspec-support (3.13.0)
179
+ rubocop (1.64.0)
180
+ json (~> 2.3)
181
+ language_server-protocol (>= 3.17.0)
180
182
  parallel (~> 1.10)
181
- parser (>= 3.1.0.0)
183
+ parser (>= 3.3.0.2)
182
184
  rainbow (>= 2.2.2, < 4.0)
183
185
  regexp_parser (>= 1.8, < 3.0)
184
- rexml
185
- rubocop-ast (>= 1.17.0, < 2.0)
186
+ rexml (>= 3.2.5, < 4.0)
187
+ rubocop-ast (>= 1.31.1, < 2.0)
186
188
  ruby-progressbar (~> 1.7)
187
- unicode-display_width (>= 1.4.0, < 3.0)
188
- rubocop-ast (1.23.0)
189
- parser (>= 3.1.1.0)
190
- rubocop-rspec (2.11.1)
191
- rubocop (~> 1.19)
192
- ruby-progressbar (1.11.0)
189
+ unicode-display_width (>= 2.4.0, < 3.0)
190
+ rubocop-ast (1.31.3)
191
+ parser (>= 3.3.1.0)
192
+ rubocop-capybara (2.19.0)
193
+ rubocop (~> 1.41)
194
+ rubocop-factory_bot (2.24.0)
195
+ rubocop (~> 1.33)
196
+ rubocop-rspec (2.24.1)
197
+ rubocop (~> 1.33)
198
+ rubocop-capybara (~> 2.17)
199
+ rubocop-factory_bot (~> 2.22)
200
+ ruby-progressbar (1.13.0)
193
201
  ruby2_keywords (0.0.5)
194
- sawyer (0.8.2)
202
+ sawyer (0.9.2)
195
203
  addressable (>= 2.3.5)
196
- faraday (> 0.8, < 2.0)
204
+ faraday (>= 0.17.3, < 3)
197
205
  scanf (1.0.0)
198
- sparql (3.2.5)
199
- builder (~> 3.2)
200
- ebnf (~> 2.2, >= 2.3.1)
206
+ sparql (3.2.6)
207
+ builder (~> 3.2, >= 3.2.4)
208
+ ebnf (~> 2.3, >= 2.3.5)
201
209
  logger (~> 1.5)
202
- rdf (~> 3.2, >= 3.2.8)
203
- rdf-aggregate-repo (~> 3.2)
210
+ rdf (~> 3.2, >= 3.2.11)
211
+ rdf-aggregate-repo (~> 3.2, >= 3.2.1)
204
212
  rdf-xsd (~> 3.2)
205
- sparql-client (~> 3.2, >= 3.2.1)
206
- sxp (~> 1.2, >= 1.2.2)
207
- sparql-client (3.2.1)
208
- net-http-persistent (~> 4.0, >= 4.0.1)
209
- rdf (~> 3.2, >= 3.2.6)
210
- sxp (1.2.3)
213
+ sparql-client (~> 3.2, >= 3.2.2)
214
+ sxp (~> 1.2, >= 1.2.4)
215
+ sparql-client (3.2.2)
216
+ net-http-persistent (~> 4.0, >= 4.0.2)
217
+ rdf (~> 3.2, >= 3.2.11)
218
+ strscan (3.1.0)
219
+ sxp (1.2.4)
211
220
  matrix (~> 0.4)
212
221
  rdf (~> 3.2)
213
222
  systemu (2.6.5)
214
223
  terminal-table (3.0.2)
215
224
  unicode-display_width (>= 1.1.1, < 3)
216
- timecop (0.9.6)
225
+ timecop (0.9.8)
217
226
  tzinfo (2.0.6)
218
227
  concurrent-ruby (~> 1.0)
219
228
  unf (0.1.4)
220
229
  unf_ext
221
230
  unf_ext (0.0.8.2)
222
- unicode-display_width (2.3.0)
223
- unicode-types (1.8.0)
231
+ unicode-display_width (2.5.0)
232
+ unicode-types (1.9.0)
224
233
  uuid (2.3.9)
225
234
  macaddr (~> 1.0)
226
235
  validatable (1.6.7)
227
236
  vcr (5.1.0)
228
- webmock (3.18.1)
237
+ webmock (3.19.1)
229
238
  addressable (>= 2.8.0)
230
239
  crack (>= 0.3.2)
231
240
  hashdiff (>= 0.4.0, < 2.0.0)
@@ -235,17 +244,17 @@ PLATFORMS
235
244
 
236
245
  DEPENDENCIES
237
246
  bundler (~> 2.0)
238
- danger (~> 8.0)
247
+ danger (~> 9.3)
239
248
  pry (~> 0.10, >= 0.10.4)
240
249
  pry-byebug (~> 3.6)
241
250
  pushmi_pullyu!
242
251
  rake (~> 13.0)
243
- rspec (~> 3.0)
244
- rubocop (~> 1.23)
245
- rubocop-rspec (~> 2.6)
246
- timecop (~> 0.8)
252
+ rspec (~> 3.13)
253
+ rubocop (~> 1.64)
254
+ rubocop-rspec (~> 2.24)
255
+ timecop (~> 0.9)
247
256
  vcr (~> 5.0)
248
- webmock (~> 3.3)
257
+ webmock (~> 3.19)
249
258
 
250
259
  BUNDLED WITH
251
260
  2.3.12
@@ -32,7 +32,10 @@ swift:
32
32
  auth_url: http://localhost:8080/auth/v1.0
33
33
  project_name: demo
34
34
  project_domain_name: default
35
- container: ERA
35
+ container: era
36
+ # These 2 extra parameters are now required for keystone v3 authentication
37
+ auth_version: v3
38
+ user_domain: default
36
39
 
37
40
  rollbar:
38
41
  token: 'abc123xyz'
@@ -199,6 +199,8 @@ class PushmiPullyu::CLI
199
199
  def run_preservation_cycle
200
200
  begin
201
201
  entity = queue.wait_next_item
202
+ PushmiPullyu::Logging.log_preservation_attempt(entity,
203
+ queue.get_entity_ingestion_attempt(entity))
202
204
  return unless entity && entity[:type].present? && entity[:uuid].present?
203
205
  rescue StandardError => e
204
206
  log_exception(e)
@@ -212,27 +214,29 @@ class PushmiPullyu::CLI
212
214
  # Push tarred AIP to swift API
213
215
  deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
214
216
  # Log successful preservation event to the log files
215
- PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
217
+ PushmiPullyu::Logging.log_preservation_success(deposited_file, aip_directory)
216
218
  end
217
219
  # An EntityInvalid expection means there is a problem with the entity information format so there is no point in
218
220
  # readding it to the queue as it will always fail
219
221
  rescue PushmiPullyu::AIP::EntityInvalid => e
220
222
  rescue StandardError => e
223
+ log_exception(e)
221
224
  begin
222
225
  queue.add_entity_in_timeframe(entity)
226
+ PushmiPullyu::Logging.log_preservation_fail_and_retry(entity, queue.get_entity_ingestion_attempt(entity), e)
223
227
  rescue PushmiPullyu::PreservationQueue::MaxDepositAttemptsReached => e
228
+ PushmiPullyu::Logging.log_preservation_failure(entity, queue.get_entity_ingestion_attempt(entity), e)
224
229
  log_exception(e)
225
230
  end
226
231
 
227
- # rubocop:disable Lint/RescueException
228
232
  # Something other than a StandardError exception means something happened which we were not expecting!
229
233
  # Make sure we log the problem
234
+ # rubocop:disable Lint/RescueException
230
235
  rescue Exception => e
231
- raise e
232
- # rubocop:enable Lint/RescueException
233
- ensure
234
236
  log_exception(e)
237
+ raise e
235
238
  end
239
+ # rubocop:enable Lint/RescueException
236
240
  end
237
241
 
238
242
  def run_tick_loop
@@ -244,7 +248,11 @@ class PushmiPullyu::CLI
244
248
 
245
249
  def setup_log
246
250
  if options[:daemonize]
247
- PushmiPullyu::Logging.initialize_logger(PushmiPullyu.application_log_file)
251
+ PushmiPullyu::Logging.initialize_loggers(
252
+ log_target: PushmiPullyu.application_log_file,
253
+ events_target: "#{PushmiPullyu.options[:logdir]}/preservation_events.log",
254
+ json_target: "#{PushmiPullyu.options[:logdir]}/preservation_events.json"
255
+ )
248
256
  else
249
257
  logger.formatter = PushmiPullyu::Logging::SimpleFormatter.new
250
258
  end
@@ -266,10 +274,12 @@ class PushmiPullyu::CLI
266
274
  def swift
267
275
  @swift ||= PushmiPullyu::SwiftDepositer.new(username: options[:swift][:username],
268
276
  password: options[:swift][:password],
269
- tenant: options[:swift][:tenant],
270
277
  project_name: options[:swift][:project_name],
278
+ tenant: options[:swift][:tenant],
271
279
  project_domain_name: options[:swift][:project_domain_name],
272
- auth_url: options[:swift][:auth_url])
280
+ user_domain: options[:swift][:user_domain],
281
+ auth_url: options[:swift][:auth_url],
282
+ auth_version: options[:swift][:auth_version])
273
283
  end
274
284
 
275
285
  # On first call of shutdown, this will gracefully close the main run loop
@@ -21,14 +21,17 @@ module PushmiPullyu::Logging
21
21
 
22
22
  attr_writer :logger
23
23
 
24
- def initialize_logger(log_target = $stdout)
24
+ def initialize_loggers(log_target: $stdout, events_target: $stdout, json_target: $stdout)
25
+ @preservation_logger = Logger.new(events_target)
26
+ @preservation_json_logger = Logger.new(json_target)
27
+
25
28
  @logger = Logger.new(log_target)
26
29
  @logger.level = Logger::INFO
27
30
  @logger
28
31
  end
29
32
 
30
33
  def logger
31
- @logger ||= initialize_logger
34
+ @logger ||= initialize_loggers
32
35
  end
33
36
 
34
37
  def log_aip_activity(aip_directory, message)
@@ -43,10 +46,13 @@ module PushmiPullyu::Logging
43
46
  aip_logger.close
44
47
  end
45
48
 
46
- def log_preservation_event(deposited_file, aip_directory)
47
- preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
48
- preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
49
+ def log_preservation_event(message, message_json)
50
+ logger.info(message)
51
+ @preservation_logger.info(message)
52
+ @preservation_json_logger.info("#{message_json},")
53
+ end
49
54
 
55
+ def log_preservation_success(deposited_file, aip_directory)
50
56
  message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n" \
51
57
  "Here are the details of this preservation event:\n" \
52
58
  "\tUUID: '#{deposited_file.name}'\n" \
@@ -68,15 +74,62 @@ module PushmiPullyu::Logging
68
74
  end
69
75
  end
70
76
 
71
- # Log to both the application log, and the preservation log file
72
- logger.info(message)
73
- preservation_logger.info(message)
77
+ log_preservation_event(message, preservation_success_to_json(deposited_file, aip_directory))
78
+ end
74
79
 
75
- preservation_logger.close
80
+ def log_preservation_fail_and_retry(entity, retry_attempt, exception)
81
+ message = "#{entity[:type]} failed to be deposited and will try again.\n" \
82
+ "Here are the details of this preservation event:\n" \
83
+ "\t#{entity[:type]} uuid: #{entity[:uuid]}" \
84
+ "\tReadding to preservation queue with retry attempt: #{retry_attempt}\n" \
85
+ "\tError of type: #{exception.class.name}\n" \
86
+ "\tError message: #{exception.message}\n"
87
+
88
+ message_information = {
89
+ event_type: :fail_and_retry,
90
+ event_time: Time.now.to_s,
91
+ entity_type: entity[:type],
92
+ entity_uuid: entity[:uuid],
93
+ retry_attempt: retry_attempt,
94
+ error_message: exception.message
95
+ }
96
+
97
+ log_preservation_event(message, message_information.to_json)
98
+ end
76
99
 
77
- message_json_str = preservation_event_to_json(deposited_file, aip_directory)
78
- preservation_json_logger.info("#{message_json_str},")
79
- preservation_json_logger.close
100
+ def log_preservation_failure(entity, retry_attempt, exception)
101
+ message = "#{entity[:type]} failed to be deposited.\n" \
102
+ "Here are the details of this preservation event:\n" \
103
+ "\t#{entity[:type]} uuid: #{entity[:uuid]}" \
104
+ "\tRetry attempt: #{retry_attempt}\n"
105
+
106
+ message_information = {
107
+ event_type: :fail_and_retry,
108
+ event_time: Time.now.to_s,
109
+ entity_type: entity[:type],
110
+ entity_uuid: entity[:uuid],
111
+ retry_attempt: retry_attempt,
112
+ error_message: exception.message
113
+ }
114
+
115
+ log_preservation_event(message, message_information.to_json)
116
+ end
117
+
118
+ def log_preservation_attempt(entity, retry_attempt)
119
+ message = "#{entity[:type]} will attempt to be deposited.\n" \
120
+ "Here are the details of this preservation event:\n" \
121
+ "\t#{entity[:type]} uuid: #{entity[:uuid]}" \
122
+ "\tRetry attempt: #{retry_attempt}\n"
123
+
124
+ message_information = {
125
+ event_type: :attempt,
126
+ event_time: Time.now.to_s,
127
+ entity_type: entity[:type],
128
+ entity_uuid: entity[:uuid],
129
+ retry_attempt: retry_attempt
130
+ }
131
+
132
+ log_preservation_event(message, message_information.to_json)
80
133
  end
81
134
 
82
135
  ###
@@ -107,14 +160,14 @@ module PushmiPullyu::Logging
107
160
  # note:
108
161
  # to parse, the prefix "I, ... INFO --:" in each line needs to be
109
162
  # stripped using a bash command such as "sed"
110
- def preservation_event_to_json(deposited_file, aip_directory)
163
+ def preservation_success_to_json(deposited_file, aip_directory)
111
164
  message = {}
112
165
 
113
166
  message['do_uuid'] = deposited_file.name.to_s
114
167
  message['aip_deposited_at'] = deposited_file.last_modified.to_s
115
168
  message['aip_md5sum'] = deposited_file.etag.to_s
116
169
  message['aip_sha256'] = ''
117
- message['aip_metadata'] = deposited_file.metadata.to_json.to_s
170
+ message['aip_metadata'] = deposited_file.metadata
118
171
 
119
172
  file_details = file_log_details(aip_directory)
120
173
 
@@ -138,7 +191,7 @@ module PushmiPullyu::Logging
138
191
  if @logger
139
192
  @logger.reopen
140
193
  else
141
- @logger = initialize_logger
194
+ @logger = initialize_loggers
142
195
  end
143
196
  end
144
197
 
@@ -70,6 +70,13 @@ class PushmiPullyu::PreservationQueue
70
70
  end
71
71
  end
72
72
 
73
+ def get_entity_ingestion_attempt(entity)
74
+ entity_attempts_key = "#{PushmiPullyu.options[:ingestion_prefix]}#{entity[:uuid]}"
75
+ @redis.with do |connection|
76
+ return connection.get entity_attempts_key
77
+ end
78
+ end
79
+
73
80
  def add_entity_in_timeframe(entity)
74
81
  entity_attempts_key = "#{PushmiPullyu.options[:ingestion_prefix]}#{entity[:uuid]}"
75
82
 
@@ -6,16 +6,24 @@ class PushmiPullyu::SwiftDepositer
6
6
  attr_reader :swift_connection
7
7
 
8
8
  def initialize(connection)
9
- @swift_connection = OpenStack::Connection.create(
9
+ # Generic authentication parameters
10
+ swift_connection_parameters = {
10
11
  username: connection[:username],
11
12
  api_key: connection[:password],
12
- auth_method: 'password',
13
13
  auth_url: connection[:auth_url],
14
14
  project_name: connection[:project_name],
15
- project_domain_name: connection[:project_domain_name],
16
- authtenant_name: connection[:tenant],
15
+ auth_method: 'password',
17
16
  service_type: 'object-store'
18
- )
17
+ }
18
+
19
+ if connection[:auth_version] == 'v3'
20
+ swift_connection_parameters[:user_domain] = connection[:user_domain]
21
+ elsif connection[:auth_version] == 'v1'
22
+ swift_connection_parameters[:project_domain_name] = connection[:project_domain_name]
23
+ swift_connection_parameters[:authtenant_name] = connection[:tenant]
24
+ end
25
+
26
+ @swift_connection = OpenStack::Connection.create(swift_connection_parameters)
19
27
  end
20
28
 
21
29
  def deposit_file(file_name, swift_container)
@@ -1,3 +1,3 @@
1
1
  module PushmiPullyu
2
- VERSION = '2.0.6'.freeze
2
+ VERSION = '2.1.0'.freeze
3
3
  end
data/lib/pushmi_pullyu.rb CHANGED
@@ -39,10 +39,11 @@ module PushmiPullyu
39
39
  auth_url: 'http://localhost:8080/auth/v1.0',
40
40
  project_name: 'demo',
41
41
  project_domain_name: 'default',
42
- container: 'ERA'
43
- },
44
- rollbar: {
42
+ container: 'era',
43
+ auth_version: 'v3',
44
+ user_domain: 'default'
45
45
  },
46
+ rollbar: {},
46
47
  # rubocop disable: Style/FetchEnvVar
47
48
  jupiter: {
48
49
  user: ENV.fetch('JUPITER_USER', nil),
@@ -29,22 +29,11 @@ Gem::Specification.new do |spec|
29
29
  spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
30
30
  spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
31
31
  spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
32
- spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
32
+ spec.add_runtime_dependency 'redis', '>= 3.3', '< 6.0'
33
33
  spec.add_runtime_dependency 'rest-client', '>= 1.8', '< 3.0'
34
34
  spec.add_runtime_dependency 'rollbar', '>= 2.18', '< 4.0'
35
35
  spec.add_runtime_dependency 'uuid', '~> 2.3.9'
36
36
 
37
- spec.add_development_dependency 'bundler', '~> 2.0'
38
- spec.add_development_dependency 'danger', '~> 8.0'
39
- spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
40
- spec.add_development_dependency 'pry-byebug', '~> 3.6'
41
- spec.add_development_dependency 'rake', '~> 13.0'
42
- spec.add_development_dependency 'rspec', '~> 3.0'
43
- spec.add_development_dependency 'rubocop', '~> 1.23'
44
- spec.add_development_dependency 'rubocop-rspec', '~> 2.6'
45
- spec.add_development_dependency 'timecop', '~> 0.8'
46
- spec.add_development_dependency 'vcr', '~> 5.0'
47
- spec.add_development_dependency 'webmock', '~> 3.3'
48
37
  spec.metadata = {
49
38
  'rubygems_mfa_required' => 'true'
50
39
  }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pushmi_pullyu
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.6
4
+ version: 2.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shane Murnaghan
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: exe
11
11
  cert_chain: []
12
- date: 2023-03-17 00:00:00.000000000 Z
12
+ date: 2024-05-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activesupport
@@ -162,7 +162,7 @@ dependencies:
162
162
  version: '3.3'
163
163
  - - "<"
164
164
  - !ruby/object:Gem::Version
165
- version: '5.0'
165
+ version: '6.0'
166
166
  type: :runtime
167
167
  prerelease: false
168
168
  version_requirements: !ruby/object:Gem::Requirement
@@ -172,7 +172,7 @@ dependencies:
172
172
  version: '3.3'
173
173
  - - "<"
174
174
  - !ruby/object:Gem::Version
175
- version: '5.0'
175
+ version: '6.0'
176
176
  - !ruby/object:Gem::Dependency
177
177
  name: rest-client
178
178
  requirement: !ruby/object:Gem::Requirement
@@ -227,166 +227,6 @@ dependencies:
227
227
  - - "~>"
228
228
  - !ruby/object:Gem::Version
229
229
  version: 2.3.9
230
- - !ruby/object:Gem::Dependency
231
- name: bundler
232
- requirement: !ruby/object:Gem::Requirement
233
- requirements:
234
- - - "~>"
235
- - !ruby/object:Gem::Version
236
- version: '2.0'
237
- type: :development
238
- prerelease: false
239
- version_requirements: !ruby/object:Gem::Requirement
240
- requirements:
241
- - - "~>"
242
- - !ruby/object:Gem::Version
243
- version: '2.0'
244
- - !ruby/object:Gem::Dependency
245
- name: danger
246
- requirement: !ruby/object:Gem::Requirement
247
- requirements:
248
- - - "~>"
249
- - !ruby/object:Gem::Version
250
- version: '8.0'
251
- type: :development
252
- prerelease: false
253
- version_requirements: !ruby/object:Gem::Requirement
254
- requirements:
255
- - - "~>"
256
- - !ruby/object:Gem::Version
257
- version: '8.0'
258
- - !ruby/object:Gem::Dependency
259
- name: pry
260
- requirement: !ruby/object:Gem::Requirement
261
- requirements:
262
- - - "~>"
263
- - !ruby/object:Gem::Version
264
- version: '0.10'
265
- - - ">="
266
- - !ruby/object:Gem::Version
267
- version: 0.10.4
268
- type: :development
269
- prerelease: false
270
- version_requirements: !ruby/object:Gem::Requirement
271
- requirements:
272
- - - "~>"
273
- - !ruby/object:Gem::Version
274
- version: '0.10'
275
- - - ">="
276
- - !ruby/object:Gem::Version
277
- version: 0.10.4
278
- - !ruby/object:Gem::Dependency
279
- name: pry-byebug
280
- requirement: !ruby/object:Gem::Requirement
281
- requirements:
282
- - - "~>"
283
- - !ruby/object:Gem::Version
284
- version: '3.6'
285
- type: :development
286
- prerelease: false
287
- version_requirements: !ruby/object:Gem::Requirement
288
- requirements:
289
- - - "~>"
290
- - !ruby/object:Gem::Version
291
- version: '3.6'
292
- - !ruby/object:Gem::Dependency
293
- name: rake
294
- requirement: !ruby/object:Gem::Requirement
295
- requirements:
296
- - - "~>"
297
- - !ruby/object:Gem::Version
298
- version: '13.0'
299
- type: :development
300
- prerelease: false
301
- version_requirements: !ruby/object:Gem::Requirement
302
- requirements:
303
- - - "~>"
304
- - !ruby/object:Gem::Version
305
- version: '13.0'
306
- - !ruby/object:Gem::Dependency
307
- name: rspec
308
- requirement: !ruby/object:Gem::Requirement
309
- requirements:
310
- - - "~>"
311
- - !ruby/object:Gem::Version
312
- version: '3.0'
313
- type: :development
314
- prerelease: false
315
- version_requirements: !ruby/object:Gem::Requirement
316
- requirements:
317
- - - "~>"
318
- - !ruby/object:Gem::Version
319
- version: '3.0'
320
- - !ruby/object:Gem::Dependency
321
- name: rubocop
322
- requirement: !ruby/object:Gem::Requirement
323
- requirements:
324
- - - "~>"
325
- - !ruby/object:Gem::Version
326
- version: '1.23'
327
- type: :development
328
- prerelease: false
329
- version_requirements: !ruby/object:Gem::Requirement
330
- requirements:
331
- - - "~>"
332
- - !ruby/object:Gem::Version
333
- version: '1.23'
334
- - !ruby/object:Gem::Dependency
335
- name: rubocop-rspec
336
- requirement: !ruby/object:Gem::Requirement
337
- requirements:
338
- - - "~>"
339
- - !ruby/object:Gem::Version
340
- version: '2.6'
341
- type: :development
342
- prerelease: false
343
- version_requirements: !ruby/object:Gem::Requirement
344
- requirements:
345
- - - "~>"
346
- - !ruby/object:Gem::Version
347
- version: '2.6'
348
- - !ruby/object:Gem::Dependency
349
- name: timecop
350
- requirement: !ruby/object:Gem::Requirement
351
- requirements:
352
- - - "~>"
353
- - !ruby/object:Gem::Version
354
- version: '0.8'
355
- type: :development
356
- prerelease: false
357
- version_requirements: !ruby/object:Gem::Requirement
358
- requirements:
359
- - - "~>"
360
- - !ruby/object:Gem::Version
361
- version: '0.8'
362
- - !ruby/object:Gem::Dependency
363
- name: vcr
364
- requirement: !ruby/object:Gem::Requirement
365
- requirements:
366
- - - "~>"
367
- - !ruby/object:Gem::Version
368
- version: '5.0'
369
- type: :development
370
- prerelease: false
371
- version_requirements: !ruby/object:Gem::Requirement
372
- requirements:
373
- - - "~>"
374
- - !ruby/object:Gem::Version
375
- version: '5.0'
376
- - !ruby/object:Gem::Dependency
377
- name: webmock
378
- requirement: !ruby/object:Gem::Requirement
379
- requirements:
380
- - - "~>"
381
- - !ruby/object:Gem::Version
382
- version: '3.3'
383
- type: :development
384
- prerelease: false
385
- version_requirements: !ruby/object:Gem::Requirement
386
- requirements:
387
- - - "~>"
388
- - !ruby/object:Gem::Version
389
- version: '3.3'
390
230
  description:
391
231
  email:
392
232
  - murnagha@ualberta.ca
@@ -454,7 +294,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
454
294
  - !ruby/object:Gem::Version
455
295
  version: '0'
456
296
  requirements: []
457
- rubygems_version: 3.1.6
297
+ rubygems_version: 3.3.26
458
298
  signing_key:
459
299
  specification_version: 4
460
300
  summary: Ruby application to manage flow of content from Jupiter into Swift for preservation