pushmi_pullyu 2.0.2 → 2.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -1
- data/.github/workflows/ruby.yml +3 -4
- data/.gitignore +0 -1
- data/.rubocop.yml +3 -3
- data/CHANGELOG.md +19 -0
- data/Gemfile.lock +250 -0
- data/README.md +7 -2
- data/docker-compose.yml +11 -0
- data/examples/pushmi_pullyu.yml +0 -3
- data/lib/pushmi_pullyu/aip/downloader.rb +1 -3
- data/lib/pushmi_pullyu/aip.rb +12 -9
- data/lib/pushmi_pullyu/cli.rb +16 -9
- data/lib/pushmi_pullyu/logging.rb +67 -7
- data/lib/pushmi_pullyu/preservation_queue.rb +6 -0
- data/lib/pushmi_pullyu/version.rb +1 -1
- data/lib/pushmi_pullyu.rb +6 -10
- data/pushmi_pullyu.gemspec +2 -4
- metadata +7 -39
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: aabca31fdc4036e26f6fbd682b2e9aece8179973fc1509e7eada08b7bcdaf796
|
|
4
|
+
data.tar.gz: '0678d634ac98e9a4aeefa0bfb5d6df5a4869c0db3ba1b784be38a65caf916d75'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fb769198d6bf7e609fbbe7eec02498756f185e338439983a1ad14391392e9d3f66dd8368438791e237066137edd9bbf0afe355d6ef84cb61a11b5a00acc3e48a
|
|
7
|
+
data.tar.gz: b921457537461cd5512fec7ac072f301c701a98a40ee210236c39cb098b04b9c1206ca2b2a0c759ab6e9fce4f2a4768329ded84a104c1fd8ec51e74e4b7b395b
|
data/.github/CODEOWNERS
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# https://help.github.com/articles/about-codeowners/
|
|
2
|
-
* @ConnorSheremeta @lagoan @
|
|
2
|
+
* @ConnorSheremeta @lagoan @henryzhang87 @pgwillia @piyapongch
|
data/.github/workflows/ruby.yml
CHANGED
|
@@ -18,14 +18,13 @@ jobs:
|
|
|
18
18
|
- name: Set up Ruby
|
|
19
19
|
uses: ruby/setup-ruby@v1
|
|
20
20
|
with:
|
|
21
|
-
ruby-version: 2.
|
|
22
|
-
|
|
23
|
-
run: bundle install
|
|
21
|
+
ruby-version: '2.7'
|
|
22
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
|
24
23
|
- name: Lint with RuboCop
|
|
25
24
|
run: bundle exec rubocop --parallel
|
|
26
25
|
- name: Run Danger
|
|
27
26
|
env:
|
|
28
|
-
DANGER_GITHUB_API_TOKEN: ${{ secrets.
|
|
27
|
+
DANGER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
29
28
|
run: bundle exec danger
|
|
30
29
|
- name: Run tests
|
|
31
30
|
run: bundle exec rake spec
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -12,7 +12,7 @@ AllCops:
|
|
|
12
12
|
- 'vendor/**/*'
|
|
13
13
|
- 'Dangerfile'
|
|
14
14
|
ExtraDetails: true
|
|
15
|
-
TargetRubyVersion: 2.
|
|
15
|
+
TargetRubyVersion: 2.7
|
|
16
16
|
NewCops: enable
|
|
17
17
|
|
|
18
18
|
# readability is Actually Good
|
|
@@ -36,7 +36,7 @@ Metrics/ClassLength:
|
|
|
36
36
|
Metrics/CyclomaticComplexity:
|
|
37
37
|
Enabled: false
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
Layout/LineLength:
|
|
40
40
|
Enabled: true
|
|
41
41
|
Max: 120 # default is 80
|
|
42
42
|
|
|
@@ -107,4 +107,4 @@ Naming/VariableNumber:
|
|
|
107
107
|
Enabled: false
|
|
108
108
|
|
|
109
109
|
Style/OpenStructUse:
|
|
110
|
-
Enabled: false
|
|
110
|
+
Enabled: false
|
data/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,24 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
|
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
## [2.0.4] - 2022-11-22
|
|
12
|
+
|
|
13
|
+
- Fix issue with temporary work files not being deleted after a failed swift deposit [#242](https://github.com/ualbertalib/pushmi_pullyu/issues/242)
|
|
14
|
+
- Bump to Ruby 2.7
|
|
15
|
+
- Fix issue with entity information consumed even after failed deposit [#232](https://github.com/ualbertalib/pushmi_pullyu/issues/232)
|
|
16
|
+
- Bump rspec from 3.10.0 to 3.12.0
|
|
17
|
+
- Bump rollbar from 3.3.0 to 3.3.2
|
|
18
|
+
- Bump pry-byebug from 3.8.0 to 3.10.1
|
|
19
|
+
- Bump webmock from 3.14.0 to 3.18.1
|
|
20
|
+
- Bump rubocop-rspec from 2.6.0 to 2.11.1
|
|
21
|
+
- Bump timecop from 0.9.4 to 0.9.5
|
|
22
|
+
## [2.0.3] - 2022-04-28
|
|
23
|
+
|
|
24
|
+
- Changed Danger token in Github Actions
|
|
25
|
+
- remove pg dependency
|
|
26
|
+
- unblock CI (rubocop and coveralls)
|
|
27
|
+
- Log to preservation_events.json as well in an easy to use json format.
|
|
28
|
+
|
|
11
29
|
## [2.0.2] - 2021-11-22
|
|
12
30
|
|
|
13
31
|
- Fix authentication bug when using ssl
|
|
@@ -22,4 +40,5 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
|
|
|
22
40
|
### Removed
|
|
23
41
|
- Data output for original_file information
|
|
24
42
|
|
|
43
|
+
-
|
|
25
44
|
## [1.0.6] - 2018-11-29
|
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
pushmi_pullyu (2.0.4)
|
|
5
|
+
activesupport (>= 5, < 8)
|
|
6
|
+
bagit (~> 0.4)
|
|
7
|
+
connection_pool (~> 2.2)
|
|
8
|
+
daemons (~> 1.2, >= 1.2.4)
|
|
9
|
+
minitar (~> 0.7)
|
|
10
|
+
openstack (~> 3.3, >= 3.3.10)
|
|
11
|
+
rdf (>= 1.99, < 4.0)
|
|
12
|
+
rdf-n3 (>= 1.99, < 4.0)
|
|
13
|
+
redis (>= 3.3, < 5.0)
|
|
14
|
+
rest-client (>= 1.8, < 3.0)
|
|
15
|
+
rollbar (>= 2.18, < 4.0)
|
|
16
|
+
uuid (~> 2.3.9)
|
|
17
|
+
|
|
18
|
+
GEM
|
|
19
|
+
remote: https://rubygems.org/
|
|
20
|
+
specs:
|
|
21
|
+
activesupport (7.0.2.4)
|
|
22
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
23
|
+
i18n (>= 1.6, < 2)
|
|
24
|
+
minitest (>= 5.1)
|
|
25
|
+
tzinfo (~> 2.0)
|
|
26
|
+
addressable (2.8.1)
|
|
27
|
+
public_suffix (>= 2.0.2, < 6.0)
|
|
28
|
+
amazing_print (1.4.0)
|
|
29
|
+
ast (2.4.2)
|
|
30
|
+
bagit (0.4.5)
|
|
31
|
+
docopt (~> 0.5.0)
|
|
32
|
+
validatable (~> 1.6)
|
|
33
|
+
builder (3.2.4)
|
|
34
|
+
byebug (11.1.3)
|
|
35
|
+
claide (1.0.3)
|
|
36
|
+
claide-plugins (0.9.2)
|
|
37
|
+
cork
|
|
38
|
+
nap
|
|
39
|
+
open4 (~> 1.3)
|
|
40
|
+
coderay (1.1.3)
|
|
41
|
+
colored2 (3.1.2)
|
|
42
|
+
concurrent-ruby (1.1.10)
|
|
43
|
+
connection_pool (2.2.5)
|
|
44
|
+
cork (0.3.0)
|
|
45
|
+
colored2 (~> 3.1)
|
|
46
|
+
crack (0.4.5)
|
|
47
|
+
rexml
|
|
48
|
+
daemons (1.4.1)
|
|
49
|
+
danger (8.4.2)
|
|
50
|
+
claide (~> 1.0)
|
|
51
|
+
claide-plugins (>= 0.9.2)
|
|
52
|
+
colored2 (~> 3.1)
|
|
53
|
+
cork (~> 0.1)
|
|
54
|
+
faraday (>= 0.9.0, < 2.0)
|
|
55
|
+
faraday-http-cache (~> 2.0)
|
|
56
|
+
git (~> 1.7)
|
|
57
|
+
kramdown (~> 2.3)
|
|
58
|
+
kramdown-parser-gfm (~> 1.0)
|
|
59
|
+
no_proxy_fix
|
|
60
|
+
octokit (~> 4.7)
|
|
61
|
+
terminal-table (>= 1, < 4)
|
|
62
|
+
diff-lcs (1.5.0)
|
|
63
|
+
docopt (0.5.0)
|
|
64
|
+
domain_name (0.5.20190701)
|
|
65
|
+
unf (>= 0.0.5, < 1.0.0)
|
|
66
|
+
ebnf (2.3.1)
|
|
67
|
+
amazing_print (~> 1.4)
|
|
68
|
+
htmlentities (~> 4.3)
|
|
69
|
+
rdf (~> 3.2)
|
|
70
|
+
scanf (~> 1.0)
|
|
71
|
+
sxp (~> 1.2)
|
|
72
|
+
unicode-types (~> 1.7)
|
|
73
|
+
faraday (1.8.0)
|
|
74
|
+
faraday-em_http (~> 1.0)
|
|
75
|
+
faraday-em_synchrony (~> 1.0)
|
|
76
|
+
faraday-excon (~> 1.1)
|
|
77
|
+
faraday-httpclient (~> 1.0.1)
|
|
78
|
+
faraday-net_http (~> 1.0)
|
|
79
|
+
faraday-net_http_persistent (~> 1.1)
|
|
80
|
+
faraday-patron (~> 1.0)
|
|
81
|
+
faraday-rack (~> 1.0)
|
|
82
|
+
multipart-post (>= 1.2, < 3)
|
|
83
|
+
ruby2_keywords (>= 0.0.4)
|
|
84
|
+
faraday-em_http (1.0.0)
|
|
85
|
+
faraday-em_synchrony (1.0.0)
|
|
86
|
+
faraday-excon (1.1.0)
|
|
87
|
+
faraday-http-cache (2.2.0)
|
|
88
|
+
faraday (>= 0.8)
|
|
89
|
+
faraday-httpclient (1.0.1)
|
|
90
|
+
faraday-net_http (1.0.1)
|
|
91
|
+
faraday-net_http_persistent (1.2.0)
|
|
92
|
+
faraday-patron (1.0.0)
|
|
93
|
+
faraday-rack (1.0.0)
|
|
94
|
+
git (1.9.1)
|
|
95
|
+
rchardet (~> 1.8)
|
|
96
|
+
hashdiff (1.0.1)
|
|
97
|
+
htmlentities (4.3.4)
|
|
98
|
+
http-accept (1.7.0)
|
|
99
|
+
http-cookie (1.0.4)
|
|
100
|
+
domain_name (~> 0.5)
|
|
101
|
+
i18n (1.10.0)
|
|
102
|
+
concurrent-ruby (~> 1.0)
|
|
103
|
+
json (2.6.1)
|
|
104
|
+
kramdown (2.3.1)
|
|
105
|
+
rexml
|
|
106
|
+
kramdown-parser-gfm (1.1.0)
|
|
107
|
+
kramdown (~> 2.0)
|
|
108
|
+
link_header (0.0.8)
|
|
109
|
+
logger (1.5.1)
|
|
110
|
+
macaddr (1.7.2)
|
|
111
|
+
systemu (~> 2.6.5)
|
|
112
|
+
matrix (0.4.2)
|
|
113
|
+
method_source (1.0.0)
|
|
114
|
+
mime-types (3.4.1)
|
|
115
|
+
mime-types-data (~> 3.2015)
|
|
116
|
+
mime-types-data (3.2022.0105)
|
|
117
|
+
minitar (0.9)
|
|
118
|
+
minitest (5.15.0)
|
|
119
|
+
multipart-post (2.1.1)
|
|
120
|
+
nap (1.1.0)
|
|
121
|
+
net-http-persistent (4.0.1)
|
|
122
|
+
connection_pool (~> 2.2)
|
|
123
|
+
netrc (0.11.0)
|
|
124
|
+
no_proxy_fix (0.1.2)
|
|
125
|
+
octokit (4.21.0)
|
|
126
|
+
faraday (>= 0.9)
|
|
127
|
+
sawyer (~> 0.8.0, >= 0.5.3)
|
|
128
|
+
open4 (1.3.4)
|
|
129
|
+
openstack (3.3.21)
|
|
130
|
+
json
|
|
131
|
+
parallel (1.22.1)
|
|
132
|
+
parser (3.1.2.1)
|
|
133
|
+
ast (~> 2.4.1)
|
|
134
|
+
pry (0.14.1)
|
|
135
|
+
coderay (~> 1.1)
|
|
136
|
+
method_source (~> 1.0)
|
|
137
|
+
pry-byebug (3.10.1)
|
|
138
|
+
byebug (~> 11.0)
|
|
139
|
+
pry (>= 0.13, < 0.15)
|
|
140
|
+
public_suffix (5.0.0)
|
|
141
|
+
rainbow (3.1.1)
|
|
142
|
+
rake (13.0.6)
|
|
143
|
+
rchardet (1.8.0)
|
|
144
|
+
rdf (3.2.7)
|
|
145
|
+
link_header (~> 0.0, >= 0.0.8)
|
|
146
|
+
rdf-aggregate-repo (3.2.0)
|
|
147
|
+
rdf (~> 3.2)
|
|
148
|
+
rdf-n3 (3.2.1)
|
|
149
|
+
ebnf (~> 2.2)
|
|
150
|
+
rdf (~> 3.2)
|
|
151
|
+
sparql (~> 3.2)
|
|
152
|
+
sxp (~> 1.2)
|
|
153
|
+
rdf-xsd (3.2.1)
|
|
154
|
+
rdf (~> 3.2)
|
|
155
|
+
rexml (~> 3.2)
|
|
156
|
+
redis (4.6.0)
|
|
157
|
+
regexp_parser (2.6.0)
|
|
158
|
+
rest-client (2.1.0)
|
|
159
|
+
http-accept (>= 1.7.0, < 2.0)
|
|
160
|
+
http-cookie (>= 1.0.2, < 2.0)
|
|
161
|
+
mime-types (>= 1.16, < 4.0)
|
|
162
|
+
netrc (~> 0.8)
|
|
163
|
+
rexml (3.2.5)
|
|
164
|
+
rollbar (3.3.0)
|
|
165
|
+
rspec (3.12.0)
|
|
166
|
+
rspec-core (~> 3.12.0)
|
|
167
|
+
rspec-expectations (~> 3.12.0)
|
|
168
|
+
rspec-mocks (~> 3.12.0)
|
|
169
|
+
rspec-core (3.12.0)
|
|
170
|
+
rspec-support (~> 3.12.0)
|
|
171
|
+
rspec-expectations (3.12.0)
|
|
172
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
173
|
+
rspec-support (~> 3.12.0)
|
|
174
|
+
rspec-mocks (3.12.0)
|
|
175
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
176
|
+
rspec-support (~> 3.12.0)
|
|
177
|
+
rspec-support (3.12.0)
|
|
178
|
+
rubocop (1.28.1)
|
|
179
|
+
parallel (~> 1.10)
|
|
180
|
+
parser (>= 3.1.0.0)
|
|
181
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
182
|
+
regexp_parser (>= 1.8, < 3.0)
|
|
183
|
+
rexml
|
|
184
|
+
rubocop-ast (>= 1.17.0, < 2.0)
|
|
185
|
+
ruby-progressbar (~> 1.7)
|
|
186
|
+
unicode-display_width (>= 1.4.0, < 3.0)
|
|
187
|
+
rubocop-ast (1.23.0)
|
|
188
|
+
parser (>= 3.1.1.0)
|
|
189
|
+
rubocop-rspec (2.11.1)
|
|
190
|
+
rubocop (~> 1.19)
|
|
191
|
+
ruby-progressbar (1.11.0)
|
|
192
|
+
ruby2_keywords (0.0.5)
|
|
193
|
+
sawyer (0.8.2)
|
|
194
|
+
addressable (>= 2.3.5)
|
|
195
|
+
faraday (> 0.8, < 2.0)
|
|
196
|
+
scanf (1.0.0)
|
|
197
|
+
sparql (3.2.1)
|
|
198
|
+
builder (~> 3.2)
|
|
199
|
+
ebnf (~> 2.2)
|
|
200
|
+
logger (~> 1.4)
|
|
201
|
+
rdf (~> 3.2, >= 3.2.3)
|
|
202
|
+
rdf-aggregate-repo (~> 3.2)
|
|
203
|
+
rdf-xsd (~> 3.2)
|
|
204
|
+
sparql-client (~> 3.2)
|
|
205
|
+
sxp (~> 1.2, >= 1.2.1)
|
|
206
|
+
sparql-client (3.2.1)
|
|
207
|
+
net-http-persistent (~> 4.0, >= 4.0.1)
|
|
208
|
+
rdf (~> 3.2, >= 3.2.6)
|
|
209
|
+
sxp (1.2.2)
|
|
210
|
+
matrix
|
|
211
|
+
rdf (~> 3.2)
|
|
212
|
+
systemu (2.6.5)
|
|
213
|
+
terminal-table (3.0.2)
|
|
214
|
+
unicode-display_width (>= 1.1.1, < 3)
|
|
215
|
+
timecop (0.9.5)
|
|
216
|
+
tzinfo (2.0.4)
|
|
217
|
+
concurrent-ruby (~> 1.0)
|
|
218
|
+
unf (0.1.4)
|
|
219
|
+
unf_ext
|
|
220
|
+
unf_ext (0.0.8.1)
|
|
221
|
+
unicode-display_width (2.3.0)
|
|
222
|
+
unicode-types (1.7.0)
|
|
223
|
+
uuid (2.3.9)
|
|
224
|
+
macaddr (~> 1.0)
|
|
225
|
+
validatable (1.6.7)
|
|
226
|
+
vcr (5.1.0)
|
|
227
|
+
webmock (3.18.1)
|
|
228
|
+
addressable (>= 2.8.0)
|
|
229
|
+
crack (>= 0.3.2)
|
|
230
|
+
hashdiff (>= 0.4.0, < 2.0.0)
|
|
231
|
+
|
|
232
|
+
PLATFORMS
|
|
233
|
+
x86_64-linux
|
|
234
|
+
|
|
235
|
+
DEPENDENCIES
|
|
236
|
+
bundler (~> 2.0)
|
|
237
|
+
danger (~> 8.0)
|
|
238
|
+
pry (~> 0.10, >= 0.10.4)
|
|
239
|
+
pry-byebug (~> 3.6)
|
|
240
|
+
pushmi_pullyu!
|
|
241
|
+
rake (~> 13.0)
|
|
242
|
+
rspec (~> 3.0)
|
|
243
|
+
rubocop (~> 1.23)
|
|
244
|
+
rubocop-rspec (~> 2.6)
|
|
245
|
+
timecop (~> 0.8)
|
|
246
|
+
vcr (~> 5.0)
|
|
247
|
+
webmock (~> 3.3)
|
|
248
|
+
|
|
249
|
+
BUNDLED WITH
|
|
250
|
+
2.3.12
|
data/README.md
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
|
|
7
7
|
[](https://rubygems.org/gems/pushmi_pullyu)
|
|
8
8
|
[](https://github.com/ualbertalib/pushmi_pullyu/actions)
|
|
9
|
-
[](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
|
|
10
9
|
|
|
11
10
|
PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
|
|
12
11
|
|
|
@@ -27,7 +26,7 @@ Its primary job is to manage the flow of content from Jupiter into Swift for pre
|
|
|
27
26
|
|
|
28
27
|
## Requirements
|
|
29
28
|
|
|
30
|
-
PushmiPullyu supports Ruby 2.
|
|
29
|
+
PushmiPullyu supports Ruby 2.7
|
|
31
30
|
|
|
32
31
|
## Installation
|
|
33
32
|
|
|
@@ -104,6 +103,12 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
|
104
103
|
|
|
105
104
|
## Testing
|
|
106
105
|
|
|
106
|
+
Pre-requisites:
|
|
107
|
+
|
|
108
|
+
Will need jupiter running locally and Openstack Swift running on port 8080.
|
|
109
|
+
- [Instructions for jupiter](https://github.com/ualbertalib/jupiter#docker)
|
|
110
|
+
- Use `docker-compose up -d` to start Swift (additional instructions in docker-compose.yml)
|
|
111
|
+
|
|
107
112
|
To run the test suite:
|
|
108
113
|
|
|
109
114
|
```bash
|
data/docker-compose.yml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
services:
|
|
2
|
+
swift:
|
|
3
|
+
# useful if you need to re-run the tests/spec
|
|
4
|
+
# https://docs.openstack.org/ocata/cli-reference/swift.html
|
|
5
|
+
# manual steps to set environment variables and create container
|
|
6
|
+
# swift auth -v -U test:tester -K testing -A http://localhost:8080/auth/v1.0
|
|
7
|
+
# swift auth
|
|
8
|
+
# swift post ERA
|
|
9
|
+
image: openstackswift/saio
|
|
10
|
+
ports:
|
|
11
|
+
- '8080:8080'
|
data/examples/pushmi_pullyu.yml
CHANGED
|
@@ -21,9 +21,6 @@ minimum_age: 0
|
|
|
21
21
|
redis:
|
|
22
22
|
url: redis://localhost:6379
|
|
23
23
|
|
|
24
|
-
database:
|
|
25
|
-
url: postgresql://jupiter:mysecretpassword@127.0.0.1/jupiter_development
|
|
26
|
-
|
|
27
24
|
#parameters project_name and project_domain_name are required only for keystone v3 authentication
|
|
28
25
|
swift:
|
|
29
26
|
tenant: tester
|
|
@@ -103,9 +103,7 @@ class PushmiPullyu::AIP::Downloader
|
|
|
103
103
|
|
|
104
104
|
response = @http.request(request)
|
|
105
105
|
is_success = if response.is_a?(Net::HTTPSuccess)
|
|
106
|
-
File.
|
|
107
|
-
file.write(response.body)
|
|
108
|
-
end
|
|
106
|
+
File.binwrite(local, response.body)
|
|
109
107
|
# Response was a success and the file was saved to local
|
|
110
108
|
File.exist? local
|
|
111
109
|
end
|
data/lib/pushmi_pullyu/aip.rb
CHANGED
|
@@ -6,19 +6,22 @@ module PushmiPullyu::AIP
|
|
|
6
6
|
module_function
|
|
7
7
|
|
|
8
8
|
def create(entity)
|
|
9
|
-
raise EntityInvalid if entity.
|
|
10
|
-
UUID.validate(entity[:uuid])
|
|
9
|
+
raise EntityInvalid if entity.blank? ||
|
|
10
|
+
UUID.validate(entity[:uuid]).blank? ||
|
|
11
11
|
entity[:type].blank?
|
|
12
12
|
|
|
13
13
|
aip_directory = "#{PushmiPullyu.options[:workdir]}/#{entity[:uuid]}"
|
|
14
14
|
aip_filename = "#{aip_directory}.tar"
|
|
15
|
+
begin
|
|
16
|
+
PushmiPullyu::AIP::Downloader.new(entity, aip_directory).run
|
|
17
|
+
PushmiPullyu::AIP::Creator.new(entity[:uuid], aip_directory, aip_filename).run
|
|
15
18
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
19
|
+
yield aip_filename, aip_directory
|
|
20
|
+
# Here we will ensure the files are removed even if an exception comes up.
|
|
21
|
+
# You will notice there is no rescue block. We will catch exceptions in `PushmiPullyu::CLI`
|
|
22
|
+
ensure
|
|
23
|
+
FileUtils.rm_rf(aip_filename)
|
|
24
|
+
FileUtils.rm_rf(aip_directory)
|
|
25
|
+
end
|
|
23
26
|
end
|
|
24
27
|
end
|
data/lib/pushmi_pullyu/cli.rb
CHANGED
|
@@ -182,11 +182,10 @@ class PushmiPullyu::CLI
|
|
|
182
182
|
end
|
|
183
183
|
|
|
184
184
|
def run_preservation_cycle
|
|
185
|
-
entity_json =
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
}
|
|
185
|
+
entity_json = queue.wait_next_item
|
|
186
|
+
# jupiter is submitting the entries to reddis in a hash format using fat arrows. We need to change them to colons in
|
|
187
|
+
# order to parse them correctly from json
|
|
188
|
+
entity = JSON.parse(entity_json.gsub('=>', ':'), { symbolize_names: true })
|
|
190
189
|
return unless entity[:type].present? && entity[:uuid].present?
|
|
191
190
|
|
|
192
191
|
# add additional information about the error context to errors that occur while processing this item.
|
|
@@ -199,13 +198,21 @@ class PushmiPullyu::CLI
|
|
|
199
198
|
# Log successful preservation event to the log files
|
|
200
199
|
PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
|
|
201
200
|
end
|
|
202
|
-
|
|
201
|
+
# An EntityInvalid expection means there is a problem with the entity information format so there is no point in
|
|
202
|
+
# readding it to the queue as it will always fail
|
|
203
|
+
rescue PushmiPullyu::AIP::EntityInvalid => e
|
|
204
|
+
rescue StandardError => e
|
|
205
|
+
queue.add_entity_json(entity_json)
|
|
206
|
+
|
|
207
|
+
# rubocop:disable Lint/RescueException
|
|
208
|
+
# Something other than a StandardError exception means something happened which we were not expecting!
|
|
209
|
+
# Make sure we log the problem
|
|
203
210
|
rescue Exception => e
|
|
211
|
+
raise e
|
|
212
|
+
# rubocop:enable Lint/RescueException
|
|
213
|
+
ensure
|
|
204
214
|
Rollbar.error(e)
|
|
205
215
|
logger.error(e)
|
|
206
|
-
# TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
|
|
207
|
-
# move on to the next item.
|
|
208
|
-
# rubocop:enable Lint/RescueException
|
|
209
216
|
end
|
|
210
217
|
end
|
|
211
218
|
|
|
@@ -45,13 +45,14 @@ module PushmiPullyu::Logging
|
|
|
45
45
|
|
|
46
46
|
def log_preservation_event(deposited_file, aip_directory)
|
|
47
47
|
preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
|
|
48
|
+
preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
|
|
48
49
|
|
|
49
|
-
message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
|
|
50
|
-
"Here are the details of this preservation event:\n"\
|
|
51
|
-
"\tUUID: '#{deposited_file.name}'\n"\
|
|
52
|
-
"\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
|
|
53
|
-
"\tAIP Checksum: '#{deposited_file.etag}'\n"\
|
|
54
|
-
"\tMetadata: #{deposited_file.metadata}\n"\
|
|
50
|
+
message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n" \
|
|
51
|
+
"Here are the details of this preservation event:\n" \
|
|
52
|
+
"\tUUID: '#{deposited_file.name}'\n" \
|
|
53
|
+
"\tTimestamp of Completion: '#{deposited_file.last_modified}'\n" \
|
|
54
|
+
"\tAIP Checksum: '#{deposited_file.etag}'\n" \
|
|
55
|
+
"\tMetadata: #{deposited_file.metadata}\n" \
|
|
55
56
|
|
|
56
57
|
file_details = file_log_details(aip_directory)
|
|
57
58
|
|
|
@@ -72,6 +73,65 @@ module PushmiPullyu::Logging
|
|
|
72
73
|
preservation_logger.info(message)
|
|
73
74
|
|
|
74
75
|
preservation_logger.close
|
|
76
|
+
|
|
77
|
+
message_json_str = preservation_event_to_json(deposited_file, aip_directory)
|
|
78
|
+
preservation_json_logger.info("#{message_json_str},")
|
|
79
|
+
preservation_json_logger.close
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
###
|
|
83
|
+
# Provides an alternative logging method in json format for the convenience of
|
|
84
|
+
# parsing in the process of auditing against OpenStack Swift preservation.
|
|
85
|
+
#
|
|
86
|
+
# output format:
|
|
87
|
+
# I, [2022-04-06T11:07:21.983875 #20791] INFO -- : \
|
|
88
|
+
# {
|
|
89
|
+
# "do_uuid": "83b5d21f-a60a-43ba-945a-f03deec64a1d",
|
|
90
|
+
# "aip_deposited_at": "Thu, 07 Apr 2022 16:37:00 GMT",
|
|
91
|
+
# "aip_md5sum": "fe5832a510799b04c1c503e46dc3b589",
|
|
92
|
+
# "aip_sha256": "",
|
|
93
|
+
# "aip_metadata": "{\"project-id\":\"83b5d21f-a60a-43ba-945a-f03deec64a1d\",
|
|
94
|
+
# \"aip-version\":\"1.0\",
|
|
95
|
+
# \"project\":\"ERA\",
|
|
96
|
+
# \"promise\":\"bronze\"}",
|
|
97
|
+
# "aip_file_details": [
|
|
98
|
+
# {
|
|
99
|
+
# "fileset_uuid": "b2c6ac0f-f2ed-489e-bbae-bd26465207aa",
|
|
100
|
+
# "file_name": "Spallacci_Amanda_202103_PhD.pdf",
|
|
101
|
+
# "file_type": "pdf",
|
|
102
|
+
# "file_size": "2051363"
|
|
103
|
+
# }
|
|
104
|
+
# ]
|
|
105
|
+
# }
|
|
106
|
+
#
|
|
107
|
+
# note:
|
|
108
|
+
# to parse, the prefix "I, ... INFO --:" in each line needs to be
|
|
109
|
+
# stripped using a bash command such as "sed"
|
|
110
|
+
def preservation_event_to_json(deposited_file, aip_directory)
|
|
111
|
+
message = {}
|
|
112
|
+
|
|
113
|
+
message['do_uuid'] = deposited_file.name.to_s
|
|
114
|
+
message['aip_deposited_at'] = deposited_file.last_modified.to_s
|
|
115
|
+
message['aip_md5sum'] = deposited_file.etag.to_s
|
|
116
|
+
message['aip_sha256'] = ''
|
|
117
|
+
message['aip_metadata'] = deposited_file.metadata.to_json.to_s
|
|
118
|
+
|
|
119
|
+
file_details = file_log_details(aip_directory)
|
|
120
|
+
|
|
121
|
+
tmp_details = []
|
|
122
|
+
if file_details.present?
|
|
123
|
+
file_details.each do |file_detail|
|
|
124
|
+
tmp_hash = {}
|
|
125
|
+
tmp_hash['fileset_uuid'] = file_detail[:fileset_name].to_s
|
|
126
|
+
tmp_hash['file_name'] = file_detail[:file_name].to_s
|
|
127
|
+
tmp_hash['file_type'] = file_detail[:file_extension].to_s
|
|
128
|
+
tmp_hash['file_size'] = file_detail[:file_size].to_s
|
|
129
|
+
tmp_details << tmp_hash
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
message['aip_file_details'] = tmp_details
|
|
134
|
+
message.to_json
|
|
75
135
|
end
|
|
76
136
|
|
|
77
137
|
def reopen
|
|
@@ -95,7 +155,7 @@ module PushmiPullyu::Logging
|
|
|
95
155
|
fileset_name: File.dirname(file).split('/')[-1],
|
|
96
156
|
file_name: File.basename(file),
|
|
97
157
|
file_size: File.size(file),
|
|
98
|
-
file_extension: File.extname(file).strip.downcase[1
|
|
158
|
+
file_extension: File.extname(file).strip.downcase[1..]
|
|
99
159
|
}
|
|
100
160
|
end
|
|
101
161
|
end
|
data/lib/pushmi_pullyu.rb
CHANGED
|
@@ -40,18 +40,14 @@ module PushmiPullyu
|
|
|
40
40
|
},
|
|
41
41
|
rollbar: {
|
|
42
42
|
},
|
|
43
|
-
|
|
44
|
-
encoding: 'utf8',
|
|
45
|
-
pool: ENV['RAILS_MAX_THREADS'] || 5,
|
|
46
|
-
url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
|
|
47
|
-
database: 'jupiter_development'
|
|
48
|
-
},
|
|
43
|
+
# rubocop disable: Style/FetchEnvVar
|
|
49
44
|
jupiter: {
|
|
50
|
-
user: ENV
|
|
51
|
-
api_key: ENV
|
|
52
|
-
jupiter_url: ENV
|
|
53
|
-
aip_api_path: ENV
|
|
45
|
+
user: ENV.fetch('JUPITER_USER', nil),
|
|
46
|
+
api_key: ENV.fetch('JUPITER_API_KEY', nil),
|
|
47
|
+
jupiter_url: ENV.fetch('JUPITER_URL', nil) || 'http://era.lvh.me:3000/',
|
|
48
|
+
aip_api_path: ENV.fetch('JUPITER_AIP_API_PATH', nil) || 'aip/v1'
|
|
54
49
|
}
|
|
50
|
+
# rubocop enable: Style/FetchEnvVar
|
|
55
51
|
}.freeze
|
|
56
52
|
|
|
57
53
|
def self.options
|
data/pushmi_pullyu.gemspec
CHANGED
|
@@ -19,15 +19,14 @@ Gem::Specification.new do |spec|
|
|
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
20
20
|
spec.require_paths = ['lib']
|
|
21
21
|
|
|
22
|
-
spec.required_ruby_version = '>= 2.
|
|
22
|
+
spec.required_ruby_version = '>= 2.7'
|
|
23
23
|
|
|
24
|
-
spec.add_runtime_dependency 'activesupport', '>= 5', '<
|
|
24
|
+
spec.add_runtime_dependency 'activesupport', '>= 5', '< 8'
|
|
25
25
|
spec.add_runtime_dependency 'bagit', '~> 0.4'
|
|
26
26
|
spec.add_runtime_dependency 'connection_pool', '~> 2.2'
|
|
27
27
|
spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
|
|
28
28
|
spec.add_runtime_dependency 'minitar', '~> 0.7'
|
|
29
29
|
spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
|
|
30
|
-
spec.add_runtime_dependency 'pg', '>= 1.0', '< 1.2'
|
|
31
30
|
spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
|
|
32
31
|
spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
|
|
33
32
|
spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
|
|
@@ -36,7 +35,6 @@ Gem::Specification.new do |spec|
|
|
|
36
35
|
spec.add_runtime_dependency 'uuid', '~> 2.3.9'
|
|
37
36
|
|
|
38
37
|
spec.add_development_dependency 'bundler', '~> 2.0'
|
|
39
|
-
spec.add_development_dependency 'coveralls', '~> 0.8'
|
|
40
38
|
spec.add_development_dependency 'danger', '~> 8.0'
|
|
41
39
|
spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
|
|
42
40
|
spec.add_development_dependency 'pry-byebug', '~> 3.6'
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pushmi_pullyu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.0.
|
|
4
|
+
version: 2.0.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shane Murnaghan
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2022-11-22 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: activesupport
|
|
@@ -20,7 +20,7 @@ dependencies:
|
|
|
20
20
|
version: '5'
|
|
21
21
|
- - "<"
|
|
22
22
|
- !ruby/object:Gem::Version
|
|
23
|
-
version: '
|
|
23
|
+
version: '8'
|
|
24
24
|
type: :runtime
|
|
25
25
|
prerelease: false
|
|
26
26
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -30,7 +30,7 @@ dependencies:
|
|
|
30
30
|
version: '5'
|
|
31
31
|
- - "<"
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
33
|
+
version: '8'
|
|
34
34
|
- !ruby/object:Gem::Dependency
|
|
35
35
|
name: bagit
|
|
36
36
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -113,26 +113,6 @@ dependencies:
|
|
|
113
113
|
- - ">="
|
|
114
114
|
- !ruby/object:Gem::Version
|
|
115
115
|
version: 3.3.10
|
|
116
|
-
- !ruby/object:Gem::Dependency
|
|
117
|
-
name: pg
|
|
118
|
-
requirement: !ruby/object:Gem::Requirement
|
|
119
|
-
requirements:
|
|
120
|
-
- - ">="
|
|
121
|
-
- !ruby/object:Gem::Version
|
|
122
|
-
version: '1.0'
|
|
123
|
-
- - "<"
|
|
124
|
-
- !ruby/object:Gem::Version
|
|
125
|
-
version: '1.2'
|
|
126
|
-
type: :runtime
|
|
127
|
-
prerelease: false
|
|
128
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
129
|
-
requirements:
|
|
130
|
-
- - ">="
|
|
131
|
-
- !ruby/object:Gem::Version
|
|
132
|
-
version: '1.0'
|
|
133
|
-
- - "<"
|
|
134
|
-
- !ruby/object:Gem::Version
|
|
135
|
-
version: '1.2'
|
|
136
116
|
- !ruby/object:Gem::Dependency
|
|
137
117
|
name: rdf
|
|
138
118
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -261,20 +241,6 @@ dependencies:
|
|
|
261
241
|
- - "~>"
|
|
262
242
|
- !ruby/object:Gem::Version
|
|
263
243
|
version: '2.0'
|
|
264
|
-
- !ruby/object:Gem::Dependency
|
|
265
|
-
name: coveralls
|
|
266
|
-
requirement: !ruby/object:Gem::Requirement
|
|
267
|
-
requirements:
|
|
268
|
-
- - "~>"
|
|
269
|
-
- !ruby/object:Gem::Version
|
|
270
|
-
version: '0.8'
|
|
271
|
-
type: :development
|
|
272
|
-
prerelease: false
|
|
273
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
274
|
-
requirements:
|
|
275
|
-
- - "~>"
|
|
276
|
-
- !ruby/object:Gem::Version
|
|
277
|
-
version: '0.8'
|
|
278
244
|
- !ruby/object:Gem::Dependency
|
|
279
245
|
name: danger
|
|
280
246
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -443,12 +409,14 @@ files:
|
|
|
443
409
|
- CHANGELOG.md
|
|
444
410
|
- Dangerfile
|
|
445
411
|
- Gemfile
|
|
412
|
+
- Gemfile.lock
|
|
446
413
|
- LICENSE.txt
|
|
447
414
|
- README.md
|
|
448
415
|
- Rakefile
|
|
449
416
|
- bin/console
|
|
450
417
|
- bin/setup
|
|
451
418
|
- config/.gitkeep
|
|
419
|
+
- docker-compose.yml
|
|
452
420
|
- docs/images/pushmi-pullyu.png
|
|
453
421
|
- docs/images/system-infrastructure-diagram.png
|
|
454
422
|
- examples/pushmi_pullyu.yml
|
|
@@ -479,7 +447,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
479
447
|
requirements:
|
|
480
448
|
- - ">="
|
|
481
449
|
- !ruby/object:Gem::Version
|
|
482
|
-
version: '2.
|
|
450
|
+
version: '2.7'
|
|
483
451
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
484
452
|
requirements:
|
|
485
453
|
- - ">="
|