pushmi_pullyu 2.0.2 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -1
- data/.github/workflows/ruby.yml +3 -4
- data/.gitignore +0 -1
- data/.rubocop.yml +3 -3
- data/CHANGELOG.md +19 -0
- data/Gemfile.lock +250 -0
- data/README.md +7 -2
- data/docker-compose.yml +11 -0
- data/examples/pushmi_pullyu.yml +0 -3
- data/lib/pushmi_pullyu/aip/downloader.rb +1 -3
- data/lib/pushmi_pullyu/aip.rb +12 -9
- data/lib/pushmi_pullyu/cli.rb +16 -9
- data/lib/pushmi_pullyu/logging.rb +67 -7
- data/lib/pushmi_pullyu/preservation_queue.rb +6 -0
- data/lib/pushmi_pullyu/version.rb +1 -1
- data/lib/pushmi_pullyu.rb +6 -10
- data/pushmi_pullyu.gemspec +2 -4
- metadata +7 -39
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: aabca31fdc4036e26f6fbd682b2e9aece8179973fc1509e7eada08b7bcdaf796
|
|
4
|
+
data.tar.gz: '0678d634ac98e9a4aeefa0bfb5d6df5a4869c0db3ba1b784be38a65caf916d75'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fb769198d6bf7e609fbbe7eec02498756f185e338439983a1ad14391392e9d3f66dd8368438791e237066137edd9bbf0afe355d6ef84cb61a11b5a00acc3e48a
|
|
7
|
+
data.tar.gz: b921457537461cd5512fec7ac072f301c701a98a40ee210236c39cb098b04b9c1206ca2b2a0c759ab6e9fce4f2a4768329ded84a104c1fd8ec51e74e4b7b395b
|
data/.github/CODEOWNERS
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# https://help.github.com/articles/about-codeowners/
|
|
2
|
-
* @ConnorSheremeta @lagoan @
|
|
2
|
+
* @ConnorSheremeta @lagoan @henryzhang87 @pgwillia @piyapongch
|
data/.github/workflows/ruby.yml
CHANGED
|
@@ -18,14 +18,13 @@ jobs:
|
|
|
18
18
|
- name: Set up Ruby
|
|
19
19
|
uses: ruby/setup-ruby@v1
|
|
20
20
|
with:
|
|
21
|
-
ruby-version: 2.
|
|
22
|
-
|
|
23
|
-
run: bundle install
|
|
21
|
+
ruby-version: '2.7'
|
|
22
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
|
24
23
|
- name: Lint with RuboCop
|
|
25
24
|
run: bundle exec rubocop --parallel
|
|
26
25
|
- name: Run Danger
|
|
27
26
|
env:
|
|
28
|
-
DANGER_GITHUB_API_TOKEN: ${{ secrets.
|
|
27
|
+
DANGER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
29
28
|
run: bundle exec danger
|
|
30
29
|
- name: Run tests
|
|
31
30
|
run: bundle exec rake spec
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -12,7 +12,7 @@ AllCops:
|
|
|
12
12
|
- 'vendor/**/*'
|
|
13
13
|
- 'Dangerfile'
|
|
14
14
|
ExtraDetails: true
|
|
15
|
-
TargetRubyVersion: 2.
|
|
15
|
+
TargetRubyVersion: 2.7
|
|
16
16
|
NewCops: enable
|
|
17
17
|
|
|
18
18
|
# readability is Actually Good
|
|
@@ -36,7 +36,7 @@ Metrics/ClassLength:
|
|
|
36
36
|
Metrics/CyclomaticComplexity:
|
|
37
37
|
Enabled: false
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
Layout/LineLength:
|
|
40
40
|
Enabled: true
|
|
41
41
|
Max: 120 # default is 80
|
|
42
42
|
|
|
@@ -107,4 +107,4 @@ Naming/VariableNumber:
|
|
|
107
107
|
Enabled: false
|
|
108
108
|
|
|
109
109
|
Style/OpenStructUse:
|
|
110
|
-
Enabled: false
|
|
110
|
+
Enabled: false
|
data/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,24 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
|
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
## [2.0.4] - 2022-11-22
|
|
12
|
+
|
|
13
|
+
- Fix issue with temporary work files not being deleted after a failed swift deposit [#242](https://github.com/ualbertalib/pushmi_pullyu/issues/242)
|
|
14
|
+
- Bump to Ruby 2.7
|
|
15
|
+
- Fix issue with entity information consumed even after failed deposit [#232](https://github.com/ualbertalib/pushmi_pullyu/issues/232)
|
|
16
|
+
- Bump rspec from 3.10.0 to 3.12.0
|
|
17
|
+
- Bump rollbar from 3.3.0 to 3.3.2
|
|
18
|
+
- Bump pry-byebug from 3.8.0 to 3.10.1
|
|
19
|
+
- Bump webmock from 3.14.0 to 3.18.1
|
|
20
|
+
- Bump rubocop-rspec from 2.6.0 to 2.11.1
|
|
21
|
+
- Bump timecop from 0.9.4 to 0.9.5
|
|
22
|
+
## [2.0.3] - 2022-04-28
|
|
23
|
+
|
|
24
|
+
- Changed Danger token in Github Actions
|
|
25
|
+
- remove pg dependency
|
|
26
|
+
- unblock CI (rubocop and coveralls)
|
|
27
|
+
- Log to preservation_events.json as well in an easy to use json format.
|
|
28
|
+
|
|
11
29
|
## [2.0.2] - 2021-11-22
|
|
12
30
|
|
|
13
31
|
- Fix authentication bug when using ssl
|
|
@@ -22,4 +40,5 @@ and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org
|
|
|
22
40
|
### Removed
|
|
23
41
|
- Data output for original_file information
|
|
24
42
|
|
|
43
|
+
-
|
|
25
44
|
## [1.0.6] - 2018-11-29
|
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
pushmi_pullyu (2.0.4)
|
|
5
|
+
activesupport (>= 5, < 8)
|
|
6
|
+
bagit (~> 0.4)
|
|
7
|
+
connection_pool (~> 2.2)
|
|
8
|
+
daemons (~> 1.2, >= 1.2.4)
|
|
9
|
+
minitar (~> 0.7)
|
|
10
|
+
openstack (~> 3.3, >= 3.3.10)
|
|
11
|
+
rdf (>= 1.99, < 4.0)
|
|
12
|
+
rdf-n3 (>= 1.99, < 4.0)
|
|
13
|
+
redis (>= 3.3, < 5.0)
|
|
14
|
+
rest-client (>= 1.8, < 3.0)
|
|
15
|
+
rollbar (>= 2.18, < 4.0)
|
|
16
|
+
uuid (~> 2.3.9)
|
|
17
|
+
|
|
18
|
+
GEM
|
|
19
|
+
remote: https://rubygems.org/
|
|
20
|
+
specs:
|
|
21
|
+
activesupport (7.0.2.4)
|
|
22
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
23
|
+
i18n (>= 1.6, < 2)
|
|
24
|
+
minitest (>= 5.1)
|
|
25
|
+
tzinfo (~> 2.0)
|
|
26
|
+
addressable (2.8.1)
|
|
27
|
+
public_suffix (>= 2.0.2, < 6.0)
|
|
28
|
+
amazing_print (1.4.0)
|
|
29
|
+
ast (2.4.2)
|
|
30
|
+
bagit (0.4.5)
|
|
31
|
+
docopt (~> 0.5.0)
|
|
32
|
+
validatable (~> 1.6)
|
|
33
|
+
builder (3.2.4)
|
|
34
|
+
byebug (11.1.3)
|
|
35
|
+
claide (1.0.3)
|
|
36
|
+
claide-plugins (0.9.2)
|
|
37
|
+
cork
|
|
38
|
+
nap
|
|
39
|
+
open4 (~> 1.3)
|
|
40
|
+
coderay (1.1.3)
|
|
41
|
+
colored2 (3.1.2)
|
|
42
|
+
concurrent-ruby (1.1.10)
|
|
43
|
+
connection_pool (2.2.5)
|
|
44
|
+
cork (0.3.0)
|
|
45
|
+
colored2 (~> 3.1)
|
|
46
|
+
crack (0.4.5)
|
|
47
|
+
rexml
|
|
48
|
+
daemons (1.4.1)
|
|
49
|
+
danger (8.4.2)
|
|
50
|
+
claide (~> 1.0)
|
|
51
|
+
claide-plugins (>= 0.9.2)
|
|
52
|
+
colored2 (~> 3.1)
|
|
53
|
+
cork (~> 0.1)
|
|
54
|
+
faraday (>= 0.9.0, < 2.0)
|
|
55
|
+
faraday-http-cache (~> 2.0)
|
|
56
|
+
git (~> 1.7)
|
|
57
|
+
kramdown (~> 2.3)
|
|
58
|
+
kramdown-parser-gfm (~> 1.0)
|
|
59
|
+
no_proxy_fix
|
|
60
|
+
octokit (~> 4.7)
|
|
61
|
+
terminal-table (>= 1, < 4)
|
|
62
|
+
diff-lcs (1.5.0)
|
|
63
|
+
docopt (0.5.0)
|
|
64
|
+
domain_name (0.5.20190701)
|
|
65
|
+
unf (>= 0.0.5, < 1.0.0)
|
|
66
|
+
ebnf (2.3.1)
|
|
67
|
+
amazing_print (~> 1.4)
|
|
68
|
+
htmlentities (~> 4.3)
|
|
69
|
+
rdf (~> 3.2)
|
|
70
|
+
scanf (~> 1.0)
|
|
71
|
+
sxp (~> 1.2)
|
|
72
|
+
unicode-types (~> 1.7)
|
|
73
|
+
faraday (1.8.0)
|
|
74
|
+
faraday-em_http (~> 1.0)
|
|
75
|
+
faraday-em_synchrony (~> 1.0)
|
|
76
|
+
faraday-excon (~> 1.1)
|
|
77
|
+
faraday-httpclient (~> 1.0.1)
|
|
78
|
+
faraday-net_http (~> 1.0)
|
|
79
|
+
faraday-net_http_persistent (~> 1.1)
|
|
80
|
+
faraday-patron (~> 1.0)
|
|
81
|
+
faraday-rack (~> 1.0)
|
|
82
|
+
multipart-post (>= 1.2, < 3)
|
|
83
|
+
ruby2_keywords (>= 0.0.4)
|
|
84
|
+
faraday-em_http (1.0.0)
|
|
85
|
+
faraday-em_synchrony (1.0.0)
|
|
86
|
+
faraday-excon (1.1.0)
|
|
87
|
+
faraday-http-cache (2.2.0)
|
|
88
|
+
faraday (>= 0.8)
|
|
89
|
+
faraday-httpclient (1.0.1)
|
|
90
|
+
faraday-net_http (1.0.1)
|
|
91
|
+
faraday-net_http_persistent (1.2.0)
|
|
92
|
+
faraday-patron (1.0.0)
|
|
93
|
+
faraday-rack (1.0.0)
|
|
94
|
+
git (1.9.1)
|
|
95
|
+
rchardet (~> 1.8)
|
|
96
|
+
hashdiff (1.0.1)
|
|
97
|
+
htmlentities (4.3.4)
|
|
98
|
+
http-accept (1.7.0)
|
|
99
|
+
http-cookie (1.0.4)
|
|
100
|
+
domain_name (~> 0.5)
|
|
101
|
+
i18n (1.10.0)
|
|
102
|
+
concurrent-ruby (~> 1.0)
|
|
103
|
+
json (2.6.1)
|
|
104
|
+
kramdown (2.3.1)
|
|
105
|
+
rexml
|
|
106
|
+
kramdown-parser-gfm (1.1.0)
|
|
107
|
+
kramdown (~> 2.0)
|
|
108
|
+
link_header (0.0.8)
|
|
109
|
+
logger (1.5.1)
|
|
110
|
+
macaddr (1.7.2)
|
|
111
|
+
systemu (~> 2.6.5)
|
|
112
|
+
matrix (0.4.2)
|
|
113
|
+
method_source (1.0.0)
|
|
114
|
+
mime-types (3.4.1)
|
|
115
|
+
mime-types-data (~> 3.2015)
|
|
116
|
+
mime-types-data (3.2022.0105)
|
|
117
|
+
minitar (0.9)
|
|
118
|
+
minitest (5.15.0)
|
|
119
|
+
multipart-post (2.1.1)
|
|
120
|
+
nap (1.1.0)
|
|
121
|
+
net-http-persistent (4.0.1)
|
|
122
|
+
connection_pool (~> 2.2)
|
|
123
|
+
netrc (0.11.0)
|
|
124
|
+
no_proxy_fix (0.1.2)
|
|
125
|
+
octokit (4.21.0)
|
|
126
|
+
faraday (>= 0.9)
|
|
127
|
+
sawyer (~> 0.8.0, >= 0.5.3)
|
|
128
|
+
open4 (1.3.4)
|
|
129
|
+
openstack (3.3.21)
|
|
130
|
+
json
|
|
131
|
+
parallel (1.22.1)
|
|
132
|
+
parser (3.1.2.1)
|
|
133
|
+
ast (~> 2.4.1)
|
|
134
|
+
pry (0.14.1)
|
|
135
|
+
coderay (~> 1.1)
|
|
136
|
+
method_source (~> 1.0)
|
|
137
|
+
pry-byebug (3.10.1)
|
|
138
|
+
byebug (~> 11.0)
|
|
139
|
+
pry (>= 0.13, < 0.15)
|
|
140
|
+
public_suffix (5.0.0)
|
|
141
|
+
rainbow (3.1.1)
|
|
142
|
+
rake (13.0.6)
|
|
143
|
+
rchardet (1.8.0)
|
|
144
|
+
rdf (3.2.7)
|
|
145
|
+
link_header (~> 0.0, >= 0.0.8)
|
|
146
|
+
rdf-aggregate-repo (3.2.0)
|
|
147
|
+
rdf (~> 3.2)
|
|
148
|
+
rdf-n3 (3.2.1)
|
|
149
|
+
ebnf (~> 2.2)
|
|
150
|
+
rdf (~> 3.2)
|
|
151
|
+
sparql (~> 3.2)
|
|
152
|
+
sxp (~> 1.2)
|
|
153
|
+
rdf-xsd (3.2.1)
|
|
154
|
+
rdf (~> 3.2)
|
|
155
|
+
rexml (~> 3.2)
|
|
156
|
+
redis (4.6.0)
|
|
157
|
+
regexp_parser (2.6.0)
|
|
158
|
+
rest-client (2.1.0)
|
|
159
|
+
http-accept (>= 1.7.0, < 2.0)
|
|
160
|
+
http-cookie (>= 1.0.2, < 2.0)
|
|
161
|
+
mime-types (>= 1.16, < 4.0)
|
|
162
|
+
netrc (~> 0.8)
|
|
163
|
+
rexml (3.2.5)
|
|
164
|
+
rollbar (3.3.0)
|
|
165
|
+
rspec (3.12.0)
|
|
166
|
+
rspec-core (~> 3.12.0)
|
|
167
|
+
rspec-expectations (~> 3.12.0)
|
|
168
|
+
rspec-mocks (~> 3.12.0)
|
|
169
|
+
rspec-core (3.12.0)
|
|
170
|
+
rspec-support (~> 3.12.0)
|
|
171
|
+
rspec-expectations (3.12.0)
|
|
172
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
173
|
+
rspec-support (~> 3.12.0)
|
|
174
|
+
rspec-mocks (3.12.0)
|
|
175
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
176
|
+
rspec-support (~> 3.12.0)
|
|
177
|
+
rspec-support (3.12.0)
|
|
178
|
+
rubocop (1.28.1)
|
|
179
|
+
parallel (~> 1.10)
|
|
180
|
+
parser (>= 3.1.0.0)
|
|
181
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
182
|
+
regexp_parser (>= 1.8, < 3.0)
|
|
183
|
+
rexml
|
|
184
|
+
rubocop-ast (>= 1.17.0, < 2.0)
|
|
185
|
+
ruby-progressbar (~> 1.7)
|
|
186
|
+
unicode-display_width (>= 1.4.0, < 3.0)
|
|
187
|
+
rubocop-ast (1.23.0)
|
|
188
|
+
parser (>= 3.1.1.0)
|
|
189
|
+
rubocop-rspec (2.11.1)
|
|
190
|
+
rubocop (~> 1.19)
|
|
191
|
+
ruby-progressbar (1.11.0)
|
|
192
|
+
ruby2_keywords (0.0.5)
|
|
193
|
+
sawyer (0.8.2)
|
|
194
|
+
addressable (>= 2.3.5)
|
|
195
|
+
faraday (> 0.8, < 2.0)
|
|
196
|
+
scanf (1.0.0)
|
|
197
|
+
sparql (3.2.1)
|
|
198
|
+
builder (~> 3.2)
|
|
199
|
+
ebnf (~> 2.2)
|
|
200
|
+
logger (~> 1.4)
|
|
201
|
+
rdf (~> 3.2, >= 3.2.3)
|
|
202
|
+
rdf-aggregate-repo (~> 3.2)
|
|
203
|
+
rdf-xsd (~> 3.2)
|
|
204
|
+
sparql-client (~> 3.2)
|
|
205
|
+
sxp (~> 1.2, >= 1.2.1)
|
|
206
|
+
sparql-client (3.2.1)
|
|
207
|
+
net-http-persistent (~> 4.0, >= 4.0.1)
|
|
208
|
+
rdf (~> 3.2, >= 3.2.6)
|
|
209
|
+
sxp (1.2.2)
|
|
210
|
+
matrix
|
|
211
|
+
rdf (~> 3.2)
|
|
212
|
+
systemu (2.6.5)
|
|
213
|
+
terminal-table (3.0.2)
|
|
214
|
+
unicode-display_width (>= 1.1.1, < 3)
|
|
215
|
+
timecop (0.9.5)
|
|
216
|
+
tzinfo (2.0.4)
|
|
217
|
+
concurrent-ruby (~> 1.0)
|
|
218
|
+
unf (0.1.4)
|
|
219
|
+
unf_ext
|
|
220
|
+
unf_ext (0.0.8.1)
|
|
221
|
+
unicode-display_width (2.3.0)
|
|
222
|
+
unicode-types (1.7.0)
|
|
223
|
+
uuid (2.3.9)
|
|
224
|
+
macaddr (~> 1.0)
|
|
225
|
+
validatable (1.6.7)
|
|
226
|
+
vcr (5.1.0)
|
|
227
|
+
webmock (3.18.1)
|
|
228
|
+
addressable (>= 2.8.0)
|
|
229
|
+
crack (>= 0.3.2)
|
|
230
|
+
hashdiff (>= 0.4.0, < 2.0.0)
|
|
231
|
+
|
|
232
|
+
PLATFORMS
|
|
233
|
+
x86_64-linux
|
|
234
|
+
|
|
235
|
+
DEPENDENCIES
|
|
236
|
+
bundler (~> 2.0)
|
|
237
|
+
danger (~> 8.0)
|
|
238
|
+
pry (~> 0.10, >= 0.10.4)
|
|
239
|
+
pry-byebug (~> 3.6)
|
|
240
|
+
pushmi_pullyu!
|
|
241
|
+
rake (~> 13.0)
|
|
242
|
+
rspec (~> 3.0)
|
|
243
|
+
rubocop (~> 1.23)
|
|
244
|
+
rubocop-rspec (~> 2.6)
|
|
245
|
+
timecop (~> 0.8)
|
|
246
|
+
vcr (~> 5.0)
|
|
247
|
+
webmock (~> 3.3)
|
|
248
|
+
|
|
249
|
+
BUNDLED WITH
|
|
250
|
+
2.3.12
|
data/README.md
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
|
|
7
7
|
[](https://rubygems.org/gems/pushmi_pullyu)
|
|
8
8
|
[](https://github.com/ualbertalib/pushmi_pullyu/actions)
|
|
9
|
-
[](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
|
|
10
9
|
|
|
11
10
|
PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
|
|
12
11
|
|
|
@@ -27,7 +26,7 @@ Its primary job is to manage the flow of content from Jupiter into Swift for pre
|
|
|
27
26
|
|
|
28
27
|
## Requirements
|
|
29
28
|
|
|
30
|
-
PushmiPullyu supports Ruby 2.
|
|
29
|
+
PushmiPullyu supports Ruby 2.7
|
|
31
30
|
|
|
32
31
|
## Installation
|
|
33
32
|
|
|
@@ -104,6 +103,12 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
|
104
103
|
|
|
105
104
|
## Testing
|
|
106
105
|
|
|
106
|
+
Pre-requisites:
|
|
107
|
+
|
|
108
|
+
Will need jupiter running locally and Openstack Swift running on port 8080.
|
|
109
|
+
- [Instructions for jupiter](https://github.com/ualbertalib/jupiter#docker)
|
|
110
|
+
- Use `docker-compose up -d` to start Swift (additional instructions in docker-compose.yml)
|
|
111
|
+
|
|
107
112
|
To run the test suite:
|
|
108
113
|
|
|
109
114
|
```bash
|
data/docker-compose.yml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
services:
|
|
2
|
+
swift:
|
|
3
|
+
# useful if you need to re-run the tests/spec
|
|
4
|
+
# https://docs.openstack.org/ocata/cli-reference/swift.html
|
|
5
|
+
# manual steps to set environment variables and create container
|
|
6
|
+
# swift auth -v -U test:tester -K testing -A http://localhost:8080/auth/v1.0
|
|
7
|
+
# swift auth
|
|
8
|
+
# swift post ERA
|
|
9
|
+
image: openstackswift/saio
|
|
10
|
+
ports:
|
|
11
|
+
- '8080:8080'
|
data/examples/pushmi_pullyu.yml
CHANGED
|
@@ -21,9 +21,6 @@ minimum_age: 0
|
|
|
21
21
|
redis:
|
|
22
22
|
url: redis://localhost:6379
|
|
23
23
|
|
|
24
|
-
database:
|
|
25
|
-
url: postgresql://jupiter:mysecretpassword@127.0.0.1/jupiter_development
|
|
26
|
-
|
|
27
24
|
#parameters project_name and project_domain_name are required only for keystone v3 authentication
|
|
28
25
|
swift:
|
|
29
26
|
tenant: tester
|
|
@@ -103,9 +103,7 @@ class PushmiPullyu::AIP::Downloader
|
|
|
103
103
|
|
|
104
104
|
response = @http.request(request)
|
|
105
105
|
is_success = if response.is_a?(Net::HTTPSuccess)
|
|
106
|
-
File.
|
|
107
|
-
file.write(response.body)
|
|
108
|
-
end
|
|
106
|
+
File.binwrite(local, response.body)
|
|
109
107
|
# Response was a success and the file was saved to local
|
|
110
108
|
File.exist? local
|
|
111
109
|
end
|
data/lib/pushmi_pullyu/aip.rb
CHANGED
|
@@ -6,19 +6,22 @@ module PushmiPullyu::AIP
|
|
|
6
6
|
module_function
|
|
7
7
|
|
|
8
8
|
def create(entity)
|
|
9
|
-
raise EntityInvalid if entity.
|
|
10
|
-
UUID.validate(entity[:uuid])
|
|
9
|
+
raise EntityInvalid if entity.blank? ||
|
|
10
|
+
UUID.validate(entity[:uuid]).blank? ||
|
|
11
11
|
entity[:type].blank?
|
|
12
12
|
|
|
13
13
|
aip_directory = "#{PushmiPullyu.options[:workdir]}/#{entity[:uuid]}"
|
|
14
14
|
aip_filename = "#{aip_directory}.tar"
|
|
15
|
+
begin
|
|
16
|
+
PushmiPullyu::AIP::Downloader.new(entity, aip_directory).run
|
|
17
|
+
PushmiPullyu::AIP::Creator.new(entity[:uuid], aip_directory, aip_filename).run
|
|
15
18
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
19
|
+
yield aip_filename, aip_directory
|
|
20
|
+
# Here we will ensure the files are removed even if an exception comes up.
|
|
21
|
+
# You will notice there is no rescue block. We will catch exceptions in `PushmiPullyu::CLI`
|
|
22
|
+
ensure
|
|
23
|
+
FileUtils.rm_rf(aip_filename)
|
|
24
|
+
FileUtils.rm_rf(aip_directory)
|
|
25
|
+
end
|
|
23
26
|
end
|
|
24
27
|
end
|
data/lib/pushmi_pullyu/cli.rb
CHANGED
|
@@ -182,11 +182,10 @@ class PushmiPullyu::CLI
|
|
|
182
182
|
end
|
|
183
183
|
|
|
184
184
|
def run_preservation_cycle
|
|
185
|
-
entity_json =
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
}
|
|
185
|
+
entity_json = queue.wait_next_item
|
|
186
|
+
# jupiter is submitting the entries to reddis in a hash format using fat arrows. We need to change them to colons in
|
|
187
|
+
# order to parse them correctly from json
|
|
188
|
+
entity = JSON.parse(entity_json.gsub('=>', ':'), { symbolize_names: true })
|
|
190
189
|
return unless entity[:type].present? && entity[:uuid].present?
|
|
191
190
|
|
|
192
191
|
# add additional information about the error context to errors that occur while processing this item.
|
|
@@ -199,13 +198,21 @@ class PushmiPullyu::CLI
|
|
|
199
198
|
# Log successful preservation event to the log files
|
|
200
199
|
PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
|
|
201
200
|
end
|
|
202
|
-
|
|
201
|
+
# An EntityInvalid expection means there is a problem with the entity information format so there is no point in
|
|
202
|
+
# readding it to the queue as it will always fail
|
|
203
|
+
rescue PushmiPullyu::AIP::EntityInvalid => e
|
|
204
|
+
rescue StandardError => e
|
|
205
|
+
queue.add_entity_json(entity_json)
|
|
206
|
+
|
|
207
|
+
# rubocop:disable Lint/RescueException
|
|
208
|
+
# Something other than a StandardError exception means something happened which we were not expecting!
|
|
209
|
+
# Make sure we log the problem
|
|
203
210
|
rescue Exception => e
|
|
211
|
+
raise e
|
|
212
|
+
# rubocop:enable Lint/RescueException
|
|
213
|
+
ensure
|
|
204
214
|
Rollbar.error(e)
|
|
205
215
|
logger.error(e)
|
|
206
|
-
# TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
|
|
207
|
-
# move on to the next item.
|
|
208
|
-
# rubocop:enable Lint/RescueException
|
|
209
216
|
end
|
|
210
217
|
end
|
|
211
218
|
|
|
@@ -45,13 +45,14 @@ module PushmiPullyu::Logging
|
|
|
45
45
|
|
|
46
46
|
def log_preservation_event(deposited_file, aip_directory)
|
|
47
47
|
preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
|
|
48
|
+
preservation_json_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.json")
|
|
48
49
|
|
|
49
|
-
message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
|
|
50
|
-
"Here are the details of this preservation event:\n"\
|
|
51
|
-
"\tUUID: '#{deposited_file.name}'\n"\
|
|
52
|
-
"\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
|
|
53
|
-
"\tAIP Checksum: '#{deposited_file.etag}'\n"\
|
|
54
|
-
"\tMetadata: #{deposited_file.metadata}\n"\
|
|
50
|
+
message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n" \
|
|
51
|
+
"Here are the details of this preservation event:\n" \
|
|
52
|
+
"\tUUID: '#{deposited_file.name}'\n" \
|
|
53
|
+
"\tTimestamp of Completion: '#{deposited_file.last_modified}'\n" \
|
|
54
|
+
"\tAIP Checksum: '#{deposited_file.etag}'\n" \
|
|
55
|
+
"\tMetadata: #{deposited_file.metadata}\n" \
|
|
55
56
|
|
|
56
57
|
file_details = file_log_details(aip_directory)
|
|
57
58
|
|
|
@@ -72,6 +73,65 @@ module PushmiPullyu::Logging
|
|
|
72
73
|
preservation_logger.info(message)
|
|
73
74
|
|
|
74
75
|
preservation_logger.close
|
|
76
|
+
|
|
77
|
+
message_json_str = preservation_event_to_json(deposited_file, aip_directory)
|
|
78
|
+
preservation_json_logger.info("#{message_json_str},")
|
|
79
|
+
preservation_json_logger.close
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
###
|
|
83
|
+
# Provides an alternative logging method in json format for the convenience of
|
|
84
|
+
# parsing in the process of auditing against OpenStack Swift preservation.
|
|
85
|
+
#
|
|
86
|
+
# output format:
|
|
87
|
+
# I, [2022-04-06T11:07:21.983875 #20791] INFO -- : \
|
|
88
|
+
# {
|
|
89
|
+
# "do_uuid": "83b5d21f-a60a-43ba-945a-f03deec64a1d",
|
|
90
|
+
# "aip_deposited_at": "Thu, 07 Apr 2022 16:37:00 GMT",
|
|
91
|
+
# "aip_md5sum": "fe5832a510799b04c1c503e46dc3b589",
|
|
92
|
+
# "aip_sha256": "",
|
|
93
|
+
# "aip_metadata": "{\"project-id\":\"83b5d21f-a60a-43ba-945a-f03deec64a1d\",
|
|
94
|
+
# \"aip-version\":\"1.0\",
|
|
95
|
+
# \"project\":\"ERA\",
|
|
96
|
+
# \"promise\":\"bronze\"}",
|
|
97
|
+
# "aip_file_details": [
|
|
98
|
+
# {
|
|
99
|
+
# "fileset_uuid": "b2c6ac0f-f2ed-489e-bbae-bd26465207aa",
|
|
100
|
+
# "file_name": "Spallacci_Amanda_202103_PhD.pdf",
|
|
101
|
+
# "file_type": "pdf",
|
|
102
|
+
# "file_size": "2051363"
|
|
103
|
+
# }
|
|
104
|
+
# ]
|
|
105
|
+
# }
|
|
106
|
+
#
|
|
107
|
+
# note:
|
|
108
|
+
# to parse, the prefix "I, ... INFO --:" in each line needs to be
|
|
109
|
+
# stripped using a bash command such as "sed"
|
|
110
|
+
def preservation_event_to_json(deposited_file, aip_directory)
|
|
111
|
+
message = {}
|
|
112
|
+
|
|
113
|
+
message['do_uuid'] = deposited_file.name.to_s
|
|
114
|
+
message['aip_deposited_at'] = deposited_file.last_modified.to_s
|
|
115
|
+
message['aip_md5sum'] = deposited_file.etag.to_s
|
|
116
|
+
message['aip_sha256'] = ''
|
|
117
|
+
message['aip_metadata'] = deposited_file.metadata.to_json.to_s
|
|
118
|
+
|
|
119
|
+
file_details = file_log_details(aip_directory)
|
|
120
|
+
|
|
121
|
+
tmp_details = []
|
|
122
|
+
if file_details.present?
|
|
123
|
+
file_details.each do |file_detail|
|
|
124
|
+
tmp_hash = {}
|
|
125
|
+
tmp_hash['fileset_uuid'] = file_detail[:fileset_name].to_s
|
|
126
|
+
tmp_hash['file_name'] = file_detail[:file_name].to_s
|
|
127
|
+
tmp_hash['file_type'] = file_detail[:file_extension].to_s
|
|
128
|
+
tmp_hash['file_size'] = file_detail[:file_size].to_s
|
|
129
|
+
tmp_details << tmp_hash
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
message['aip_file_details'] = tmp_details
|
|
134
|
+
message.to_json
|
|
75
135
|
end
|
|
76
136
|
|
|
77
137
|
def reopen
|
|
@@ -95,7 +155,7 @@ module PushmiPullyu::Logging
|
|
|
95
155
|
fileset_name: File.dirname(file).split('/')[-1],
|
|
96
156
|
file_name: File.basename(file),
|
|
97
157
|
file_size: File.size(file),
|
|
98
|
-
file_extension: File.extname(file).strip.downcase[1
|
|
158
|
+
file_extension: File.extname(file).strip.downcase[1..]
|
|
99
159
|
}
|
|
100
160
|
end
|
|
101
161
|
end
|
data/lib/pushmi_pullyu.rb
CHANGED
|
@@ -40,18 +40,14 @@ module PushmiPullyu
|
|
|
40
40
|
},
|
|
41
41
|
rollbar: {
|
|
42
42
|
},
|
|
43
|
-
|
|
44
|
-
encoding: 'utf8',
|
|
45
|
-
pool: ENV['RAILS_MAX_THREADS'] || 5,
|
|
46
|
-
url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
|
|
47
|
-
database: 'jupiter_development'
|
|
48
|
-
},
|
|
43
|
+
# rubocop disable: Style/FetchEnvVar
|
|
49
44
|
jupiter: {
|
|
50
|
-
user: ENV
|
|
51
|
-
api_key: ENV
|
|
52
|
-
jupiter_url: ENV
|
|
53
|
-
aip_api_path: ENV
|
|
45
|
+
user: ENV.fetch('JUPITER_USER', nil),
|
|
46
|
+
api_key: ENV.fetch('JUPITER_API_KEY', nil),
|
|
47
|
+
jupiter_url: ENV.fetch('JUPITER_URL', nil) || 'http://era.lvh.me:3000/',
|
|
48
|
+
aip_api_path: ENV.fetch('JUPITER_AIP_API_PATH', nil) || 'aip/v1'
|
|
54
49
|
}
|
|
50
|
+
# rubocop enable: Style/FetchEnvVar
|
|
55
51
|
}.freeze
|
|
56
52
|
|
|
57
53
|
def self.options
|
data/pushmi_pullyu.gemspec
CHANGED
|
@@ -19,15 +19,14 @@ Gem::Specification.new do |spec|
|
|
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
20
20
|
spec.require_paths = ['lib']
|
|
21
21
|
|
|
22
|
-
spec.required_ruby_version = '>= 2.
|
|
22
|
+
spec.required_ruby_version = '>= 2.7'
|
|
23
23
|
|
|
24
|
-
spec.add_runtime_dependency 'activesupport', '>= 5', '<
|
|
24
|
+
spec.add_runtime_dependency 'activesupport', '>= 5', '< 8'
|
|
25
25
|
spec.add_runtime_dependency 'bagit', '~> 0.4'
|
|
26
26
|
spec.add_runtime_dependency 'connection_pool', '~> 2.2'
|
|
27
27
|
spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
|
|
28
28
|
spec.add_runtime_dependency 'minitar', '~> 0.7'
|
|
29
29
|
spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
|
|
30
|
-
spec.add_runtime_dependency 'pg', '>= 1.0', '< 1.2'
|
|
31
30
|
spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
|
|
32
31
|
spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
|
|
33
32
|
spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
|
|
@@ -36,7 +35,6 @@ Gem::Specification.new do |spec|
|
|
|
36
35
|
spec.add_runtime_dependency 'uuid', '~> 2.3.9'
|
|
37
36
|
|
|
38
37
|
spec.add_development_dependency 'bundler', '~> 2.0'
|
|
39
|
-
spec.add_development_dependency 'coveralls', '~> 0.8'
|
|
40
38
|
spec.add_development_dependency 'danger', '~> 8.0'
|
|
41
39
|
spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
|
|
42
40
|
spec.add_development_dependency 'pry-byebug', '~> 3.6'
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pushmi_pullyu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.0.
|
|
4
|
+
version: 2.0.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shane Murnaghan
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2022-11-22 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: activesupport
|
|
@@ -20,7 +20,7 @@ dependencies:
|
|
|
20
20
|
version: '5'
|
|
21
21
|
- - "<"
|
|
22
22
|
- !ruby/object:Gem::Version
|
|
23
|
-
version: '
|
|
23
|
+
version: '8'
|
|
24
24
|
type: :runtime
|
|
25
25
|
prerelease: false
|
|
26
26
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -30,7 +30,7 @@ dependencies:
|
|
|
30
30
|
version: '5'
|
|
31
31
|
- - "<"
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
33
|
+
version: '8'
|
|
34
34
|
- !ruby/object:Gem::Dependency
|
|
35
35
|
name: bagit
|
|
36
36
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -113,26 +113,6 @@ dependencies:
|
|
|
113
113
|
- - ">="
|
|
114
114
|
- !ruby/object:Gem::Version
|
|
115
115
|
version: 3.3.10
|
|
116
|
-
- !ruby/object:Gem::Dependency
|
|
117
|
-
name: pg
|
|
118
|
-
requirement: !ruby/object:Gem::Requirement
|
|
119
|
-
requirements:
|
|
120
|
-
- - ">="
|
|
121
|
-
- !ruby/object:Gem::Version
|
|
122
|
-
version: '1.0'
|
|
123
|
-
- - "<"
|
|
124
|
-
- !ruby/object:Gem::Version
|
|
125
|
-
version: '1.2'
|
|
126
|
-
type: :runtime
|
|
127
|
-
prerelease: false
|
|
128
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
129
|
-
requirements:
|
|
130
|
-
- - ">="
|
|
131
|
-
- !ruby/object:Gem::Version
|
|
132
|
-
version: '1.0'
|
|
133
|
-
- - "<"
|
|
134
|
-
- !ruby/object:Gem::Version
|
|
135
|
-
version: '1.2'
|
|
136
116
|
- !ruby/object:Gem::Dependency
|
|
137
117
|
name: rdf
|
|
138
118
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -261,20 +241,6 @@ dependencies:
|
|
|
261
241
|
- - "~>"
|
|
262
242
|
- !ruby/object:Gem::Version
|
|
263
243
|
version: '2.0'
|
|
264
|
-
- !ruby/object:Gem::Dependency
|
|
265
|
-
name: coveralls
|
|
266
|
-
requirement: !ruby/object:Gem::Requirement
|
|
267
|
-
requirements:
|
|
268
|
-
- - "~>"
|
|
269
|
-
- !ruby/object:Gem::Version
|
|
270
|
-
version: '0.8'
|
|
271
|
-
type: :development
|
|
272
|
-
prerelease: false
|
|
273
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
274
|
-
requirements:
|
|
275
|
-
- - "~>"
|
|
276
|
-
- !ruby/object:Gem::Version
|
|
277
|
-
version: '0.8'
|
|
278
244
|
- !ruby/object:Gem::Dependency
|
|
279
245
|
name: danger
|
|
280
246
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -443,12 +409,14 @@ files:
|
|
|
443
409
|
- CHANGELOG.md
|
|
444
410
|
- Dangerfile
|
|
445
411
|
- Gemfile
|
|
412
|
+
- Gemfile.lock
|
|
446
413
|
- LICENSE.txt
|
|
447
414
|
- README.md
|
|
448
415
|
- Rakefile
|
|
449
416
|
- bin/console
|
|
450
417
|
- bin/setup
|
|
451
418
|
- config/.gitkeep
|
|
419
|
+
- docker-compose.yml
|
|
452
420
|
- docs/images/pushmi-pullyu.png
|
|
453
421
|
- docs/images/system-infrastructure-diagram.png
|
|
454
422
|
- examples/pushmi_pullyu.yml
|
|
@@ -479,7 +447,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
479
447
|
requirements:
|
|
480
448
|
- - ">="
|
|
481
449
|
- !ruby/object:Gem::Version
|
|
482
|
-
version: '2.
|
|
450
|
+
version: '2.7'
|
|
483
451
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
484
452
|
requirements:
|
|
485
453
|
- - ">="
|