pushmi_pullyu 0.2.7 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.rubocop.yml +1 -1
- data/README.md +4 -4
- data/docs/images/system-infrastructure-diagram.png +0 -0
- data/examples/pushmi_pullyu.yml +8 -4
- data/lib/pushmi_pullyu/aip/creator.rb +5 -1
- data/lib/pushmi_pullyu/aip/downloader.rb +159 -75
- data/lib/pushmi_pullyu/aip/fedora_fetcher.rb +8 -2
- data/lib/pushmi_pullyu/aip/file_list_creator.rb +115 -0
- data/lib/pushmi_pullyu/aip/owner_email_editor.rb +62 -0
- data/lib/pushmi_pullyu/aip/user.rb +2 -0
- data/lib/pushmi_pullyu/aip.rb +2 -3
- data/lib/pushmi_pullyu/cli.rb +9 -9
- data/lib/pushmi_pullyu/logging.rb +78 -40
- data/lib/pushmi_pullyu/version.rb +1 -1
- data/lib/pushmi_pullyu.rb +12 -6
- data/pushmi_pullyu.gemspec +9 -7
- metadata +70 -16
- data/lib/pushmi_pullyu/aip/solr_fetcher.rb +0 -33
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 3c97d2575b16539392c9b268d62bf037ae289fc4989dfa976ca13796f01dec01
|
4
|
+
data.tar.gz: 2b277188e5873035ebc1c9a2bebdcf7e4837016091d2bc7b4236be90a5f74cef
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 36131cfed011b68a8f006d2c25968d25717919d4eb9b8016d58e65887fbe933ea61243c649bfc66b385353c527ab81ec0c78b14d226d45379f51bfd237ef5eaa
|
7
|
+
data.tar.gz: b39f6b537f3e4617efb7c778908788b038bdd7aea9f2198c34ab5276ec5ca33ddd3a90694ee548868a3b7739aecfdb50d08448477bde8c3a6beee8092d9fcf47
|
data/.rubocop.yml
CHANGED
data/README.md
CHANGED
@@ -17,11 +17,11 @@ Its primary job is to manage the flow of content from Fedora into Swift for pres
|
|
17
17
|
|
18
18
|
## Workflow
|
19
19
|
|
20
|
-
1. Any save (create or update) on a
|
21
|
-
2. The queue (Redis) is setup to be a unique set (which only allows one
|
20
|
+
1. Any save (create or update) on a Item/Thesis in ERA/Jupiter will trigger an after save callback that will push the item's unique identifier (UUID or NOID) into a Queue.
|
21
|
+
2. The queue (Redis) is setup to be a unique set (which only allows one item's UUID to be included in the queue at a single time), and ordered by priority from First In, First out (FIFO).
|
22
22
|
3. PushmiPullyu will then monitor the queue. After a certain wait period has passed since an element has been on the queue, PushmiPullyu will then retrieve the elements off the queue and begin to process the preservation event.
|
23
|
-
4. All the GenericFile information and data required for preservation are retrieved from Fedora
|
24
|
-
5. An Archival Information Package (AIP) is created from the
|
23
|
+
4. All the GenericFile information and data required for preservation are retrieved from Fedora using multiple REST calls. A database connection to the user database fetches (via ActiveRecord )owner emails and modifies the fetched documents, where applicable.
|
24
|
+
5. An Archival Information Package (AIP) is created from the item's information. It is then bagged and tarred.
|
25
25
|
6. The AIP tar is then uploaded to Swift via a REST call.
|
26
26
|
7. On a successful Swift upload, a entry is added for this preservation event to the preservation event logs.
|
27
27
|
|
Binary file
|
data/examples/pushmi_pullyu.yml
CHANGED
@@ -8,6 +8,7 @@
|
|
8
8
|
# PushmiPullyu will run this file through ERB when reading it so you can
|
9
9
|
# even put in dynamic logic, like consuming ENV Variables.
|
10
10
|
|
11
|
+
aip_version: 'lightaip-2.0'
|
11
12
|
debug: false
|
12
13
|
logdir: log
|
13
14
|
monitor: false
|
@@ -20,15 +21,18 @@ minimum_age: 0
|
|
20
21
|
redis:
|
21
22
|
url: redis://localhost:6379
|
22
23
|
|
23
|
-
solr:
|
24
|
-
url: http://localhost:8983/solr/development
|
25
|
-
|
26
24
|
fedora:
|
27
|
-
url: http://localhost:
|
25
|
+
url: http://localhost:8080/fcrepo/rest
|
28
26
|
user: fedoraAdmin
|
29
27
|
password: fedoraAdmin
|
30
28
|
base_path: /dev
|
31
29
|
|
30
|
+
database:
|
31
|
+
encoding: utf8
|
32
|
+
url: postgresql://jupiter:mysecretpassword@127.0.0.1
|
33
|
+
database: jupiter_development
|
34
|
+
pool: 5
|
35
|
+
|
32
36
|
#parameters project_name and project_domain_name are required only for keystone v3 authentication
|
33
37
|
swift:
|
34
38
|
tenant: tester
|
@@ -22,11 +22,15 @@ class PushmiPullyu::AIP::Creator
|
|
22
22
|
private
|
23
23
|
|
24
24
|
def bag_aip
|
25
|
-
bag = BagIt::Bag.new(@aip_directory)
|
25
|
+
bag = BagIt::Bag.new(@aip_directory, bag_metadata)
|
26
26
|
bag.manifest!
|
27
27
|
raise BagInvalid unless bag.valid?
|
28
28
|
end
|
29
29
|
|
30
|
+
def bag_metadata
|
31
|
+
{ 'AIP-Version' => PushmiPullyu.options[:aip_version] }
|
32
|
+
end
|
33
|
+
|
30
34
|
def tar_bag
|
31
35
|
# We want to change the directory to the work directory path so we get the tar file to be exactly
|
32
36
|
# the contents of the noid directory and not the entire work directory structure. For example the noid.tar
|
@@ -7,7 +7,18 @@ require 'rdf/n3'
|
|
7
7
|
# related to an object
|
8
8
|
class PushmiPullyu::AIP::Downloader
|
9
9
|
|
10
|
+
PREDICATE_URIS = {
|
11
|
+
filename: 'http://purl.org/dc/terms/title',
|
12
|
+
member_files: 'http://pcdm.org/models#hasFile',
|
13
|
+
member_file_sets: 'http://pcdm.org/models#hasMember',
|
14
|
+
original_file: 'http://pcdm.org/use#OriginalFile',
|
15
|
+
type: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
|
16
|
+
}.freeze
|
17
|
+
|
18
|
+
class NoFileSets < StandardError; end
|
19
|
+
class NoMemberFiles < StandardError; end
|
10
20
|
class NoContentFilename < StandardError; end
|
21
|
+
class NoOriginalFile < StandardError; end
|
11
22
|
|
12
23
|
def initialize(noid, aip_directory)
|
13
24
|
@noid = noid
|
@@ -19,21 +30,45 @@ class PushmiPullyu::AIP::Downloader
|
|
19
30
|
|
20
31
|
PushmiPullyu.logger.info("#{@noid}: Retreiving data from Fedora ...")
|
21
32
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
download_and_log(path_spec, PushmiPullyu::AIP::FedoraFetcher.new(@noid))
|
26
|
-
end
|
33
|
+
# Main object metadata
|
34
|
+
object_downloader = PushmiPullyu::AIP::FedoraFetcher.new(@noid)
|
35
|
+
download_and_log(object_aip_paths[:main_object], object_downloader)
|
27
36
|
|
28
|
-
#
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
37
|
+
# Construct the file ordering file
|
38
|
+
list_source_uri = object_downloader.object_url + object_aip_paths.list_source.remote
|
39
|
+
create_and_log_file_order_list(list_source_uri)
|
40
|
+
|
41
|
+
member_file_set_uuids.each do |file_set_uuid|
|
42
|
+
make_file_set_directories(file_set_uuid)
|
43
|
+
|
44
|
+
# FileSet metadata
|
45
|
+
file_set_downloader = PushmiPullyu::AIP::FedoraFetcher.new(file_set_uuid)
|
46
|
+
path_spec = file_set_aip_paths(file_set_uuid)[:main_object]
|
47
|
+
download_and_log(path_spec, file_set_downloader)
|
48
|
+
|
49
|
+
# Find the original file by looping through the files in the file_set
|
50
|
+
original_file_remote_base = nil
|
51
|
+
member_files(file_set_uuid).each do |file_path|
|
52
|
+
path_spec = OpenStruct.new(
|
53
|
+
remote: "/files/#{file_path}/fcr:metadata",
|
54
|
+
# Note: local file gets clobbered on each download until it finds the right one
|
55
|
+
local: "#{file_set_dirs(file_set_uuid).metadata}/original_file_metadata.n3",
|
56
|
+
optional: true
|
57
|
+
)
|
58
|
+
download_and_log(path_spec, file_set_downloader)
|
59
|
+
if original_file?(path_spec.local)
|
60
|
+
original_file_remote_base = "/files/#{file_path}"
|
61
|
+
break
|
62
|
+
end
|
63
|
+
end
|
35
64
|
|
36
|
-
|
65
|
+
raise NoOriginalFile unless original_file_remote_base.present?
|
66
|
+
|
67
|
+
[:content, :fixity].each do |item|
|
68
|
+
path_spec = file_aip_paths(file_set_uuid, original_file_remote_base)[item]
|
69
|
+
download_and_log(path_spec, file_set_downloader)
|
70
|
+
end
|
71
|
+
end
|
37
72
|
end
|
38
73
|
|
39
74
|
private
|
@@ -43,35 +78,24 @@ class PushmiPullyu::AIP::Downloader
|
|
43
78
|
|
44
79
|
log_fetching(fedora_fetcher.object_url(path_spec.remote), output_file)
|
45
80
|
|
46
|
-
is_rdf = (output_file
|
81
|
+
is_rdf = (output_file =~ /\.n3$/)
|
82
|
+
should_add_user_email = path_spec.to_h.fetch(:should_add_user_email, false)
|
47
83
|
|
48
84
|
is_success = fedora_fetcher.download_object(output_file,
|
49
85
|
url_extra: path_spec.remote,
|
50
86
|
optional: path_spec.optional,
|
51
|
-
is_rdf: is_rdf
|
87
|
+
is_rdf: is_rdf,
|
88
|
+
should_add_user_email: should_add_user_email)
|
52
89
|
log_saved(is_success, output_file)
|
53
90
|
end
|
54
91
|
|
55
|
-
def
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
PushmiPullyu.logger.info("#{@noid}: permission object #{permission_id} found")
|
63
|
-
download_permission(permission_id)
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
def download_permission(permission_id)
|
69
|
-
path_spec = OpenStruct.new(
|
70
|
-
remote: nil,
|
71
|
-
local: "#{aip_dirs.metadata}/permission_#{permission_id}.n3",
|
72
|
-
optional: false
|
73
|
-
)
|
74
|
-
download_and_log(path_spec, PushmiPullyu::AIP::FedoraFetcher.new(permission_id))
|
92
|
+
def create_and_log_file_order_list(url)
|
93
|
+
output_file = object_aip_paths.file_ordering.local
|
94
|
+
PushmiPullyu::Logging.log_aip_activity(@aip_directory,
|
95
|
+
"#{@noid}: #{output_file} -- creating from #{url} ...")
|
96
|
+
PushmiPullyu::AIP::FileListCreator.new(url, output_file, member_file_set_uuids).run
|
97
|
+
PushmiPullyu::Logging.log_aip_activity(@aip_directory,
|
98
|
+
"#{@noid}: #{output_file} -- created")
|
75
99
|
end
|
76
100
|
|
77
101
|
### Logging
|
@@ -92,8 +116,19 @@ class PushmiPullyu::AIP::Downloader
|
|
92
116
|
@aip_dirs ||= OpenStruct.new(
|
93
117
|
objects: "#{@aip_directory}/data/objects",
|
94
118
|
metadata: "#{@aip_directory}/data/objects/metadata",
|
119
|
+
files: "#{@aip_directory}/data/objects/files",
|
120
|
+
files_metadata: "#{@aip_directory}/data/objects/metadata/files_metadata",
|
95
121
|
logs: "#{@aip_directory}/data/logs",
|
96
|
-
|
122
|
+
file_logs: "#{@aip_directory}/data/logs/files_logs"
|
123
|
+
)
|
124
|
+
end
|
125
|
+
|
126
|
+
def file_set_dirs(file_set_uuid)
|
127
|
+
@file_set_dirs ||= {}
|
128
|
+
@file_set_dirs[file_set_uuid] ||= OpenStruct.new(
|
129
|
+
metadata: "#{aip_dirs.files_metadata}/#{file_set_uuid}",
|
130
|
+
files: "#{aip_dirs.files}/#{file_set_uuid}",
|
131
|
+
logs: "#{aip_dirs.file_logs}/#{file_set_uuid}"
|
97
132
|
)
|
98
133
|
end
|
99
134
|
|
@@ -106,6 +141,14 @@ class PushmiPullyu::AIP::Downloader
|
|
106
141
|
PushmiPullyu.logger.debug("#{@noid}: Creating directories done")
|
107
142
|
end
|
108
143
|
|
144
|
+
def make_file_set_directories(file_set_uuid)
|
145
|
+
PushmiPullyu.logger.debug("#{@noid}: Creating file set #{file_set_uuid} directories ...")
|
146
|
+
file_set_dirs(file_set_uuid).to_h.each_value do |path|
|
147
|
+
FileUtils.mkdir_p(path)
|
148
|
+
end
|
149
|
+
PushmiPullyu.logger.debug("#{@noid}: Creating file set #{file_set_uuid} directories done")
|
150
|
+
end
|
151
|
+
|
109
152
|
def clean_directories
|
110
153
|
return unless File.exist?(@aip_directory)
|
111
154
|
PushmiPullyu.logger.debug("#{@noid}: Nuking directories ...")
|
@@ -114,64 +157,105 @@ class PushmiPullyu::AIP::Downloader
|
|
114
157
|
|
115
158
|
### Files
|
116
159
|
|
117
|
-
def
|
118
|
-
@
|
160
|
+
def object_aip_paths
|
161
|
+
@object_aip_paths ||= OpenStruct.new(
|
119
162
|
main_object: OpenStruct.new(
|
120
163
|
remote: nil, # Base path
|
121
164
|
local: "#{aip_dirs.metadata}/object_metadata.n3",
|
165
|
+
should_add_user_email: true,
|
122
166
|
optional: false
|
123
167
|
),
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
optional: false
|
168
|
+
list_source: OpenStruct.new(
|
169
|
+
# This is downloaded, but not saved
|
170
|
+
remote: '/list_source'
|
128
171
|
),
|
129
|
-
|
130
|
-
|
131
|
-
local: "#{aip_dirs.
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
172
|
+
# This is constructed, not downloaded
|
173
|
+
file_ordering: OpenStruct.new(
|
174
|
+
local: "#{aip_dirs.files_metadata}/file_order.xml"
|
175
|
+
)
|
176
|
+
).freeze
|
177
|
+
end
|
178
|
+
|
179
|
+
def file_set_aip_paths(file_set_uuid)
|
180
|
+
@file_set_aip_paths ||= {}
|
181
|
+
@file_set_aip_paths[file_set_uuid] ||= OpenStruct.new(
|
182
|
+
main_object: OpenStruct.new(
|
183
|
+
remote: nil, # Base file_set path
|
184
|
+
local: "#{file_set_dirs(file_set_uuid).metadata}/file_set_metadata.n3",
|
185
|
+
should_add_user_email: true,
|
137
186
|
optional: false
|
138
|
-
)
|
187
|
+
)
|
188
|
+
).freeze
|
189
|
+
end
|
139
190
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
remote: '/characterization',
|
148
|
-
local: "#{aip_dirs.logs}/content_characterization.n3",
|
149
|
-
optional: true
|
150
|
-
),
|
151
|
-
fedora3foxml: OpenStruct.new(
|
152
|
-
remote: '/fedora3foxml',
|
153
|
-
local: "#{aip_dirs.metadata}/fedora3foxml.xml",
|
154
|
-
optional: true
|
191
|
+
def file_aip_paths(file_set_uuid, original_file_remote_base)
|
192
|
+
@file_aip_paths ||= {}
|
193
|
+
@file_aip_paths[file_set_uuid] ||= OpenStruct.new(
|
194
|
+
content: OpenStruct.new(
|
195
|
+
remote: original_file_remote_base,
|
196
|
+
local: file_set_filename(file_set_uuid),
|
197
|
+
optional: false
|
155
198
|
),
|
156
|
-
|
157
|
-
remote:
|
158
|
-
local: "#{
|
159
|
-
optional:
|
199
|
+
fixity: OpenStruct.new(
|
200
|
+
remote: "#{original_file_remote_base}/fcr:fixity",
|
201
|
+
local: "#{file_set_dirs(file_set_uuid)[:logs]}/content_fixity_report.n3",
|
202
|
+
optional: false
|
160
203
|
)
|
161
204
|
).freeze
|
162
205
|
end
|
163
206
|
|
164
|
-
|
165
|
-
|
166
|
-
|
207
|
+
def member_file_set_uuids
|
208
|
+
@member_file_set_uuids ||= []
|
209
|
+
return @member_file_set_uuids unless @member_file_set_uuids.empty?
|
210
|
+
|
211
|
+
member_file_set_predicate = RDF::URI(PREDICATE_URIS[:member_file_sets])
|
212
|
+
|
213
|
+
graph = RDF::Graph.load(object_aip_paths.main_object.local)
|
214
|
+
|
215
|
+
graph.query(predicate: member_file_set_predicate) do |results|
|
216
|
+
# Get uuid from end of fedora path
|
217
|
+
@member_file_set_uuids << results.object.to_s.split('/').last
|
218
|
+
end
|
219
|
+
return @member_file_set_uuids unless @member_file_set_uuids.empty?
|
220
|
+
|
221
|
+
raise NoFileSets
|
222
|
+
end
|
223
|
+
|
224
|
+
def file_set_filename(file_set_uuid)
|
225
|
+
filename_predicate = RDF::URI(PREDICATE_URIS[:filename])
|
167
226
|
|
168
|
-
graph = RDF::Graph.load(
|
227
|
+
graph = RDF::Graph.load(file_set_aip_paths(file_set_uuid).main_object.local)
|
169
228
|
|
170
229
|
graph.query(predicate: filename_predicate) do |results|
|
171
|
-
return "#{
|
230
|
+
return "#{file_set_dirs(file_set_uuid).files}/#{results.object}"
|
172
231
|
end
|
173
232
|
|
174
233
|
raise NoContentFilename
|
175
234
|
end
|
176
235
|
|
236
|
+
def member_files(file_set_uuid)
|
237
|
+
member_file_predicate = RDF::URI(PREDICATE_URIS[:member_files])
|
238
|
+
|
239
|
+
graph = RDF::Graph.load(file_set_aip_paths(file_set_uuid).main_object.local)
|
240
|
+
|
241
|
+
member_files = []
|
242
|
+
graph.query(predicate: member_file_predicate) do |results|
|
243
|
+
# Get uuid from end of fedora path
|
244
|
+
member_files << results.object.to_s.split('/').last
|
245
|
+
end
|
246
|
+
return member_files if member_files.present?
|
247
|
+
|
248
|
+
raise NoMemberFiles
|
249
|
+
end
|
250
|
+
|
251
|
+
def original_file?(metadata_filename)
|
252
|
+
type_predicate = RDF::URI(PREDICATE_URIS[:type])
|
253
|
+
original_file_uri = RDF::URI(PREDICATE_URIS[:original_file])
|
254
|
+
graph = RDF::Graph.load(metadata_filename)
|
255
|
+
graph.query(predicate: type_predicate) do |results|
|
256
|
+
return true if results.object == original_file_uri
|
257
|
+
end
|
258
|
+
false
|
259
|
+
end
|
260
|
+
|
177
261
|
end
|
@@ -19,7 +19,8 @@ class PushmiPullyu::AIP::FedoraFetcher
|
|
19
19
|
# Return true on success, raise an error otherwise
|
20
20
|
# (or use 'optional' to return false on 404)
|
21
21
|
def download_object(download_path, url_extra: nil,
|
22
|
-
optional: false, is_rdf: false
|
22
|
+
optional: false, is_rdf: false,
|
23
|
+
should_add_user_email: false)
|
23
24
|
|
24
25
|
uri = URI(object_url(url_extra))
|
25
26
|
|
@@ -34,8 +35,13 @@ class PushmiPullyu::AIP::FedoraFetcher
|
|
34
35
|
end
|
35
36
|
|
36
37
|
if response.is_a?(Net::HTTPSuccess)
|
38
|
+
body = if should_add_user_email
|
39
|
+
PushmiPullyu::AIP::OwnerEmailEditor.new(response.body).run
|
40
|
+
else
|
41
|
+
response.body
|
42
|
+
end
|
37
43
|
file = File.open(download_path, 'wb')
|
38
|
-
file.write(
|
44
|
+
file.write(body)
|
39
45
|
file.close
|
40
46
|
return true
|
41
47
|
elsif response.is_a?(Net::HTTPNotFound)
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'rdf'
|
2
|
+
require 'rdf/n3'
|
3
|
+
|
4
|
+
class PushmiPullyu::AIP::FileListCreator
|
5
|
+
|
6
|
+
IANA = 'http://www.iana.org/assignments/relation/'.freeze
|
7
|
+
PREDICATES = {
|
8
|
+
proxy_for: RDF::URI('http://www.openarchives.org/ore/terms/proxyFor'),
|
9
|
+
first: RDF::URI(IANA + 'first'),
|
10
|
+
last: RDF::URI(IANA + 'last'),
|
11
|
+
prev: RDF::URI(IANA + 'prev'),
|
12
|
+
next: RDF::URI(IANA + 'next'),
|
13
|
+
has_part: RDF::URI('http://purl.org/dc/terms/hasPart')
|
14
|
+
}.freeze
|
15
|
+
|
16
|
+
class NoProxyURIFound < StandardError; end
|
17
|
+
class NoFirstProxyFound < StandardError; end
|
18
|
+
class FirstProxyHasPrev < StandardError; end
|
19
|
+
class ListSourceFileSetMismatch < StandardError; end
|
20
|
+
|
21
|
+
def initialize(list_source_uri, output_xml_file, file_set_uuids)
|
22
|
+
@uri = RDF::URI(list_source_uri)
|
23
|
+
@output_file = output_xml_file
|
24
|
+
|
25
|
+
# These are the known fileset uuids, used for validation
|
26
|
+
@file_set_uuids = file_set_uuids
|
27
|
+
end
|
28
|
+
|
29
|
+
def run
|
30
|
+
extract_list_source_uuids
|
31
|
+
raise ListSourceFileSetMismatch, @uri.to_s if @list_source_uuids.sort != @file_set_uuids.sort
|
32
|
+
|
33
|
+
write_output_file
|
34
|
+
end
|
35
|
+
|
36
|
+
def extract_list_source_uuids
|
37
|
+
# Note: raises IOError if can't find
|
38
|
+
# raises RDF::ReaderError if can't parse
|
39
|
+
@graph = RDF::Graph.load(@uri, validate: true)
|
40
|
+
|
41
|
+
@list_source_uuids = []
|
42
|
+
|
43
|
+
# Fetch first FileSet in list source
|
44
|
+
this_proxy = find_first_proxy
|
45
|
+
|
46
|
+
while @list_source_uuids.count <= num_proxies
|
47
|
+
@list_source_uuids << uuid_from_proxy(this_proxy)
|
48
|
+
next_proxy = find_next_proxy(this_proxy)
|
49
|
+
|
50
|
+
break if next_proxy.nil?
|
51
|
+
|
52
|
+
raise NextPreviousProxyMismatch if this_proxy != find_prev_proxy(next_proxy)
|
53
|
+
this_proxy = next_proxy
|
54
|
+
end
|
55
|
+
|
56
|
+
raise ProxyCountIncorrect if @list_source_uuids.count != num_proxies
|
57
|
+
raise LastProxyFailsValidation if this_proxy != find_last_proxy
|
58
|
+
end
|
59
|
+
|
60
|
+
def num_proxies
|
61
|
+
@num_proxies ||= @graph.query(subject: @uri, predicate: PREDICATES[:has_part]).count
|
62
|
+
end
|
63
|
+
|
64
|
+
def uuid_from_proxy(proxy_uri)
|
65
|
+
@graph.query(subject: proxy_uri, predicate: PREDICATES[:proxy_for]) do |statement|
|
66
|
+
return statement.object.to_s.split('/').last
|
67
|
+
end
|
68
|
+
raise NoProxyURIFound, proxy_uri.to_s
|
69
|
+
end
|
70
|
+
|
71
|
+
def find_first_proxy
|
72
|
+
@graph.query(subject: @uri, predicate: PREDICATES[:first]) do |statement|
|
73
|
+
first_uri = statement.object
|
74
|
+
# Validate that the first proxy doesn't have a previous one
|
75
|
+
raise FirstProxyHasPrev, @uri.to_s if find_prev_proxy(first_uri)
|
76
|
+
return first_uri
|
77
|
+
end
|
78
|
+
raise NoFirstProxyFound, @uri.to_s
|
79
|
+
end
|
80
|
+
|
81
|
+
def find_last_proxy
|
82
|
+
@graph.query(subject: @uri, predicate: PREDICATES[:last]) do |statement|
|
83
|
+
last_uri = statement.object
|
84
|
+
# Validate that the last proxy doesn't have a next one
|
85
|
+
raise LastProxyHasNext, @uri.to_s if find_next_proxy(last_uri)
|
86
|
+
return last_uri
|
87
|
+
end
|
88
|
+
raise LastProxyFound, @uri.to_s
|
89
|
+
end
|
90
|
+
|
91
|
+
def find_next_proxy(proxy_uri)
|
92
|
+
@graph.query(subject: proxy_uri, predicate: PREDICATES[:next]) do |statement|
|
93
|
+
return statement.object
|
94
|
+
end
|
95
|
+
nil
|
96
|
+
end
|
97
|
+
|
98
|
+
def find_prev_proxy(proxy_uri)
|
99
|
+
@graph.query(subject: proxy_uri, predicate: PREDICATES[:prev]) do |statement|
|
100
|
+
return statement.object
|
101
|
+
end
|
102
|
+
nil
|
103
|
+
end
|
104
|
+
|
105
|
+
def write_output_file
|
106
|
+
File.open(@output_file, 'w') do |file|
|
107
|
+
file.write("<file_order>\n")
|
108
|
+
@list_source_uuids.each do |uuid|
|
109
|
+
file.write(" <uuid>#{uuid}</uuid>\n")
|
110
|
+
end
|
111
|
+
file.write("</file_order>\n")
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
class PushmiPullyu::AIP::OwnerEmailEditor
|
4
|
+
|
5
|
+
OWNER_PREDICATE = RDF::URI('http://purl.org/ontology/bibo/owner').freeze
|
6
|
+
|
7
|
+
class NoOwnerPredicate < StandardError; end
|
8
|
+
|
9
|
+
def initialize(rdf_string)
|
10
|
+
@document = rdf_string
|
11
|
+
end
|
12
|
+
|
13
|
+
def run
|
14
|
+
ensure_database_connection
|
15
|
+
|
16
|
+
is_modified = false
|
17
|
+
prefixes = nil
|
18
|
+
# Read once to load prefixes (the @things at the top of an n3 file)
|
19
|
+
RDF::N3::Reader.new(input = @document) do |reader|
|
20
|
+
reader.each_statement { |_statement| }
|
21
|
+
prefixes = reader.prefixes
|
22
|
+
end
|
23
|
+
new_body = RDF::N3::Writer.buffer(prefixes: prefixes) do |writer|
|
24
|
+
RDF::N3::Reader.new(input = @document) do |reader|
|
25
|
+
reader.each_statement do |statement|
|
26
|
+
if statement.predicate == OWNER_PREDICATE
|
27
|
+
user = PushmiPullyu::AIP::User.find(statement.object.to_i)
|
28
|
+
writer << [statement.subject, statement.predicate, user.email]
|
29
|
+
is_modified = true
|
30
|
+
else
|
31
|
+
writer << statement
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
return new_body if is_modified
|
37
|
+
raise NoOwnerPredicate
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def ensure_database_connection
|
43
|
+
return if ActiveRecord::Base.connected?
|
44
|
+
ActiveRecord::Base.establish_connection(database_configuration)
|
45
|
+
end
|
46
|
+
|
47
|
+
def database_configuration
|
48
|
+
# Config either from URL, or with more granular options (the later taking precedence)
|
49
|
+
config = {}
|
50
|
+
uri = URI.parse(PushmiPullyu.options[:database][:url])
|
51
|
+
config[:adapter] = PushmiPullyu.options[:database][:adaptor] || uri.scheme
|
52
|
+
config[:host] = PushmiPullyu.options[:database][:host] || uri.host
|
53
|
+
config[:database] = PushmiPullyu.options[:database][:database] || uri.path.split('/')[1].to_s
|
54
|
+
config[:username] = PushmiPullyu.options[:database][:username] || uri.user
|
55
|
+
config[:password] = PushmiPullyu.options[:database][:password] || uri.password
|
56
|
+
params = CGI.parse(uri.query || '')
|
57
|
+
config[:encoding] = PushmiPullyu.options[:database][:encoding] || params['encoding'].to_a.first
|
58
|
+
config[:pool] = PushmiPullyu.options[:database][:pool] || params['pool'].to_a.first
|
59
|
+
config
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
data/lib/pushmi_pullyu/aip.rb
CHANGED
@@ -2,6 +2,7 @@ require 'fileutils'
|
|
2
2
|
|
3
3
|
module PushmiPullyu::AIP
|
4
4
|
class NoidInvalid < StandardError; end
|
5
|
+
module_function
|
5
6
|
|
6
7
|
def create(noid)
|
7
8
|
raise NoidInvalid if noid.blank? || noid.include?('/')
|
@@ -12,11 +13,9 @@ module PushmiPullyu::AIP
|
|
12
13
|
PushmiPullyu::AIP::Downloader.new(noid, aip_directory).run
|
13
14
|
PushmiPullyu::AIP::Creator.new(noid, aip_directory, aip_filename).run
|
14
15
|
|
15
|
-
yield aip_filename
|
16
|
+
yield aip_filename, aip_directory
|
16
17
|
|
17
18
|
FileUtils.rm_rf(aip_filename) if File.exist?(aip_filename)
|
18
19
|
FileUtils.rm_rf(aip_directory) if File.exist?(aip_directory)
|
19
20
|
end
|
20
|
-
|
21
|
-
module_function :create
|
22
21
|
end
|
data/lib/pushmi_pullyu/cli.rb
CHANGED
@@ -41,6 +41,7 @@ class PushmiPullyu::CLI
|
|
41
41
|
Rollbar.error(e)
|
42
42
|
raise e
|
43
43
|
end
|
44
|
+
# rubocop:enable Lint/RescueException
|
44
45
|
end
|
45
46
|
|
46
47
|
def start_server
|
@@ -73,12 +74,11 @@ class PushmiPullyu::CLI
|
|
73
74
|
end
|
74
75
|
|
75
76
|
def parse_config(config_file)
|
76
|
-
opts = {}
|
77
77
|
if File.exist?(config_file)
|
78
|
-
|
78
|
+
YAML.safe_load(ERB.new(IO.read(config_file)).result).deep_symbolize_keys || {}
|
79
|
+
else
|
80
|
+
{}
|
79
81
|
end
|
80
|
-
|
81
|
-
opts
|
82
82
|
end
|
83
83
|
|
84
84
|
# Parse the options.
|
@@ -174,20 +174,20 @@ class PushmiPullyu::CLI
|
|
174
174
|
Rollbar.scoped(noid: item) do
|
175
175
|
begin
|
176
176
|
# Download AIP from Fedora, bag and tar AIP directory and cleanup after block code
|
177
|
-
PushmiPullyu::AIP.create(item) do |aip_filename|
|
177
|
+
PushmiPullyu::AIP.create(item) do |aip_filename, aip_directory|
|
178
178
|
# Push tarred AIP to swift API
|
179
179
|
deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
|
180
180
|
# Log successful preservation event to the log files
|
181
|
-
PushmiPullyu::Logging.log_preservation_event(deposited_file)
|
181
|
+
PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
|
182
182
|
end
|
183
|
-
# rubocop:disable
|
184
|
-
rescue => e
|
183
|
+
# rubocop:disable Lint/RescueException
|
184
|
+
rescue Exception => e
|
185
185
|
Rollbar.error(e)
|
186
186
|
logger.error(e)
|
187
187
|
# TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
|
188
188
|
# move on to the next item.
|
189
189
|
end
|
190
|
-
# rubocop:
|
190
|
+
# rubocop:enable Lint/RescueException
|
191
191
|
end
|
192
192
|
end
|
193
193
|
|
@@ -13,58 +13,96 @@ module PushmiPullyu::Logging
|
|
13
13
|
|
14
14
|
end
|
15
15
|
|
16
|
-
def
|
17
|
-
|
18
|
-
@logger.level = Logger::INFO
|
19
|
-
@logger
|
16
|
+
def logger
|
17
|
+
PushmiPullyu::Logging.logger
|
20
18
|
end
|
21
19
|
|
22
|
-
|
23
|
-
@logger ||= initialize_logger
|
24
|
-
end
|
20
|
+
class << self
|
25
21
|
|
26
|
-
|
27
|
-
log_file = "#{aip_directory}/data/logs/aipcreation.log"
|
28
|
-
aip_logger = Logger.new(log_file)
|
29
|
-
aip_logger.level = logger.level
|
22
|
+
attr_writer :logger
|
30
23
|
|
31
|
-
|
32
|
-
|
33
|
-
|
24
|
+
def initialize_logger(log_target = STDOUT)
|
25
|
+
@logger = Logger.new(log_target)
|
26
|
+
@logger.level = Logger::INFO
|
27
|
+
@logger
|
28
|
+
end
|
34
29
|
|
35
|
-
|
36
|
-
|
30
|
+
def logger
|
31
|
+
@logger ||= initialize_logger
|
32
|
+
end
|
37
33
|
|
38
|
-
|
39
|
-
|
34
|
+
def log_aip_activity(aip_directory, message)
|
35
|
+
log_file = "#{aip_directory}/data/logs/aipcreation.log"
|
36
|
+
aip_logger = Logger.new(log_file)
|
37
|
+
aip_logger.level = logger.level
|
40
38
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
"\t Timestamp of Completion: '#{deposited_file.last_modified}' \n"\
|
45
|
-
"\t AIP Checksum: '#{deposited_file.etag}' \n"\
|
46
|
-
"\t Metadata: #{deposited_file.metadata} \n"
|
39
|
+
# Log to both the application log, and the log file that gets archived in the AIP
|
40
|
+
logger.info(message)
|
41
|
+
aip_logger.info(message)
|
47
42
|
|
48
|
-
|
49
|
-
|
50
|
-
preservation_logger.info(message)
|
43
|
+
aip_logger.close
|
44
|
+
end
|
51
45
|
|
52
|
-
|
53
|
-
|
46
|
+
def log_preservation_event(deposited_file, aip_directory)
|
47
|
+
preservation_logger = Logger.new("#{PushmiPullyu.options[:logdir]}/preservation_events.log")
|
54
48
|
|
55
|
-
|
56
|
-
|
57
|
-
|
49
|
+
message = "#{deposited_file.name} was successfully deposited into Swift Storage!\n"\
|
50
|
+
"Here are the details of this preservation event:\n"\
|
51
|
+
"\tNOID: '#{deposited_file.name}'\n"\
|
52
|
+
"\tTimestamp of Completion: '#{deposited_file.last_modified}'\n"\
|
53
|
+
"\tAIP Checksum: '#{deposited_file.etag}'\n"\
|
54
|
+
"\tMetadata: #{deposited_file.metadata}\n"\
|
55
|
+
|
56
|
+
file_details = file_log_details(aip_directory)
|
57
|
+
|
58
|
+
if file_details.present?
|
59
|
+
message << "\tFile Details:\n"
|
60
|
+
file_details.each do |file_detail|
|
61
|
+
message << %(\t\t{"fileset_uuid": "#{file_detail[:fileset_name]}",
|
62
|
+
\t\t"details": {
|
63
|
+
\t\t\t"file_name": "#{file_detail[:file_name]}",
|
64
|
+
\t\t\t"file_type": "#{file_detail[:file_extension]}",
|
65
|
+
\t\t\t"file_size": #{file_detail[:file_size]}
|
66
|
+
\t\t}}\n)
|
67
|
+
end
|
68
|
+
end
|
58
69
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
70
|
+
# Log to both the application log, and the preservation log file
|
71
|
+
logger.info(message)
|
72
|
+
preservation_logger.info(message)
|
73
|
+
|
74
|
+
preservation_logger.close
|
75
|
+
end
|
76
|
+
|
77
|
+
def reopen
|
78
|
+
if @logger
|
79
|
+
@logger.reopen
|
80
|
+
else
|
81
|
+
@logger = initialize_logger
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
def file_log_details(aip_directory)
|
88
|
+
file_details = []
|
89
|
+
data_files_location = "#{aip_directory}/data/objects/files"
|
90
|
+
|
91
|
+
if Dir.exist?(data_files_location)
|
92
|
+
Dir.glob("#{data_files_location}/*") do |folder|
|
93
|
+
Dir.glob("#{folder}/*") do |file|
|
94
|
+
file_details << {
|
95
|
+
fileset_name: File.dirname(file).split('/')[-1],
|
96
|
+
file_name: File.basename(file),
|
97
|
+
file_size: File.size(file),
|
98
|
+
file_extension: File.extname(file).strip.downcase[1..-1]
|
99
|
+
}
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
file_details
|
64
105
|
end
|
65
|
-
end
|
66
106
|
|
67
|
-
def logger
|
68
|
-
PushmiPullyu::Logging.logger
|
69
107
|
end
|
70
108
|
end
|
data/lib/pushmi_pullyu.rb
CHANGED
@@ -8,8 +8,11 @@ require 'pushmi_pullyu/logging'
|
|
8
8
|
require 'pushmi_pullyu/aip'
|
9
9
|
require 'pushmi_pullyu/aip/creator'
|
10
10
|
require 'pushmi_pullyu/aip/downloader'
|
11
|
-
require 'pushmi_pullyu/aip/solr_fetcher'
|
12
11
|
require 'pushmi_pullyu/aip/fedora_fetcher'
|
12
|
+
require 'pushmi_pullyu/aip/file_list_creator'
|
13
|
+
require 'pushmi_pullyu/aip/owner_email_editor'
|
14
|
+
require 'active_record'
|
15
|
+
require 'pushmi_pullyu/aip/user'
|
13
16
|
require 'pushmi_pullyu/cli'
|
14
17
|
require 'pushmi_pullyu/preservation_queue'
|
15
18
|
require 'pushmi_pullyu/swift_depositer'
|
@@ -20,6 +23,7 @@ require 'active_support/core_ext'
|
|
20
23
|
# PushmiPullyu main module
|
21
24
|
module PushmiPullyu
|
22
25
|
DEFAULTS = {
|
26
|
+
aip_version: 'lightaip-2.0',
|
23
27
|
daemonize: false,
|
24
28
|
debug: false,
|
25
29
|
logdir: 'log',
|
@@ -32,12 +36,8 @@ module PushmiPullyu
|
|
32
36
|
redis: {
|
33
37
|
url: 'redis://localhost:6379'
|
34
38
|
},
|
35
|
-
# TODO: rest of these are examples for solr/fedora/swift... feel free to fill them in correctly
|
36
|
-
solr: {
|
37
|
-
url: 'http://localhost:8983/solr/development'
|
38
|
-
},
|
39
39
|
fedora: {
|
40
|
-
url: 'http://localhost:
|
40
|
+
url: 'http://localhost:8080/fcrepo/rest',
|
41
41
|
user: 'fedoraAdmin',
|
42
42
|
password: 'fedoraAdmin',
|
43
43
|
base_path: '/dev'
|
@@ -52,6 +52,12 @@ module PushmiPullyu
|
|
52
52
|
container: 'ERA'
|
53
53
|
},
|
54
54
|
rollbar: {
|
55
|
+
},
|
56
|
+
database: {
|
57
|
+
encoding: 'utf8',
|
58
|
+
pool: ENV['RAILS_MAX_THREADS'] || 5,
|
59
|
+
url: ENV['DATABASE_URL'] || ENV['JUPITER_DATABASE_URL'] || 'postgresql://jupiter:mysecretpassword@127.0.0.1',
|
60
|
+
database: 'jupiter_development'
|
55
61
|
}
|
56
62
|
}.freeze
|
57
63
|
|
data/pushmi_pullyu.gemspec
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
lib = File.expand_path('
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
2
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
3
|
require 'pushmi_pullyu/version'
|
4
4
|
|
@@ -21,15 +21,17 @@ Gem::Specification.new do |spec|
|
|
21
21
|
|
22
22
|
spec.required_ruby_version = '>= 2.3.1'
|
23
23
|
|
24
|
+
spec.add_runtime_dependency 'activerecord', '>= 5.1.5', '< 5.3.0'
|
24
25
|
spec.add_runtime_dependency 'activesupport', '~> 5.0'
|
25
26
|
spec.add_runtime_dependency 'bagit', '~> 0.4'
|
26
27
|
spec.add_runtime_dependency 'connection_pool', '~> 2.2'
|
27
28
|
spec.add_runtime_dependency 'daemons', '~> 1.2', '>= 1.2.4'
|
28
29
|
spec.add_runtime_dependency 'minitar', '~> 0.6'
|
29
30
|
spec.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
|
30
|
-
spec.add_runtime_dependency '
|
31
|
-
spec.add_runtime_dependency 'rdf
|
32
|
-
spec.add_runtime_dependency '
|
31
|
+
spec.add_runtime_dependency 'pg', '~> 1.0.0'
|
32
|
+
spec.add_runtime_dependency 'rdf', '>= 1.99', '< 4.0'
|
33
|
+
spec.add_runtime_dependency 'rdf-n3', '>= 1.99', '< 4.0'
|
34
|
+
spec.add_runtime_dependency 'redis', '>= 3.3', '< 5.0'
|
33
35
|
spec.add_runtime_dependency 'rollbar', '~> 2.14'
|
34
36
|
|
35
37
|
spec.add_development_dependency 'bundler', '~> 1.14'
|
@@ -38,9 +40,9 @@ Gem::Specification.new do |spec|
|
|
38
40
|
spec.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
|
39
41
|
spec.add_development_dependency 'rake', '~> 12.0'
|
40
42
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
41
|
-
spec.add_development_dependency 'rubocop', '~> 0.
|
43
|
+
spec.add_development_dependency 'rubocop', '~> 0.51'
|
42
44
|
spec.add_development_dependency 'rubocop-rspec', '~> 1.10'
|
43
45
|
spec.add_development_dependency 'timecop', '~> 0.8'
|
44
|
-
spec.add_development_dependency 'vcr', '~>
|
45
|
-
spec.add_development_dependency 'webmock', '~>
|
46
|
+
spec.add_development_dependency 'vcr', '~> 4.0'
|
47
|
+
spec.add_development_dependency 'webmock', '~> 3.3'
|
46
48
|
end
|
metadata
CHANGED
@@ -1,15 +1,35 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pushmi_pullyu
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shane Murnaghan
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-07-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activerecord
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 5.1.5
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 5.3.0
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 5.1.5
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 5.3.0
|
13
33
|
- !ruby/object:Gem::Dependency
|
14
34
|
name: activesupport
|
15
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,47 +127,79 @@ dependencies:
|
|
107
127
|
- !ruby/object:Gem::Version
|
108
128
|
version: 3.3.10
|
109
129
|
- !ruby/object:Gem::Dependency
|
110
|
-
name:
|
130
|
+
name: pg
|
111
131
|
requirement: !ruby/object:Gem::Requirement
|
112
132
|
requirements:
|
113
133
|
- - "~>"
|
114
134
|
- !ruby/object:Gem::Version
|
115
|
-
version:
|
135
|
+
version: 1.0.0
|
116
136
|
type: :runtime
|
117
137
|
prerelease: false
|
118
138
|
version_requirements: !ruby/object:Gem::Requirement
|
119
139
|
requirements:
|
120
140
|
- - "~>"
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: 1.0.0
|
143
|
+
- !ruby/object:Gem::Dependency
|
144
|
+
name: rdf
|
145
|
+
requirement: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - ">="
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '1.99'
|
150
|
+
- - "<"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '4.0'
|
153
|
+
type: :runtime
|
154
|
+
prerelease: false
|
155
|
+
version_requirements: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
121
158
|
- !ruby/object:Gem::Version
|
122
159
|
version: '1.99'
|
160
|
+
- - "<"
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: '4.0'
|
123
163
|
- !ruby/object:Gem::Dependency
|
124
164
|
name: rdf-n3
|
125
165
|
requirement: !ruby/object:Gem::Requirement
|
126
166
|
requirements:
|
127
|
-
- - "
|
167
|
+
- - ">="
|
128
168
|
- !ruby/object:Gem::Version
|
129
169
|
version: '1.99'
|
170
|
+
- - "<"
|
171
|
+
- !ruby/object:Gem::Version
|
172
|
+
version: '4.0'
|
130
173
|
type: :runtime
|
131
174
|
prerelease: false
|
132
175
|
version_requirements: !ruby/object:Gem::Requirement
|
133
176
|
requirements:
|
134
|
-
- - "
|
177
|
+
- - ">="
|
135
178
|
- !ruby/object:Gem::Version
|
136
179
|
version: '1.99'
|
180
|
+
- - "<"
|
181
|
+
- !ruby/object:Gem::Version
|
182
|
+
version: '4.0'
|
137
183
|
- !ruby/object:Gem::Dependency
|
138
184
|
name: redis
|
139
185
|
requirement: !ruby/object:Gem::Requirement
|
140
186
|
requirements:
|
141
|
-
- - "
|
187
|
+
- - ">="
|
142
188
|
- !ruby/object:Gem::Version
|
143
189
|
version: '3.3'
|
190
|
+
- - "<"
|
191
|
+
- !ruby/object:Gem::Version
|
192
|
+
version: '5.0'
|
144
193
|
type: :runtime
|
145
194
|
prerelease: false
|
146
195
|
version_requirements: !ruby/object:Gem::Requirement
|
147
196
|
requirements:
|
148
|
-
- - "
|
197
|
+
- - ">="
|
149
198
|
- !ruby/object:Gem::Version
|
150
199
|
version: '3.3'
|
200
|
+
- - "<"
|
201
|
+
- !ruby/object:Gem::Version
|
202
|
+
version: '5.0'
|
151
203
|
- !ruby/object:Gem::Dependency
|
152
204
|
name: rollbar
|
153
205
|
requirement: !ruby/object:Gem::Requirement
|
@@ -258,14 +310,14 @@ dependencies:
|
|
258
310
|
requirements:
|
259
311
|
- - "~>"
|
260
312
|
- !ruby/object:Gem::Version
|
261
|
-
version: '0.
|
313
|
+
version: '0.51'
|
262
314
|
type: :development
|
263
315
|
prerelease: false
|
264
316
|
version_requirements: !ruby/object:Gem::Requirement
|
265
317
|
requirements:
|
266
318
|
- - "~>"
|
267
319
|
- !ruby/object:Gem::Version
|
268
|
-
version: '0.
|
320
|
+
version: '0.51'
|
269
321
|
- !ruby/object:Gem::Dependency
|
270
322
|
name: rubocop-rspec
|
271
323
|
requirement: !ruby/object:Gem::Requirement
|
@@ -300,28 +352,28 @@ dependencies:
|
|
300
352
|
requirements:
|
301
353
|
- - "~>"
|
302
354
|
- !ruby/object:Gem::Version
|
303
|
-
version: '
|
355
|
+
version: '4.0'
|
304
356
|
type: :development
|
305
357
|
prerelease: false
|
306
358
|
version_requirements: !ruby/object:Gem::Requirement
|
307
359
|
requirements:
|
308
360
|
- - "~>"
|
309
361
|
- !ruby/object:Gem::Version
|
310
|
-
version: '
|
362
|
+
version: '4.0'
|
311
363
|
- !ruby/object:Gem::Dependency
|
312
364
|
name: webmock
|
313
365
|
requirement: !ruby/object:Gem::Requirement
|
314
366
|
requirements:
|
315
367
|
- - "~>"
|
316
368
|
- !ruby/object:Gem::Version
|
317
|
-
version: '
|
369
|
+
version: '3.3'
|
318
370
|
type: :development
|
319
371
|
prerelease: false
|
320
372
|
version_requirements: !ruby/object:Gem::Requirement
|
321
373
|
requirements:
|
322
374
|
- - "~>"
|
323
375
|
- !ruby/object:Gem::Version
|
324
|
-
version: '
|
376
|
+
version: '3.3'
|
325
377
|
description:
|
326
378
|
email:
|
327
379
|
- murnagha@ualberta.ca
|
@@ -354,7 +406,9 @@ files:
|
|
354
406
|
- lib/pushmi_pullyu/aip/creator.rb
|
355
407
|
- lib/pushmi_pullyu/aip/downloader.rb
|
356
408
|
- lib/pushmi_pullyu/aip/fedora_fetcher.rb
|
357
|
-
- lib/pushmi_pullyu/aip/
|
409
|
+
- lib/pushmi_pullyu/aip/file_list_creator.rb
|
410
|
+
- lib/pushmi_pullyu/aip/owner_email_editor.rb
|
411
|
+
- lib/pushmi_pullyu/aip/user.rb
|
358
412
|
- lib/pushmi_pullyu/cli.rb
|
359
413
|
- lib/pushmi_pullyu/logging.rb
|
360
414
|
- lib/pushmi_pullyu/preservation_queue.rb
|
@@ -384,7 +438,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
384
438
|
version: '0'
|
385
439
|
requirements: []
|
386
440
|
rubyforge_project:
|
387
|
-
rubygems_version: 2.
|
441
|
+
rubygems_version: 2.7.6
|
388
442
|
signing_key:
|
389
443
|
specification_version: 4
|
390
444
|
summary: Ruby application to manage flow of content from Fedora into Swift for preservation
|
@@ -1,33 +0,0 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 'net/http'
|
3
|
-
|
4
|
-
class PushmiPullyu::AIP::SolrFetcher
|
5
|
-
|
6
|
-
class SolrFetchError < StandardError; end
|
7
|
-
|
8
|
-
def initialize(noid)
|
9
|
-
@noid = noid
|
10
|
-
end
|
11
|
-
|
12
|
-
def fetch_permission_object_ids
|
13
|
-
hash = JSON.parse(run_query_json)
|
14
|
-
|
15
|
-
return [] if hash['response']['docs'].empty?
|
16
|
-
|
17
|
-
hash['response']['docs'].map { |hit| hit['id'] }
|
18
|
-
end
|
19
|
-
|
20
|
-
private
|
21
|
-
|
22
|
-
# Return fetched results, else raise an error
|
23
|
-
def run_query_json
|
24
|
-
response = Net::HTTP.get_response(
|
25
|
-
URI("#{PushmiPullyu.options[:solr][:url]}/select?q=accessTo_ssim:#{@noid}&fl=id&wt=json")
|
26
|
-
)
|
27
|
-
|
28
|
-
return response.body if response.is_a?(Net::HTTPSuccess)
|
29
|
-
|
30
|
-
raise SolrFetchError
|
31
|
-
end
|
32
|
-
|
33
|
-
end
|