pushmi_pullyu 1.0.5 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/CODEOWNERS +2 -0
- data/.github/PULL_REQUEST_TEMPLATE +11 -0
- data/.github/dependabot.yml +12 -0
- data/.github/workflows/ruby.yml +31 -0
- data/.rubocop.yml +21 -1
- data/CHANGELOG.md +25 -0
- data/Dangerfile +7 -3
- data/README.md +4 -5
- data/examples/pushmi_pullyu.yml +6 -6
- data/lib/pushmi_pullyu/aip/downloader.rb +133 -180
- data/lib/pushmi_pullyu/aip.rb +9 -6
- data/lib/pushmi_pullyu/cli.rb +30 -25
- data/lib/pushmi_pullyu/logging.rb +6 -6
- data/lib/pushmi_pullyu/swift_depositer.rb +1 -1
- data/lib/pushmi_pullyu/version.rb +1 -1
- data/lib/pushmi_pullyu.rb +10 -17
- data/pushmi_pullyu.gemspec +18 -14
- metadata +69 -47
- data/.travis.yml +0 -16
- data/lib/pushmi_pullyu/aip/fedora_fetcher.rb +0 -66
- data/lib/pushmi_pullyu/aip/file_list_creator.rb +0 -121
- data/lib/pushmi_pullyu/aip/owner_email_editor.rb +0 -50
- data/lib/pushmi_pullyu/aip/user.rb +0 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: e0e6a8b121842bd88b72a166f8ff85b9b7d78e8dd294c248c573af3981582a81
|
4
|
+
data.tar.gz: 29e1631a92cd88dcb827769d6d6ef1e89172cc099fa88eaa1193406f4df604ba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4daaab3f235775f372c1141f1eabc0f8f122272115105d07f08797d492d2475c003e7145ae9aa39df7ce7a0592da6f0eb3ff3bc2bd65e71758299dbaaf6ba535
|
7
|
+
data.tar.gz: f96752a30fc91992224d2a76c39106a1992aa4eb2a2eec4b9f09b30b503a403b74353b6b0e6eecec0e85911760c02797f3fc9a1bb51dc35e32fc6dc299925450
|
data/.github/CODEOWNERS
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [ master ]
|
6
|
+
pull_request:
|
7
|
+
branches: [ master ]
|
8
|
+
|
9
|
+
jobs:
|
10
|
+
test:
|
11
|
+
runs-on: ubuntu-latest
|
12
|
+
services:
|
13
|
+
redis:
|
14
|
+
image: redis:alpine
|
15
|
+
ports: ["6379:6379"]
|
16
|
+
steps:
|
17
|
+
- uses: actions/checkout@v2
|
18
|
+
- name: Set up Ruby
|
19
|
+
uses: ruby/setup-ruby@v1
|
20
|
+
with:
|
21
|
+
ruby-version: 2.6.6
|
22
|
+
- name: Install dependencies
|
23
|
+
run: bundle install
|
24
|
+
- name: Lint with RuboCop
|
25
|
+
run: bundle exec rubocop --parallel
|
26
|
+
- name: Run Danger
|
27
|
+
env:
|
28
|
+
DANGER_GITHUB_API_TOKEN: ${{ secrets.DANGER_GITHUB_API_TOKEN }}
|
29
|
+
run: bundle exec danger
|
30
|
+
- name: Run tests
|
31
|
+
run: bundle exec rake spec
|
data/.rubocop.yml
CHANGED
@@ -10,8 +10,10 @@ AllCops:
|
|
10
10
|
Exclude:
|
11
11
|
- 'tmp/**/*'
|
12
12
|
- 'vendor/**/*'
|
13
|
+
- 'Dangerfile'
|
13
14
|
ExtraDetails: true
|
14
|
-
TargetRubyVersion: 2.
|
15
|
+
TargetRubyVersion: 2.5
|
16
|
+
NewCops: enable
|
15
17
|
|
16
18
|
# readability is Actually Good
|
17
19
|
Layout/EmptyLinesAroundClassBody:
|
@@ -59,6 +61,15 @@ Style/ClassAndModuleChildren:
|
|
59
61
|
Style/Documentation:
|
60
62
|
Enabled: false
|
61
63
|
|
64
|
+
Style/HashEachMethods:
|
65
|
+
Enabled: true
|
66
|
+
|
67
|
+
Style/HashTransformKeys:
|
68
|
+
Enabled: true
|
69
|
+
|
70
|
+
Style/HashTransformValues:
|
71
|
+
Enabled: true
|
72
|
+
|
62
73
|
Naming/FileName:
|
63
74
|
Exclude:
|
64
75
|
- Dangerfile
|
@@ -86,5 +97,14 @@ RSpec/ExampleLength:
|
|
86
97
|
RSpec/MultipleExpectations:
|
87
98
|
Enabled: false
|
88
99
|
|
100
|
+
RSpec/MultipleMemoizedHelpers:
|
101
|
+
Enabled: false
|
102
|
+
|
89
103
|
RSpec/DescribedClass:
|
90
104
|
EnforcedStyle: explicit
|
105
|
+
|
106
|
+
Naming/VariableNumber:
|
107
|
+
Enabled: false
|
108
|
+
|
109
|
+
Style/OpenStructUse:
|
110
|
+
Enabled: false
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# Changelog
|
2
|
+
All notable changes to PushmiPullyu project will be documented in this file.
|
3
|
+
|
4
|
+
PushmiPullyu is a Ruby application, whose primary job is to manage the flow of content from [Jupiter](https://github.com/ualbertalib/jupiter/) into Swift for preservation.
|
5
|
+
|
6
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
7
|
+
and releases in PushmiPullyu adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
8
|
+
|
9
|
+
## [Unreleased]
|
10
|
+
|
11
|
+
## [2.0.2] - 2021-11-22
|
12
|
+
|
13
|
+
- Fix authentication bug when using ssl
|
14
|
+
- bump rubocop and rubocop-rspec
|
15
|
+
|
16
|
+
## [2.0.1] - 2021-02-02
|
17
|
+
|
18
|
+
- Fix dependency declaration for UUID gem
|
19
|
+
|
20
|
+
## [2.0.0] - 2020-12-14
|
21
|
+
|
22
|
+
### Removed
|
23
|
+
- Data output for original_file information
|
24
|
+
|
25
|
+
## [1.0.6] - 2018-11-29
|
data/Dangerfile
CHANGED
@@ -13,9 +13,13 @@ end
|
|
13
13
|
# just leaving a title
|
14
14
|
warn('Please add a detailed summary in the description.') if github.pr_body.length < 5
|
15
15
|
|
16
|
-
#
|
17
|
-
|
18
|
-
|
16
|
+
# Let people say that this isn't worth a CHANGELOG entry in the PR if they choose
|
17
|
+
declared_trivial = (github.pr_title + github.pr_body).include?('#trivial') || !has_app_changes
|
18
|
+
|
19
|
+
if !git.modified_files.include?('CHANGELOG.md') && !declared_trivial
|
20
|
+
error_message = "Please include a CHANGELOG entry. \nYou can find it at " \
|
21
|
+
'[CHANGELOG.md](https://github.com/ualbertalib/pushmi_pullyu/blob/master/CHANGELOG.md).'
|
22
|
+
fail(error_message, sticky: false)
|
19
23
|
end
|
20
24
|
|
21
25
|
# Warn when there is a big PR
|
data/README.md
CHANGED
@@ -5,22 +5,21 @@
|
|
5
5
|
</p>
|
6
6
|
|
7
7
|
[](https://rubygems.org/gems/pushmi_pullyu)
|
8
|
-
[](https://travis-ci.org/ualbertalib/pushmi_pullyu)
|
8
|
+
[](https://github.com/ualbertalib/pushmi_pullyu/actions)
|
10
9
|
[](https://coveralls.io/github/ualbertalib/pushmi_pullyu?branch=master)
|
11
10
|
|
12
11
|
PushmiPullyu is a Ruby application, running behind the firewall that protects our Swift environment.
|
13
12
|
|
14
|
-
Its primary job is to manage the flow of content from
|
13
|
+
Its primary job is to manage the flow of content from Jupiter into Swift for preservation.
|
15
14
|
|
16
15
|

|
17
16
|
|
18
17
|
## Workflow
|
19
18
|
|
20
|
-
1. Any save (create or update) on a Item/Thesis in ERA/Jupiter will trigger an after save callback that will push the item's unique identifier (UUID
|
19
|
+
1. Any save (create or update) on a Item/Thesis in ERA/Jupiter will trigger an after save callback that will push the item's unique identifier (UUID) into a Queue.
|
21
20
|
2. The queue (Redis) is setup to be a unique set (which only allows one item's UUID to be included in the queue at a single time), and ordered by priority from First In, First out (FIFO).
|
22
21
|
3. PushmiPullyu will then monitor the queue. After a certain wait period has passed since an element has been on the queue, PushmiPullyu will then retrieve the elements off the queue and begin to process the preservation event.
|
23
|
-
4. All the
|
22
|
+
4. All the Item/Thesis information and data required for preservation are retrieved from Jupiter using multiple REST calls to Jupiter's AIP API.
|
24
23
|
5. An Archival Information Package (AIP) is created from the item's information. It is then bagged and tarred.
|
25
24
|
6. The AIP tar is then uploaded to Swift via a REST call.
|
26
25
|
7. On a successful Swift upload, a entry is added for this preservation event to the preservation event logs.
|
data/examples/pushmi_pullyu.yml
CHANGED
@@ -21,12 +21,6 @@ minimum_age: 0
|
|
21
21
|
redis:
|
22
22
|
url: redis://localhost:6379
|
23
23
|
|
24
|
-
fedora:
|
25
|
-
url: http://localhost:8080/fcrepo/rest
|
26
|
-
user: fedoraAdmin
|
27
|
-
password: fedoraAdmin
|
28
|
-
base_path: /dev
|
29
|
-
|
30
24
|
database:
|
31
25
|
url: postgresql://jupiter:mysecretpassword@127.0.0.1/jupiter_development
|
32
26
|
|
@@ -44,3 +38,9 @@ rollbar:
|
|
44
38
|
token: 'abc123xyz'
|
45
39
|
proxy_host: 'your_proxy_host_url'
|
46
40
|
proxy_port: '80'
|
41
|
+
|
42
|
+
jupiter:
|
43
|
+
user: jupiter@ualberta.ca
|
44
|
+
api_key: 5042c4ad-6d22-486d-bc63-2b9e5b9a630a
|
45
|
+
jupiter_url: http://localhost:3000/
|
46
|
+
aip_api_path: aip/v1
|
@@ -2,261 +2,214 @@ require 'fileutils'
|
|
2
2
|
require 'ostruct'
|
3
3
|
require 'rdf'
|
4
4
|
require 'rdf/n3'
|
5
|
+
require 'net/http'
|
6
|
+
require 'uri'
|
7
|
+
require 'digest'
|
5
8
|
|
6
|
-
# Download all of the metadata/datastreams and associated data
|
7
|
-
# related to an object
|
9
|
+
# Download all of the metadata/datastreams and associated data related to an object
|
8
10
|
class PushmiPullyu::AIP::Downloader
|
9
11
|
|
10
12
|
PREDICATE_URIS = {
|
11
13
|
filename: 'http://purl.org/dc/terms/title',
|
12
14
|
member_files: 'http://pcdm.org/models#hasFile',
|
13
15
|
member_file_sets: 'http://pcdm.org/models#hasMember',
|
14
|
-
original_file: 'http://pcdm.org/use#OriginalFile',
|
15
16
|
type: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
|
16
17
|
}.freeze
|
17
18
|
|
18
|
-
class
|
19
|
-
class
|
20
|
-
class
|
21
|
-
class NoOriginalFile < StandardError; end
|
19
|
+
class JupiterDownloadError < StandardError; end
|
20
|
+
class JupiterCopyError < StandardError; end
|
21
|
+
class JupiterAuthenticationError < StandardError; end
|
22
22
|
|
23
|
-
def initialize(
|
24
|
-
@
|
23
|
+
def initialize(entity, aip_directory)
|
24
|
+
@entity = entity
|
25
|
+
@entity_identifier = "[#{entity[:type]} - #{entity[:uuid]}]".freeze
|
25
26
|
@aip_directory = aip_directory
|
26
27
|
end
|
27
28
|
|
28
29
|
def run
|
29
|
-
|
30
|
+
PushmiPullyu.logger.info("#{@entity_identifier}: Retreiving data from Jupiter ...")
|
30
31
|
|
31
|
-
|
32
|
+
authenticate_http_calls
|
33
|
+
make_directories
|
32
34
|
|
33
35
|
# Main object metadata
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
download_and_log(
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
member_files(file_set_uuid).each do |file_path|
|
52
|
-
path_spec = OpenStruct.new(
|
53
|
-
remote: "/files/#{file_path}/fcr:metadata",
|
54
|
-
# Note: local file gets clobbered on each download until it finds the right one
|
55
|
-
local: "#{file_set_dirs(file_set_uuid).metadata}/original_file_metadata.n3",
|
56
|
-
optional: true
|
57
|
-
)
|
58
|
-
download_and_log(path_spec, file_set_downloader)
|
59
|
-
if original_file?(path_spec.local)
|
60
|
-
original_file_remote_base = "/files/#{file_path}"
|
61
|
-
break
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
raise NoOriginalFile unless original_file_remote_base.present?
|
66
|
-
|
67
|
-
[:content, :fixity].each do |item|
|
68
|
-
path_spec = file_aip_paths(file_set_uuid, original_file_remote_base)[item]
|
69
|
-
download_and_log(path_spec, file_set_downloader)
|
70
|
-
end
|
36
|
+
download_and_log(object_aip_paths[:main_object_remote],
|
37
|
+
object_aip_paths[:main_object_local])
|
38
|
+
download_and_log(object_aip_paths[:file_sets_remote],
|
39
|
+
object_aip_paths[:file_sets_local])
|
40
|
+
|
41
|
+
# Get file paths for processing
|
42
|
+
file_paths = get_file_paths(object_aip_paths[:file_paths_remote])
|
43
|
+
|
44
|
+
file_paths[:files].each do |file_path|
|
45
|
+
file_uuid = file_path[:file_uuid]
|
46
|
+
make_file_set_directories(file_uuid)
|
47
|
+
copy_and_log(file_uuid, file_path)
|
48
|
+
file_aip_path = file_aip_paths(file_uuid)
|
49
|
+
download_and_log(file_aip_path[:fixity_remote],
|
50
|
+
file_aip_path[:fixity_local])
|
51
|
+
download_and_log(file_aip_path[:file_set_remote],
|
52
|
+
file_aip_path[:file_set_local])
|
71
53
|
end
|
72
54
|
end
|
73
55
|
|
74
56
|
private
|
75
57
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
58
|
+
def copy_and_log(file_uuid, file_path)
|
59
|
+
remote = file_path[:file_path]
|
60
|
+
remote_checksum = file_path[:file_checksum]
|
61
|
+
files_path = file_set_dirs(file_uuid)[:files]
|
62
|
+
output_file = "#{files_path}/#{file_path[:file_name]}"
|
63
|
+
log_downloading(remote, output_file)
|
64
|
+
FileUtils.copy_file(remote, output_file)
|
80
65
|
|
81
|
-
|
82
|
-
|
66
|
+
is_success = File.exist?(output_file) &&
|
67
|
+
File.size(remote) == File.size(output_file) &&
|
68
|
+
compare_md5(output_file, remote_checksum)
|
83
69
|
|
84
|
-
is_success = fedora_fetcher.download_object(output_file,
|
85
|
-
url_extra: path_spec.remote,
|
86
|
-
optional: path_spec.optional,
|
87
|
-
is_rdf: is_rdf,
|
88
|
-
should_add_user_email: should_add_user_email)
|
89
70
|
log_saved(is_success, output_file)
|
71
|
+
|
72
|
+
raise JupiterCopyError unless is_success
|
73
|
+
end
|
74
|
+
|
75
|
+
def compare_md5(local, remote_checksum)
|
76
|
+
local_md5 = Digest::MD5.file local
|
77
|
+
local_md5.base64digest == remote_checksum
|
78
|
+
end
|
79
|
+
|
80
|
+
def authenticate_http_calls
|
81
|
+
@uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
|
82
|
+
@http = Net::HTTP.new(@uri.host, @uri.port)
|
83
|
+
@http.use_ssl = true if @uri.instance_of? URI::HTTPS
|
84
|
+
request = Net::HTTP::Post.new("#{@uri.request_uri}auth/system")
|
85
|
+
request.set_form_data(
|
86
|
+
email: PushmiPullyu.options[:jupiter][:user],
|
87
|
+
api_key: PushmiPullyu.options[:jupiter][:api_key]
|
88
|
+
)
|
89
|
+
response = @http.request(request)
|
90
|
+
# If we cannot find the set-cookie header then the session was not set
|
91
|
+
raise JupiterAuthenticationError if response.response['set-cookie'].nil?
|
92
|
+
|
93
|
+
@cookies = response.response['set-cookie']
|
94
|
+
end
|
95
|
+
|
96
|
+
def download_and_log(remote, local)
|
97
|
+
log_downloading(remote, local)
|
98
|
+
|
99
|
+
@uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
|
100
|
+
request = Net::HTTP::Get.new(@uri.request_uri + remote)
|
101
|
+
# add previously stored cookies
|
102
|
+
request['Cookie'] = @cookies
|
103
|
+
|
104
|
+
response = @http.request(request)
|
105
|
+
is_success = if response.is_a?(Net::HTTPSuccess)
|
106
|
+
File.open(local, 'wb') do |file|
|
107
|
+
file.write(response.body)
|
108
|
+
end
|
109
|
+
# Response was a success and the file was saved to local
|
110
|
+
File.exist? local
|
111
|
+
end
|
112
|
+
|
113
|
+
log_saved(is_success, local)
|
114
|
+
raise JupiterDownloadError unless is_success
|
115
|
+
end
|
116
|
+
|
117
|
+
def get_file_paths(url)
|
118
|
+
request = Net::HTTP::Get.new(@uri.request_uri + url)
|
119
|
+
# add previously stored cookies
|
120
|
+
request['Cookie'] = @cookies
|
121
|
+
|
122
|
+
response = @http.request(request)
|
123
|
+
|
124
|
+
JSON.parse(response.body, symbolize_names: true)
|
90
125
|
end
|
91
126
|
|
92
|
-
def
|
93
|
-
|
94
|
-
|
95
|
-
"#{@noid}: #{output_file} -- creating from #{url} ...")
|
96
|
-
PushmiPullyu::AIP::FileListCreator.new(url, output_file, member_file_set_uuids).run
|
97
|
-
PushmiPullyu::Logging.log_aip_activity(@aip_directory,
|
98
|
-
"#{@noid}: #{output_file} -- created")
|
127
|
+
def object_uri
|
128
|
+
aip_api_url = PushmiPullyu.options[:jupiter][:aip_api_path]
|
129
|
+
@object_uri ||= "#{aip_api_url}/#{@entity[:type]}/#{@entity[:uuid]}"
|
99
130
|
end
|
100
131
|
|
101
132
|
### Logging
|
102
133
|
|
103
|
-
def
|
104
|
-
message = "#{@
|
134
|
+
def log_downloading(url, output_file)
|
135
|
+
message = "#{@entity_identifier}: #{output_file} -- Downloading from #{url} ..."
|
105
136
|
PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
|
106
137
|
end
|
107
138
|
|
108
139
|
def log_saved(is_success, output_file)
|
109
|
-
message = "#{@
|
140
|
+
message = "#{@entity_identifier}: #{output_file} -- #{is_success ? 'Saved' : 'Failed'}"
|
110
141
|
PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
|
111
142
|
end
|
112
143
|
|
113
144
|
### Directories
|
114
145
|
|
115
146
|
def aip_dirs
|
116
|
-
@aip_dirs ||=
|
147
|
+
@aip_dirs ||= {
|
117
148
|
objects: "#{@aip_directory}/data/objects",
|
118
149
|
metadata: "#{@aip_directory}/data/objects/metadata",
|
119
150
|
files: "#{@aip_directory}/data/objects/files",
|
120
151
|
files_metadata: "#{@aip_directory}/data/objects/metadata/files_metadata",
|
121
152
|
logs: "#{@aip_directory}/data/logs",
|
122
153
|
file_logs: "#{@aip_directory}/data/logs/files_logs"
|
123
|
-
|
154
|
+
}
|
124
155
|
end
|
125
156
|
|
126
157
|
def file_set_dirs(file_set_uuid)
|
127
158
|
@file_set_dirs ||= {}
|
128
|
-
@file_set_dirs[file_set_uuid] ||=
|
129
|
-
metadata: "#{aip_dirs
|
130
|
-
files: "#{aip_dirs
|
131
|
-
logs: "#{aip_dirs
|
132
|
-
|
159
|
+
@file_set_dirs[file_set_uuid] ||= {
|
160
|
+
metadata: "#{aip_dirs[:files_metadata]}/#{file_set_uuid}",
|
161
|
+
files: "#{aip_dirs[:files]}/#{file_set_uuid}",
|
162
|
+
logs: "#{aip_dirs[:file_logs]}/#{file_set_uuid}"
|
163
|
+
}
|
133
164
|
end
|
134
165
|
|
135
166
|
def make_directories
|
167
|
+
PushmiPullyu.logger.debug("#{@entity_identifier}: Creating directories ...")
|
136
168
|
clean_directories
|
137
|
-
|
138
|
-
aip_dirs.to_h.each_value do |path|
|
169
|
+
aip_dirs.each_value do |path|
|
139
170
|
FileUtils.mkdir_p(path)
|
140
171
|
end
|
141
|
-
PushmiPullyu.logger.debug("#{@
|
172
|
+
PushmiPullyu.logger.debug("#{@entity_identifier}: Creating directories done")
|
142
173
|
end
|
143
174
|
|
144
175
|
def make_file_set_directories(file_set_uuid)
|
145
|
-
PushmiPullyu.logger.debug("#{@
|
146
|
-
file_set_dirs(file_set_uuid).
|
176
|
+
PushmiPullyu.logger.debug("#{@entity_identifier}: Creating file set #{file_set_uuid} directories ...")
|
177
|
+
file_set_dirs(file_set_uuid).each_value do |path|
|
147
178
|
FileUtils.mkdir_p(path)
|
148
179
|
end
|
149
|
-
PushmiPullyu.logger.debug("#{@
|
180
|
+
PushmiPullyu.logger.debug("#{@entity_identifier}: Creating file set #{file_set_uuid} directories done")
|
150
181
|
end
|
151
182
|
|
152
183
|
def clean_directories
|
153
184
|
return unless File.exist?(@aip_directory)
|
154
185
|
|
155
|
-
PushmiPullyu.logger.debug("#{@
|
186
|
+
PushmiPullyu.logger.debug("#{@entity_identifier}: Nuking directories ...")
|
156
187
|
FileUtils.rm_rf(@aip_directory)
|
157
188
|
end
|
158
189
|
|
159
190
|
### Files
|
160
191
|
|
161
192
|
def object_aip_paths
|
162
|
-
@object_aip_paths ||=
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
local: "#{aip_dirs.files_metadata}/file_order.xml"
|
176
|
-
)
|
177
|
-
).freeze
|
178
|
-
end
|
179
|
-
|
180
|
-
def file_set_aip_paths(file_set_uuid)
|
181
|
-
@file_set_aip_paths ||= {}
|
182
|
-
@file_set_aip_paths[file_set_uuid] ||= OpenStruct.new(
|
183
|
-
main_object: OpenStruct.new(
|
184
|
-
remote: nil, # Base file_set path
|
185
|
-
local: "#{file_set_dirs(file_set_uuid).metadata}/file_set_metadata.n3",
|
186
|
-
should_add_user_email: true,
|
187
|
-
optional: false
|
188
|
-
)
|
189
|
-
).freeze
|
190
|
-
end
|
191
|
-
|
192
|
-
def file_aip_paths(file_set_uuid, original_file_remote_base)
|
193
|
+
@object_aip_paths ||= {
|
194
|
+
# Base path
|
195
|
+
main_object_remote: object_uri,
|
196
|
+
main_object_local: "#{aip_dirs[:metadata]}/object_metadata.n3",
|
197
|
+
file_sets_remote: "#{object_uri}/filesets",
|
198
|
+
file_sets_local: "#{aip_dirs[:files_metadata]}/file_order.xml",
|
199
|
+
# This is downloaded for processing but not saved
|
200
|
+
file_paths_remote: "#{object_uri}/file_paths"
|
201
|
+
}.freeze
|
202
|
+
end
|
203
|
+
|
204
|
+
def file_aip_paths(file_set_uuid)
|
205
|
+
file_set_paths = file_set_dirs(file_set_uuid)
|
193
206
|
@file_aip_paths ||= {}
|
194
|
-
@file_aip_paths[file_set_uuid] ||=
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
fixity: OpenStruct.new(
|
201
|
-
remote: "#{original_file_remote_base}/fcr:fixity",
|
202
|
-
local: "#{file_set_dirs(file_set_uuid)[:logs]}/content_fixity_report.n3",
|
203
|
-
optional: false
|
204
|
-
)
|
205
|
-
).freeze
|
206
|
-
end
|
207
|
-
|
208
|
-
def member_file_set_uuids
|
209
|
-
@member_file_set_uuids ||= []
|
210
|
-
return @member_file_set_uuids unless @member_file_set_uuids.empty?
|
211
|
-
|
212
|
-
member_file_set_predicate = RDF::URI(PREDICATE_URIS[:member_file_sets])
|
213
|
-
|
214
|
-
graph = RDF::Graph.load(object_aip_paths.main_object.local)
|
215
|
-
|
216
|
-
graph.query(predicate: member_file_set_predicate) do |results|
|
217
|
-
# Get uuid from end of fedora path
|
218
|
-
@member_file_set_uuids << results.object.to_s.split('/').last
|
219
|
-
end
|
220
|
-
return @member_file_set_uuids unless @member_file_set_uuids.empty?
|
221
|
-
|
222
|
-
raise NoFileSets
|
223
|
-
end
|
224
|
-
|
225
|
-
def file_set_filename(file_set_uuid)
|
226
|
-
filename_predicate = RDF::URI(PREDICATE_URIS[:filename])
|
227
|
-
|
228
|
-
graph = RDF::Graph.load(file_set_aip_paths(file_set_uuid).main_object.local)
|
229
|
-
|
230
|
-
graph.query(predicate: filename_predicate) do |results|
|
231
|
-
return "#{file_set_dirs(file_set_uuid).files}/#{results.object}"
|
232
|
-
end
|
233
|
-
|
234
|
-
raise NoContentFilename
|
235
|
-
end
|
236
|
-
|
237
|
-
def member_files(file_set_uuid)
|
238
|
-
member_file_predicate = RDF::URI(PREDICATE_URIS[:member_files])
|
239
|
-
|
240
|
-
graph = RDF::Graph.load(file_set_aip_paths(file_set_uuid).main_object.local)
|
241
|
-
|
242
|
-
member_files = []
|
243
|
-
graph.query(predicate: member_file_predicate) do |results|
|
244
|
-
# Get uuid from end of fedora path
|
245
|
-
member_files << results.object.to_s.split('/').last
|
246
|
-
end
|
247
|
-
return member_files if member_files.present?
|
248
|
-
|
249
|
-
raise NoMemberFiles
|
250
|
-
end
|
251
|
-
|
252
|
-
def original_file?(metadata_filename)
|
253
|
-
type_predicate = RDF::URI(PREDICATE_URIS[:type])
|
254
|
-
original_file_uri = RDF::URI(PREDICATE_URIS[:original_file])
|
255
|
-
graph = RDF::Graph.load(metadata_filename)
|
256
|
-
graph.query(predicate: type_predicate) do |results|
|
257
|
-
return true if results.object == original_file_uri
|
258
|
-
end
|
259
|
-
false
|
207
|
+
@file_aip_paths[file_set_uuid] ||= {
|
208
|
+
fixity_remote: "#{object_uri}/filesets/#{file_set_uuid}/fixity",
|
209
|
+
fixity_local: "#{file_set_paths[:logs]}/content_fixity_report.n3",
|
210
|
+
file_set_remote: "#{object_uri}/filesets/#{file_set_uuid}",
|
211
|
+
file_set_local: "#{file_set_paths[:metadata]}/file_set_metadata.n3"
|
212
|
+
}.freeze
|
260
213
|
end
|
261
214
|
|
262
215
|
end
|
data/lib/pushmi_pullyu/aip.rb
CHANGED
@@ -1,17 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
|
+
require 'uuid'
|
2
3
|
|
3
4
|
module PushmiPullyu::AIP
|
4
|
-
class
|
5
|
+
class EntityInvalid < StandardError; end
|
5
6
|
module_function
|
6
7
|
|
7
|
-
def create(
|
8
|
-
raise
|
8
|
+
def create(entity)
|
9
|
+
raise EntityInvalid if entity.nil? ||
|
10
|
+
UUID.validate(entity[:uuid]) != true ||
|
11
|
+
entity[:type].blank?
|
9
12
|
|
10
|
-
aip_directory = "#{PushmiPullyu.options[:workdir]}/#{
|
13
|
+
aip_directory = "#{PushmiPullyu.options[:workdir]}/#{entity[:uuid]}"
|
11
14
|
aip_filename = "#{aip_directory}.tar"
|
12
15
|
|
13
|
-
PushmiPullyu::AIP::Downloader.new(
|
14
|
-
PushmiPullyu::AIP::Creator.new(
|
16
|
+
PushmiPullyu::AIP::Downloader.new(entity, aip_directory).run
|
17
|
+
PushmiPullyu::AIP::Creator.new(entity[:uuid], aip_directory, aip_filename).run
|
15
18
|
|
16
19
|
yield aip_filename, aip_directory
|
17
20
|
|