valkyrie 3.0.3 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +1 -1
- data/.lando.yml +11 -4
- data/.rubocop_todo.yml +1 -0
- data/CHANGELOG.md +17 -0
- data/lib/valkyrie/specs/shared_specs/storage_adapter.rb +66 -1
- data/lib/valkyrie/storage/disk.rb +6 -0
- data/lib/valkyrie/storage/fedora.rb +123 -10
- data/lib/valkyrie/storage/memory.rb +64 -4
- data/lib/valkyrie/storage/versioned_disk.rb +182 -0
- data/lib/valkyrie/storage.rb +1 -0
- data/lib/valkyrie/storage_adapter.rb +1 -0
- data/lib/valkyrie/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e6705b30a79e047e4892d78b3f60a2fb71a9acc4b4687272a7e169e216177a81
|
4
|
+
data.tar.gz: c4cbe96c86968b611d1932f505cb020c1071e6fa0938f11b25f4e78c3c7b840b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 80cc406ed49c5256d4744c83eaf0e849722d28b151a6f47b5dfdb9c69871a82253f9b3c2b8331f92b6a253792b1c982f5bfcd9c198cd631f84613e50c57c4d5f
|
7
|
+
data.tar.gz: b7b397512adac100586c392be50ecc9690562d7282078b2e627dc558aa37f096472543130c11eedf60dd4a4a5c091e6496d86064009583c85d110cb2aed2e151
|
data/.circleci/config.yml
CHANGED
@@ -28,7 +28,7 @@ jobs:
|
|
28
28
|
environment:
|
29
29
|
CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC"
|
30
30
|
JAVA_OPTIONS: "-Djetty.http.port=8998 -Dfcrepo.dynamic.jms.port=61618 -Dfcrepo.dynamic.stomp.port=61614"
|
31
|
-
- image: fcrepo/fcrepo:6.
|
31
|
+
- image: fcrepo/fcrepo:6.4.0
|
32
32
|
environment:
|
33
33
|
CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
|
34
34
|
JAVA_OPTS: "-Djetty.http.port=8978 -Dfcrepo.dynamic.jms.port=61619 -Dfcrepo.dynamic.stomp.port=61615 -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
|
data/.lando.yml
CHANGED
@@ -24,19 +24,26 @@ services:
|
|
24
24
|
- fedora4:/data
|
25
25
|
ports:
|
26
26
|
- 8988:8080
|
27
|
-
|
27
|
+
environment:
|
28
|
+
CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC"
|
29
|
+
portforward: 8988
|
28
30
|
valkyrie_fedora_5:
|
29
31
|
type: compose
|
30
32
|
app_mount: false
|
31
33
|
volumes:
|
32
34
|
fedora5:
|
33
35
|
services:
|
34
|
-
image:
|
35
|
-
command:
|
36
|
+
image: fcrepo/fcrepo:5.1.1-multiplatform
|
37
|
+
command:
|
38
|
+
- "catalina.sh"
|
39
|
+
- "run"
|
36
40
|
volumes:
|
37
41
|
- fedora5:/data
|
38
42
|
ports:
|
39
43
|
- 8998:8080
|
44
|
+
environment:
|
45
|
+
CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
|
46
|
+
JAVA_OPTS: "-Dfcrepo.dynamic.jms.port=61620 -Dfcrepo.dynamic.stomp.port=61617 -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
|
40
47
|
portforward: true
|
41
48
|
valkyrie_fedora_6:
|
42
49
|
type: compose
|
@@ -44,7 +51,7 @@ services:
|
|
44
51
|
volumes:
|
45
52
|
fedora6:
|
46
53
|
services:
|
47
|
-
image: fcrepo/fcrepo:6.
|
54
|
+
image: fcrepo/fcrepo:6.4.0
|
48
55
|
command:
|
49
56
|
- "catalina.sh"
|
50
57
|
- "run"
|
data/.rubocop_todo.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,20 @@
|
|
1
|
+
# v3.1.0 2023-09-15
|
2
|
+
|
3
|
+
## Changes since last release
|
4
|
+
|
5
|
+
* Add storage adapter versioning. ([tpendragon](https://github.com/tpendragon))
|
6
|
+
- Special thanks to [marrus-sh](https://github.com/marrus-sh) and
|
7
|
+
[no-reply](https://github.com/no-reply) for their work testing and
|
8
|
+
finding a good interface for using versions in an application.
|
9
|
+
|
10
|
+
Additional thanks to the following for code review and issue reports leading to
|
11
|
+
this release:
|
12
|
+
|
13
|
+
* [dlpierce](https://github.com/dlpierce)
|
14
|
+
* [hackartisan](https://github.com/hackartisan)
|
15
|
+
* [marrus-sh](https://github.com/marrus-sh)
|
16
|
+
* [no-reply](https://github.com/no-reply)
|
17
|
+
|
1
18
|
# v3.0.3 2023-05-15
|
2
19
|
|
3
20
|
## Changes since last release
|
@@ -16,6 +16,11 @@ RSpec.shared_examples 'a Valkyrie::StorageAdapter' do
|
|
16
16
|
it { is_expected.to respond_to(:find_by).with_keywords(:id) }
|
17
17
|
it { is_expected.to respond_to(:delete).with_keywords(:id) }
|
18
18
|
it { is_expected.to respond_to(:upload).with_keywords(:file, :resource, :original_filename) }
|
19
|
+
it { is_expected.to respond_to(:supports?) }
|
20
|
+
|
21
|
+
it "returns false for non-existing features" do
|
22
|
+
expect(storage_adapter.supports?(:bad_feature_not_real_dont_implement)).to eq false
|
23
|
+
end
|
19
24
|
|
20
25
|
it "can upload a file which is just an IO" do
|
21
26
|
io_file = Tempfile.new('temp_io')
|
@@ -50,7 +55,7 @@ RSpec.shared_examples 'a Valkyrie::StorageAdapter' do
|
|
50
55
|
end
|
51
56
|
|
52
57
|
it "can upload, validate, re-fetch, and delete a file" do
|
53
|
-
resource = Valkyrie::Specs::CustomResource.new(id: "test")
|
58
|
+
resource = Valkyrie::Specs::CustomResource.new(id: "test#{SecureRandom.uuid}")
|
54
59
|
sha1 = Digest::SHA1.file(file).to_s
|
55
60
|
size = file.size
|
56
61
|
expect(uploaded_file = storage_adapter.upload(file: file, original_filename: 'foo.jpg', resource: resource, fake_upload_argument: true)).to be_kind_of Valkyrie::StorageAdapter::File
|
@@ -77,4 +82,64 @@ RSpec.shared_examples 'a Valkyrie::StorageAdapter' do
|
|
77
82
|
expect { storage_adapter.find_by(id: uploaded_file.id) }.to raise_error Valkyrie::StorageAdapter::FileNotFound
|
78
83
|
expect { storage_adapter.find_by(id: Valkyrie::ID.new("noexist")) }.to raise_error Valkyrie::StorageAdapter::FileNotFound
|
79
84
|
end
|
85
|
+
|
86
|
+
it "can upload and find new versions" do
|
87
|
+
pending "Versioning not supported" unless storage_adapter.supports?(:versions)
|
88
|
+
resource = Valkyrie::Specs::CustomResource.new(id: "test#{SecureRandom.uuid}")
|
89
|
+
uploaded_file = storage_adapter.upload(file: file, original_filename: 'foo.jpg', resource: resource, fake_upload_argument: true)
|
90
|
+
expect(uploaded_file.version_id).not_to be_blank
|
91
|
+
|
92
|
+
f = Tempfile.new
|
93
|
+
f.puts "Test File"
|
94
|
+
f.rewind
|
95
|
+
|
96
|
+
# upload_version
|
97
|
+
new_version = storage_adapter.upload_version(id: uploaded_file.id, file: f)
|
98
|
+
expect(uploaded_file.id).to eq new_version.id
|
99
|
+
expect(uploaded_file.version_id).not_to eq new_version.version_id
|
100
|
+
|
101
|
+
# find_versions
|
102
|
+
# Two versions of the same file have the same id, but different version_ids,
|
103
|
+
# use case: I want to store metadata about a file when it's uploaded as a
|
104
|
+
# version and refer to it consistently.
|
105
|
+
versions = storage_adapter.find_versions(id: new_version.id)
|
106
|
+
expect(versions.length).to eq 2
|
107
|
+
expect(versions.first.id).to eq new_version.id
|
108
|
+
expect(versions.first.version_id).to eq new_version.version_id
|
109
|
+
|
110
|
+
expect(versions.last.id).to eq uploaded_file.id
|
111
|
+
expect(versions.last.version_id).to eq uploaded_file.version_id
|
112
|
+
|
113
|
+
expect(versions.first.size).not_to eq versions.last.size
|
114
|
+
|
115
|
+
expect(storage_adapter.find_by(id: uploaded_file.version_id).version_id).to eq uploaded_file.version_id
|
116
|
+
|
117
|
+
# Deleting a version should leave the current versions
|
118
|
+
if storage_adapter.supports?(:version_deletion)
|
119
|
+
storage_adapter.delete(id: uploaded_file.version_id)
|
120
|
+
expect(storage_adapter.find_versions(id: uploaded_file.id).length).to eq 1
|
121
|
+
expect { storage_adapter.find_by(id: uploaded_file.version_id) }.to raise_error Valkyrie::StorageAdapter::FileNotFound
|
122
|
+
end
|
123
|
+
current_length = storage_adapter.find_versions(id: new_version.id).length
|
124
|
+
|
125
|
+
# Restoring a previous version is just pumping its file into upload_version
|
126
|
+
newest_version = storage_adapter.upload_version(file: new_version, id: new_version.id)
|
127
|
+
expect(newest_version.version_id).not_to eq new_version.id
|
128
|
+
expect(storage_adapter.find_by(id: newest_version.id).version_id).to eq newest_version.version_id
|
129
|
+
|
130
|
+
# I can restore a version twice
|
131
|
+
newest_version = storage_adapter.upload_version(file: new_version, id: new_version.id)
|
132
|
+
expect(newest_version.version_id).not_to eq new_version.id
|
133
|
+
expect(storage_adapter.find_by(id: newest_version.id).version_id).to eq newest_version.version_id
|
134
|
+
expect(storage_adapter.find_versions(id: newest_version.id).length).to eq current_length + 2
|
135
|
+
|
136
|
+
# NOTE: We originally wanted deleting the current record to push it into the
|
137
|
+
# versions history, but FCRepo 4/5/6 doesn't work that way, so we changed to
|
138
|
+
# instead make deleting delete everything.
|
139
|
+
storage_adapter.delete(id: new_version.id)
|
140
|
+
expect { storage_adapter.find_by(id: new_version.id) }.to raise_error Valkyrie::StorageAdapter::FileNotFound
|
141
|
+
expect(storage_adapter.find_versions(id: new_version.id).length).to eq 0
|
142
|
+
ensure
|
143
|
+
f&.close
|
144
|
+
end
|
80
145
|
end
|
@@ -27,6 +27,12 @@ module Valkyrie::Storage
|
|
27
27
|
id.to_s.start_with?("disk://#{base_path}")
|
28
28
|
end
|
29
29
|
|
30
|
+
# @param feature [Symbol] Feature to test for.
|
31
|
+
# @return [Boolean] true if the adapter supports the given feature
|
32
|
+
def supports?(_feature)
|
33
|
+
false
|
34
|
+
end
|
35
|
+
|
30
36
|
def file_path(id)
|
31
37
|
id.to_s.gsub(/^disk:\/\//, '')
|
32
38
|
end
|
@@ -19,12 +19,21 @@ module Valkyrie::Storage
|
|
19
19
|
id.to_s.start_with?(PROTOCOL)
|
20
20
|
end
|
21
21
|
|
22
|
+
# @param feature [Symbol] Feature to test for.
|
23
|
+
# @return [Boolean] true if the adapter supports the given feature
|
24
|
+
def supports?(feature)
|
25
|
+
return true if feature == :versions
|
26
|
+
# Fedora 6 auto versions and you can't delete versions.
|
27
|
+
return true if feature == :version_deletion && fedora_version != 6
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
22
31
|
# Return the file associated with the given identifier
|
23
32
|
# @param id [Valkyrie::ID]
|
24
33
|
# @return [Valkyrie::StorageAdapter::StreamFile]
|
25
34
|
# @raise Valkyrie::StorageAdapter::FileNotFound if nothing is found
|
26
35
|
def find_by(id:)
|
27
|
-
|
36
|
+
perform_find(id: id)
|
28
37
|
end
|
29
38
|
|
30
39
|
# @param file [IO]
|
@@ -37,24 +46,123 @@ module Valkyrie::Storage
|
|
37
46
|
def upload(file:, original_filename:, resource:, content_type: "application/octet-stream", # rubocop:disable Metrics/ParameterLists
|
38
47
|
resource_uri_transformer: default_resource_uri_transformer, **_extra_arguments)
|
39
48
|
identifier = resource_uri_transformer.call(resource, base_url) + '/original'
|
49
|
+
upload_file(fedora_uri: identifier, io: file, content_type: content_type, original_filename: original_filename)
|
50
|
+
# Fedora 6 auto versions, so check to see if there's a version for this
|
51
|
+
# initial upload. If not, then mint one (fedora 4/5)
|
52
|
+
version_id = current_version_id(id: valkyrie_identifier(uri: identifier)) || mint_version(identifier, latest_version(identifier))
|
53
|
+
perform_find(id: Valkyrie::ID.new(identifier.to_s.sub(/^.+\/\//, PROTOCOL)), version_id: version_id)
|
54
|
+
end
|
55
|
+
|
56
|
+
# @param id [Valkyrie::ID] ID of the Valkyrie::StorageAdapter::StreamFile to
|
57
|
+
# version.
|
58
|
+
# @param file [IO]
|
59
|
+
def upload_version(id:, file:)
|
60
|
+
uri = fedora_identifier(id: id)
|
61
|
+
# Fedora 6 has auto versioning, so have to sleep if it's too soon after last
|
62
|
+
# upload.
|
63
|
+
if fedora_version == 6 && current_version_id(id: id).to_s.split("/").last == Time.current.utc.strftime("%Y%m%d%H%M%S")
|
64
|
+
sleep(0.5)
|
65
|
+
return upload_version(id: id, file: file)
|
66
|
+
end
|
67
|
+
upload_file(fedora_uri: uri, io: file)
|
68
|
+
version_id = mint_version(uri, latest_version(uri))
|
69
|
+
perform_find(id: Valkyrie::ID.new(uri.to_s.sub(/^.+\/\//, PROTOCOL)), version_id: version_id)
|
70
|
+
end
|
71
|
+
|
72
|
+
# @param id [Valkyrie::ID]
|
73
|
+
# @return [Array<Valkyrie::StorageAdapter::StreamFile>]
|
74
|
+
def find_versions(id:)
|
75
|
+
uri = fedora_identifier(id: id)
|
76
|
+
version_list = version_list(uri)
|
77
|
+
version_list.map do |version|
|
78
|
+
id = valkyrie_identifier(uri: version["@id"])
|
79
|
+
perform_find(id: id, version_id: id)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# Delete the file in Fedora associated with the given identifier.
|
84
|
+
# @param id [Valkyrie::ID]
|
85
|
+
def delete(id:)
|
86
|
+
connection.http.delete(fedora_identifier(id: id))
|
87
|
+
end
|
88
|
+
|
89
|
+
def version_list(fedora_uri)
|
90
|
+
version_list = connection.http.get do |request|
|
91
|
+
request.url "#{fedora_uri}/fcr:versions"
|
92
|
+
request.headers["Accept"] = "application/ld+json"
|
93
|
+
end
|
94
|
+
return [] unless version_list.success?
|
95
|
+
version_graph = JSON.parse(version_list.body)&.first
|
96
|
+
if fedora_version == 4
|
97
|
+
version_graph&.fetch("http://fedora.info/definitions/v4/repository#hasVersion", [])
|
98
|
+
else
|
99
|
+
# Fedora 5/6 use Memento.
|
100
|
+
version_graph&.fetch("http://www.w3.org/ns/ldp#contains", [])&.sort_by { |x| x["@id"] }&.reverse
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def upload_file(fedora_uri:, io:, content_type: "application/octet-stream", original_filename: "default")
|
40
105
|
sha1 = [5, 6].include?(fedora_version) ? "sha" : "sha1"
|
41
106
|
connection.http.put do |request|
|
42
|
-
request.url
|
107
|
+
request.url fedora_uri
|
43
108
|
request.headers['Content-Type'] = content_type
|
44
|
-
request.headers['Content-Length'] =
|
109
|
+
request.headers['Content-Length'] = io.length.to_s if io.respond_to?(:length)
|
45
110
|
request.headers['Content-Disposition'] = "attachment; filename=\"#{original_filename}\""
|
46
|
-
request.headers['digest'] = "#{sha1}=#{Digest::SHA1.file(
|
111
|
+
request.headers['digest'] = "#{sha1}=#{Digest::SHA1.file(io)}" if io.respond_to?(:to_str)
|
47
112
|
request.headers['link'] = "<http://www.w3.org/ns/ldp#NonRDFSource>; rel=\"type\""
|
48
|
-
io = Faraday::UploadIO.new(
|
113
|
+
io = Faraday::UploadIO.new(io, content_type, original_filename)
|
49
114
|
request.body = io
|
50
115
|
end
|
51
|
-
find_by(id: Valkyrie::ID.new(identifier.to_s.sub(/^.+\/\//, PROTOCOL)))
|
52
116
|
end
|
53
117
|
|
54
|
-
#
|
55
|
-
#
|
56
|
-
def
|
57
|
-
|
118
|
+
# Returns a new version identifier to mint. Defaults to version1, but will
|
119
|
+
# increment to version2 etc if one found. Only for Fedora 4.
|
120
|
+
def latest_version(identifier)
|
121
|
+
# Only version 4 needs a version ID, 5/6 both mint using timestamps.
|
122
|
+
return :not_applicable if fedora_version != 4
|
123
|
+
version_list = version_list(identifier)
|
124
|
+
return "version1" if version_list.blank?
|
125
|
+
last_version = version_list.first["@id"]
|
126
|
+
last_version_number = last_version.split("/").last.gsub("version", "").to_i
|
127
|
+
"version#{last_version_number + 1}"
|
128
|
+
end
|
129
|
+
|
130
|
+
# @param [Valkyrie::ID] id A storage ID that's not a version, to get the
|
131
|
+
# version ID of.
|
132
|
+
def current_version_id(id:)
|
133
|
+
version_list = version_list(fedora_identifier(id: id))
|
134
|
+
return nil if version_list.blank?
|
135
|
+
valkyrie_identifier(uri: version_list.first["@id"])
|
136
|
+
end
|
137
|
+
|
138
|
+
def perform_find(id:, version_id: nil)
|
139
|
+
current_id = Valkyrie::ID.new(id.to_s.split("/fcr:versions").first)
|
140
|
+
version_id ||= id if id != current_id
|
141
|
+
# No version got passed and we're asking for a current_id, gotta get the
|
142
|
+
# version ID
|
143
|
+
return perform_find(id: current_id, version_id: (current_version_id(id: id) || :empty)) if version_id.nil?
|
144
|
+
Valkyrie::StorageAdapter::StreamFile.new(id: current_id, io: response(id: id), version_id: version_id)
|
145
|
+
end
|
146
|
+
|
147
|
+
# @param identifier [String] Fedora URI to mint a version for.
|
148
|
+
# @return [Valkyrie::ID] version_id of the minted version.
|
149
|
+
# Versions are created AFTER content is uploaded, except for Fedora 6 which
|
150
|
+
# auto versions.
|
151
|
+
def mint_version(identifier, version_name = "version1")
|
152
|
+
response = connection.http.post do |request|
|
153
|
+
request.url "#{identifier}/fcr:versions"
|
154
|
+
request.headers['Slug'] = version_name if fedora_version == 4
|
155
|
+
end
|
156
|
+
# If there's a deletion marker, don't return anything. (Fedora 4)
|
157
|
+
return nil if response.status == 410
|
158
|
+
# This is awful, but versioning is locked to per-second increments,
|
159
|
+
# returns a 409 in Fedora 5 if there's a conflict.
|
160
|
+
if response.status == 409
|
161
|
+
sleep(0.5)
|
162
|
+
return mint_version(identifier, version_name)
|
163
|
+
end
|
164
|
+
raise "Version unable to be created" unless response.status == 201
|
165
|
+
valkyrie_identifier(uri: response.headers["location"].gsub("/fcr:metadata", ""))
|
58
166
|
end
|
59
167
|
|
60
168
|
class IOProxy
|
@@ -81,6 +189,11 @@ module Valkyrie::Storage
|
|
81
189
|
RDF::URI(identifier)
|
82
190
|
end
|
83
191
|
|
192
|
+
def valkyrie_identifier(uri:)
|
193
|
+
id = uri.to_s.sub("http://", PROTOCOL)
|
194
|
+
Valkyrie::ID.new(id)
|
195
|
+
end
|
196
|
+
|
84
197
|
private
|
85
198
|
|
86
199
|
# @return [IOProxy]
|
@@ -17,7 +17,32 @@ module Valkyrie::Storage
|
|
17
17
|
# @return [Valkyrie::StorageAdapter::StreamFile]
|
18
18
|
def upload(file:, original_filename:, resource: nil, **_extra_arguments)
|
19
19
|
identifier = Valkyrie::ID.new("memory://#{resource.id}")
|
20
|
-
|
20
|
+
version_id = Valkyrie::ID.new("#{identifier}##{SecureRandom.uuid}")
|
21
|
+
cache[identifier] ||= {}
|
22
|
+
cache[identifier][:current] = Valkyrie::StorageAdapter::StreamFile.new(id: identifier, io: file, version_id: version_id)
|
23
|
+
end
|
24
|
+
|
25
|
+
# @param file [IO]
|
26
|
+
# @param original_filename [String]
|
27
|
+
# @param previous_version_id [Valkyrie::ID]
|
28
|
+
# @param _extra_arguments [Hash] additional arguments which may be passed to
|
29
|
+
# other adapters.
|
30
|
+
# @return [Valkyrie::StorageAdapter::StreamFile]
|
31
|
+
def upload_version(id:, file:)
|
32
|
+
# Get previous file and add a UUID to the end of it.
|
33
|
+
new_file = Valkyrie::StorageAdapter::StreamFile.new(id: id, io: file, version_id: Valkyrie::ID.new("#{id}##{SecureRandom.uuid}"))
|
34
|
+
current_file = cache[id][:current]
|
35
|
+
cache[id][:current] = new_file
|
36
|
+
cache[id][:versions] ||= []
|
37
|
+
cache[id][:versions].prepend(current_file) if current_file
|
38
|
+
new_file
|
39
|
+
end
|
40
|
+
|
41
|
+
# @param id [Valkyrie::ID]
|
42
|
+
# @return [Array<Valkyrie::StorageAdapter::StreamFile>]
|
43
|
+
def find_versions(id:)
|
44
|
+
return [] if cache[id].nil?
|
45
|
+
[cache[id][:current] || nil].compact + cache[id].fetch(:versions, [])
|
21
46
|
end
|
22
47
|
|
23
48
|
# Return the file associated with the given identifier
|
@@ -25,8 +50,18 @@ module Valkyrie::Storage
|
|
25
50
|
# @return [Valkyrie::StorageAdapter::StreamFile]
|
26
51
|
# @raise Valkyrie::StorageAdapter::FileNotFound if nothing is found
|
27
52
|
def find_by(id:)
|
28
|
-
|
29
|
-
cache[
|
53
|
+
no_version_id, _version = id_and_version(id)
|
54
|
+
raise Valkyrie::StorageAdapter::FileNotFound unless cache[no_version_id]
|
55
|
+
version =
|
56
|
+
if id == no_version_id
|
57
|
+
cache[id][:current]
|
58
|
+
else
|
59
|
+
find_versions(id: no_version_id).find do |file|
|
60
|
+
file.version_id == id
|
61
|
+
end
|
62
|
+
end
|
63
|
+
raise Valkyrie::StorageAdapter::FileNotFound unless version
|
64
|
+
version
|
30
65
|
end
|
31
66
|
|
32
67
|
# @param id [Valkyrie::ID]
|
@@ -35,10 +70,35 @@ module Valkyrie::Storage
|
|
35
70
|
id.to_s.start_with?("memory://")
|
36
71
|
end
|
37
72
|
|
73
|
+
# @param feature [Symbol] Feature to test for.
|
74
|
+
# @return [Boolean] true if the adapter supports the given feature
|
75
|
+
def supports?(feature)
|
76
|
+
case feature
|
77
|
+
when :versions
|
78
|
+
true
|
79
|
+
when :version_deletion
|
80
|
+
true
|
81
|
+
else
|
82
|
+
false
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def id_and_version(id)
|
87
|
+
id, version = id.to_s.split("#")
|
88
|
+
[Valkyrie::ID.new(id), version]
|
89
|
+
end
|
90
|
+
|
38
91
|
# Delete the file on disk associated with the given identifier.
|
39
92
|
# @param id [Valkyrie::ID]
|
40
93
|
def delete(id:)
|
41
|
-
|
94
|
+
base_id, version = id_and_version(id)
|
95
|
+
if version && cache[base_id][:current]&.version_id != id
|
96
|
+
cache[base_id][:versions].reject! do |file|
|
97
|
+
file.version_id == id
|
98
|
+
end
|
99
|
+
else
|
100
|
+
cache.delete(base_id)
|
101
|
+
end
|
42
102
|
nil
|
43
103
|
end
|
44
104
|
end
|
@@ -0,0 +1,182 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Valkyrie::Storage
|
3
|
+
# The VersionedDisk adapter implements versioned storage on disk by storing
|
4
|
+
# the timestamp of the file's creation as part of the file name
|
5
|
+
# (v-timestamp-filename.jpg). If the
|
6
|
+
# current file is deleted it creates a DeletionMarker, which is an empty file
|
7
|
+
# with "deletionmarker" in the name of the file.
|
8
|
+
class VersionedDisk
|
9
|
+
attr_reader :base_path, :path_generator, :file_mover
|
10
|
+
def initialize(base_path:, path_generator: ::Valkyrie::Storage::Disk::BucketedStorage, file_mover: FileUtils.method(:cp))
|
11
|
+
@base_path = Pathname.new(base_path.to_s)
|
12
|
+
@path_generator = path_generator.new(base_path: base_path)
|
13
|
+
@file_mover = file_mover
|
14
|
+
end
|
15
|
+
|
16
|
+
# @param file [IO]
|
17
|
+
# @param original_filename [String]
|
18
|
+
# @param resource [Valkyrie::Resource]
|
19
|
+
# @param _extra_arguments [Hash] additional arguments which may be passed to other adapters
|
20
|
+
# @return [Valkyrie::StorageAdapter::File]
|
21
|
+
def upload(file:, original_filename:, resource: nil, paused: false, **extra_arguments)
|
22
|
+
version_timestamp = current_timestamp
|
23
|
+
new_path = path_generator.generate(resource: resource, file: file, original_filename: "v-#{version_timestamp}-#{original_filename}")
|
24
|
+
# If we've gone faster than milliseconds here, pause a millisecond and
|
25
|
+
# re-call. Probably only an issue for test suites.
|
26
|
+
return sleep(0.001) && upload(file: file, original_filename: original_filename, resource: resource, paused: true, **extra_arguments) if !paused && File.exist?(new_path)
|
27
|
+
FileUtils.mkdir_p(new_path.parent)
|
28
|
+
file_mover.call(file.try(:path) || file.try(:disk_path), new_path)
|
29
|
+
find_by(id: Valkyrie::ID.new("versiondisk://#{new_path}"))
|
30
|
+
end
|
31
|
+
|
32
|
+
def current_timestamp
|
33
|
+
Time.now.strftime("%s%L")
|
34
|
+
end
|
35
|
+
|
36
|
+
# @param id [Valkyrie::ID] ID of the Valkyrie::StorageAdapter::File to
|
37
|
+
# version.
|
38
|
+
# @param file [IO]
|
39
|
+
# @param paused [Boolean] set to true when upload_version had to pause for a
|
40
|
+
# millisecond to get a later timestamp. Internal only - do not set.
|
41
|
+
def upload_version(id:, file:, paused: false)
|
42
|
+
version_timestamp = current_timestamp
|
43
|
+
# Get the existing version_id so we can calculate the next path from it.
|
44
|
+
current_version_id = version_id(id)
|
45
|
+
current_version_id = current_version_id.version_files[1] if current_version_id.deletion_marker?
|
46
|
+
existing_path = current_version_id.file_path
|
47
|
+
new_path = Pathname.new(existing_path.gsub(current_version_id.version, version_timestamp.to_s))
|
48
|
+
# If we've gone faster than milliseconds here, pause a millisecond and
|
49
|
+
# re-call.
|
50
|
+
return sleep(0.001) && upload_version(id: id, file: file, paused: true) if !paused && File.exist?(new_path)
|
51
|
+
FileUtils.mkdir_p(new_path.parent)
|
52
|
+
file_mover.call(file.try(:path) || file.try(:disk_path), new_path)
|
53
|
+
find_by(id: Valkyrie::ID.new("versiondisk://#{new_path}"))
|
54
|
+
end
|
55
|
+
|
56
|
+
# @param id [Valkyrie::ID]
|
57
|
+
# @return [Boolean] true if this adapter can handle this type of identifer
|
58
|
+
def handles?(id:)
|
59
|
+
id.to_s.start_with?("versiondisk://#{base_path}")
|
60
|
+
end
|
61
|
+
|
62
|
+
# @param feature [Symbol] Feature to test for.
|
63
|
+
# @return [Boolean] true if the adapter supports the given feature
|
64
|
+
def supports?(feature)
|
65
|
+
return true if feature == :versions || feature == :version_deletion
|
66
|
+
false
|
67
|
+
end
|
68
|
+
|
69
|
+
# Return the file associated with the given identifier
|
70
|
+
# @param id [Valkyrie::ID]
|
71
|
+
# @return [Valkyrie::StorageAdapter::File]
|
72
|
+
# @raise Valkyrie::StorageAdapter::FileNotFound if nothing is found
|
73
|
+
def find_by(id:)
|
74
|
+
version_id = version_id(id)
|
75
|
+
raise Valkyrie::StorageAdapter::FileNotFound if version_id.nil? || version_id&.deletion_marker?
|
76
|
+
Valkyrie::StorageAdapter::File.new(id: version_id.current_reference_id.id, io: ::Valkyrie::Storage::Disk::LazyFile.open(version_id.file_path, 'rb'), version_id: version_id.id)
|
77
|
+
rescue Errno::ENOENT
|
78
|
+
raise Valkyrie::StorageAdapter::FileNotFound
|
79
|
+
end
|
80
|
+
|
81
|
+
# Delete the file on disk associated with the given identifier.
|
82
|
+
# @param id [Valkyrie::ID]
|
83
|
+
def delete(id:)
|
84
|
+
id = version_id(id).resolve_current
|
85
|
+
if id.current?
|
86
|
+
id.version_files.each do |version_id|
|
87
|
+
FileUtils.rm_rf(version_id.file_path)
|
88
|
+
end
|
89
|
+
elsif File.exist?(id.file_path)
|
90
|
+
FileUtils.rm_rf(id.file_path)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
# @param id [Valkyrie::ID]
|
95
|
+
# @return [Array<Valkyrie::StorageAdapter::File>]
|
96
|
+
def find_versions(id:)
|
97
|
+
version_files(id: id).select { |x| !x.to_s.include?("deletionmarker") }.map do |file|
|
98
|
+
find_by(id: Valkyrie::ID.new("versiondisk://#{file}"))
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def version_files(id:)
|
103
|
+
root = Pathname.new(file_path(id))
|
104
|
+
id = VersionId.new(id)
|
105
|
+
root.parent.children.select { |file| file.basename.to_s.end_with?(id.filename) }.sort.reverse
|
106
|
+
end
|
107
|
+
|
108
|
+
def file_path(version_id)
|
109
|
+
version_id.to_s.gsub(/^versiondisk:\/\//, '')
|
110
|
+
end
|
111
|
+
|
112
|
+
# @return VersionId A VersionId value that's resolved a current reference,
|
113
|
+
# so we can access the `version_id` and current reference.
|
114
|
+
def version_id(id)
|
115
|
+
id = VersionId.new(id)
|
116
|
+
return id unless id.versioned?
|
117
|
+
id.resolve_current
|
118
|
+
end
|
119
|
+
|
120
|
+
# A small value class that holds a version id and methods for knowing things about it.
|
121
|
+
# Examples of version ids in this adapter:
|
122
|
+
# * "versiondisk://te/st/test/v-current-filename.jpg" (never actually saved this way on disk, just used as a reference)
|
123
|
+
# * "versiondisk://te/st/test/v-1694195675462560794-filename.jpg" (this timestamped form would be saved on disk)
|
124
|
+
# * "versiondisk://te/st/test/v-1694195675462560794-deletionmarker-filename.jpg" (this file is saved on disk but empty)
|
125
|
+
class VersionId
|
126
|
+
attr_reader :id
|
127
|
+
def initialize(id)
|
128
|
+
@id = id
|
129
|
+
end
|
130
|
+
|
131
|
+
def current_reference_id
|
132
|
+
self.class.new(Valkyrie::ID.new(string_id.gsub(version, "current")))
|
133
|
+
end
|
134
|
+
|
135
|
+
# @return [VersionID] the version_id for the current file
|
136
|
+
def resolve_current
|
137
|
+
return self unless reference?
|
138
|
+
version_files.first
|
139
|
+
end
|
140
|
+
|
141
|
+
def file_path
|
142
|
+
@file_path ||= string_id.gsub(/^versiondisk:\/\//, '')
|
143
|
+
end
|
144
|
+
|
145
|
+
def version_files
|
146
|
+
root = Pathname.new(file_path)
|
147
|
+
root.parent.children.select { |file| file.basename.to_s.end_with?(filename) }.sort.reverse.map do |file|
|
148
|
+
VersionId.new(Valkyrie::ID.new("versiondisk://#{file}"))
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def deletion_marker?
|
153
|
+
string_id.include?("deletionmarker")
|
154
|
+
end
|
155
|
+
|
156
|
+
def current?
|
157
|
+
version_files.first.id == id
|
158
|
+
end
|
159
|
+
|
160
|
+
# @return [Boolean] Whether this id is referential (e.g. "current") or absolute (e.g. a timestamp)
|
161
|
+
def reference?
|
162
|
+
version == "current"
|
163
|
+
end
|
164
|
+
|
165
|
+
def versioned?
|
166
|
+
string_id.include?("v-")
|
167
|
+
end
|
168
|
+
|
169
|
+
def version
|
170
|
+
string_id.split("v-").last.split("-", 2).first
|
171
|
+
end
|
172
|
+
|
173
|
+
def filename
|
174
|
+
string_id.split("v-").last.split("-", 2).last.gsub("deletionmarker-", "")
|
175
|
+
end
|
176
|
+
|
177
|
+
def string_id
|
178
|
+
id.to_s
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
data/lib/valkyrie/storage.rb
CHANGED
data/lib/valkyrie/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: valkyrie
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Trey Pendragon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-09-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dry-struct
|
@@ -535,6 +535,7 @@ files:
|
|
535
535
|
- lib/valkyrie/storage/disk.rb
|
536
536
|
- lib/valkyrie/storage/fedora.rb
|
537
537
|
- lib/valkyrie/storage/memory.rb
|
538
|
+
- lib/valkyrie/storage/versioned_disk.rb
|
538
539
|
- lib/valkyrie/storage_adapter.rb
|
539
540
|
- lib/valkyrie/types.rb
|
540
541
|
- lib/valkyrie/value_mapper.rb
|