valkyrie 3.0.2 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +1 -1
- data/.lando.yml +11 -4
- data/.rubocop_todo.yml +1 -0
- data/CHANGELOG.md +29 -0
- data/lib/valkyrie/change_set.rb +5 -3
- data/lib/valkyrie/specs/shared_specs/storage_adapter.rb +66 -1
- data/lib/valkyrie/storage/disk.rb +6 -0
- data/lib/valkyrie/storage/fedora.rb +123 -10
- data/lib/valkyrie/storage/memory.rb +64 -4
- data/lib/valkyrie/storage/versioned_disk.rb +182 -0
- data/lib/valkyrie/storage.rb +1 -0
- data/lib/valkyrie/storage_adapter.rb +1 -0
- data/lib/valkyrie/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e6705b30a79e047e4892d78b3f60a2fb71a9acc4b4687272a7e169e216177a81
|
4
|
+
data.tar.gz: c4cbe96c86968b611d1932f505cb020c1071e6fa0938f11b25f4e78c3c7b840b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 80cc406ed49c5256d4744c83eaf0e849722d28b151a6f47b5dfdb9c69871a82253f9b3c2b8331f92b6a253792b1c982f5bfcd9c198cd631f84613e50c57c4d5f
|
7
|
+
data.tar.gz: b7b397512adac100586c392be50ecc9690562d7282078b2e627dc558aa37f096472543130c11eedf60dd4a4a5c091e6496d86064009583c85d110cb2aed2e151
|
data/.circleci/config.yml
CHANGED
@@ -28,7 +28,7 @@ jobs:
|
|
28
28
|
environment:
|
29
29
|
CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC"
|
30
30
|
JAVA_OPTIONS: "-Djetty.http.port=8998 -Dfcrepo.dynamic.jms.port=61618 -Dfcrepo.dynamic.stomp.port=61614"
|
31
|
-
- image: fcrepo/fcrepo:6.
|
31
|
+
- image: fcrepo/fcrepo:6.4.0
|
32
32
|
environment:
|
33
33
|
CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
|
34
34
|
JAVA_OPTS: "-Djetty.http.port=8978 -Dfcrepo.dynamic.jms.port=61619 -Dfcrepo.dynamic.stomp.port=61615 -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
|
data/.lando.yml
CHANGED
@@ -24,19 +24,26 @@ services:
|
|
24
24
|
- fedora4:/data
|
25
25
|
ports:
|
26
26
|
- 8988:8080
|
27
|
-
|
27
|
+
environment:
|
28
|
+
CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC"
|
29
|
+
portforward: 8988
|
28
30
|
valkyrie_fedora_5:
|
29
31
|
type: compose
|
30
32
|
app_mount: false
|
31
33
|
volumes:
|
32
34
|
fedora5:
|
33
35
|
services:
|
34
|
-
image:
|
35
|
-
command:
|
36
|
+
image: fcrepo/fcrepo:5.1.1-multiplatform
|
37
|
+
command:
|
38
|
+
- "catalina.sh"
|
39
|
+
- "run"
|
36
40
|
volumes:
|
37
41
|
- fedora5:/data
|
38
42
|
ports:
|
39
43
|
- 8998:8080
|
44
|
+
environment:
|
45
|
+
CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
|
46
|
+
JAVA_OPTS: "-Dfcrepo.dynamic.jms.port=61620 -Dfcrepo.dynamic.stomp.port=61617 -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
|
40
47
|
portforward: true
|
41
48
|
valkyrie_fedora_6:
|
42
49
|
type: compose
|
@@ -44,7 +51,7 @@ services:
|
|
44
51
|
volumes:
|
45
52
|
fedora6:
|
46
53
|
services:
|
47
|
-
image: fcrepo/fcrepo:6.
|
54
|
+
image: fcrepo/fcrepo:6.4.0
|
48
55
|
command:
|
49
56
|
- "catalina.sh"
|
50
57
|
- "run"
|
data/.rubocop_todo.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,32 @@
|
|
1
|
+
# v3.1.0 2023-09-15
|
2
|
+
|
3
|
+
## Changes since last release
|
4
|
+
|
5
|
+
* Add storage adapter versioning. ([tpendragon](https://github.com/tpendragon))
|
6
|
+
- Special thanks to [marrus-sh](https://github.com/marrus-sh) and
|
7
|
+
[no-reply](https://github.com/no-reply) for their work testing and
|
8
|
+
finding a good interface for using versions in an application.
|
9
|
+
|
10
|
+
Additional thanks to the following for code review and issue reports leading to
|
11
|
+
this release:
|
12
|
+
|
13
|
+
* [dlpierce](https://github.com/dlpierce)
|
14
|
+
* [hackartisan](https://github.com/hackartisan)
|
15
|
+
* [marrus-sh](https://github.com/marrus-sh)
|
16
|
+
* [no-reply](https://github.com/no-reply)
|
17
|
+
|
18
|
+
# v3.0.3 2023-05-15
|
19
|
+
|
20
|
+
## Changes since last release
|
21
|
+
|
22
|
+
* Avoid `delegate` in ChangeSet. ([dlpierce](https://github.com/dlpierce))
|
23
|
+
|
24
|
+
Additional thanks to the following for code review and issue reports leading to
|
25
|
+
this release:
|
26
|
+
|
27
|
+
* [ShanaLMoore](https://github.com/ShanaLMoore)
|
28
|
+
* [jeremyf](https://github.com/jeremyf)
|
29
|
+
|
1
30
|
# v3.0.2 2023-04-10
|
2
31
|
|
3
32
|
## Changes since last release
|
data/lib/valkyrie/change_set.rb
CHANGED
@@ -77,9 +77,11 @@ module Valkyrie
|
|
77
77
|
send(key) if respond_to?(key)
|
78
78
|
end
|
79
79
|
|
80
|
-
|
81
|
-
|
82
|
-
|
80
|
+
[:internal_resource, :created_at, :updated_at, :model_name, :optimistic_locking_enabled?, :attributes].each do |method_name|
|
81
|
+
define_method(method_name) do |*args|
|
82
|
+
resource.public_send(method_name, *args)
|
83
|
+
end
|
84
|
+
end
|
83
85
|
|
84
86
|
# Prepopulates all fields with defaults defined in the changeset. This is an
|
85
87
|
# override of Reform::Form's method to allow for single-valued fields to
|
@@ -16,6 +16,11 @@ RSpec.shared_examples 'a Valkyrie::StorageAdapter' do
|
|
16
16
|
it { is_expected.to respond_to(:find_by).with_keywords(:id) }
|
17
17
|
it { is_expected.to respond_to(:delete).with_keywords(:id) }
|
18
18
|
it { is_expected.to respond_to(:upload).with_keywords(:file, :resource, :original_filename) }
|
19
|
+
it { is_expected.to respond_to(:supports?) }
|
20
|
+
|
21
|
+
it "returns false for non-existing features" do
|
22
|
+
expect(storage_adapter.supports?(:bad_feature_not_real_dont_implement)).to eq false
|
23
|
+
end
|
19
24
|
|
20
25
|
it "can upload a file which is just an IO" do
|
21
26
|
io_file = Tempfile.new('temp_io')
|
@@ -50,7 +55,7 @@ RSpec.shared_examples 'a Valkyrie::StorageAdapter' do
|
|
50
55
|
end
|
51
56
|
|
52
57
|
it "can upload, validate, re-fetch, and delete a file" do
|
53
|
-
resource = Valkyrie::Specs::CustomResource.new(id: "test")
|
58
|
+
resource = Valkyrie::Specs::CustomResource.new(id: "test#{SecureRandom.uuid}")
|
54
59
|
sha1 = Digest::SHA1.file(file).to_s
|
55
60
|
size = file.size
|
56
61
|
expect(uploaded_file = storage_adapter.upload(file: file, original_filename: 'foo.jpg', resource: resource, fake_upload_argument: true)).to be_kind_of Valkyrie::StorageAdapter::File
|
@@ -77,4 +82,64 @@ RSpec.shared_examples 'a Valkyrie::StorageAdapter' do
|
|
77
82
|
expect { storage_adapter.find_by(id: uploaded_file.id) }.to raise_error Valkyrie::StorageAdapter::FileNotFound
|
78
83
|
expect { storage_adapter.find_by(id: Valkyrie::ID.new("noexist")) }.to raise_error Valkyrie::StorageAdapter::FileNotFound
|
79
84
|
end
|
85
|
+
|
86
|
+
it "can upload and find new versions" do
|
87
|
+
pending "Versioning not supported" unless storage_adapter.supports?(:versions)
|
88
|
+
resource = Valkyrie::Specs::CustomResource.new(id: "test#{SecureRandom.uuid}")
|
89
|
+
uploaded_file = storage_adapter.upload(file: file, original_filename: 'foo.jpg', resource: resource, fake_upload_argument: true)
|
90
|
+
expect(uploaded_file.version_id).not_to be_blank
|
91
|
+
|
92
|
+
f = Tempfile.new
|
93
|
+
f.puts "Test File"
|
94
|
+
f.rewind
|
95
|
+
|
96
|
+
# upload_version
|
97
|
+
new_version = storage_adapter.upload_version(id: uploaded_file.id, file: f)
|
98
|
+
expect(uploaded_file.id).to eq new_version.id
|
99
|
+
expect(uploaded_file.version_id).not_to eq new_version.version_id
|
100
|
+
|
101
|
+
# find_versions
|
102
|
+
# Two versions of the same file have the same id, but different version_ids,
|
103
|
+
# use case: I want to store metadata about a file when it's uploaded as a
|
104
|
+
# version and refer to it consistently.
|
105
|
+
versions = storage_adapter.find_versions(id: new_version.id)
|
106
|
+
expect(versions.length).to eq 2
|
107
|
+
expect(versions.first.id).to eq new_version.id
|
108
|
+
expect(versions.first.version_id).to eq new_version.version_id
|
109
|
+
|
110
|
+
expect(versions.last.id).to eq uploaded_file.id
|
111
|
+
expect(versions.last.version_id).to eq uploaded_file.version_id
|
112
|
+
|
113
|
+
expect(versions.first.size).not_to eq versions.last.size
|
114
|
+
|
115
|
+
expect(storage_adapter.find_by(id: uploaded_file.version_id).version_id).to eq uploaded_file.version_id
|
116
|
+
|
117
|
+
# Deleting a version should leave the current versions
|
118
|
+
if storage_adapter.supports?(:version_deletion)
|
119
|
+
storage_adapter.delete(id: uploaded_file.version_id)
|
120
|
+
expect(storage_adapter.find_versions(id: uploaded_file.id).length).to eq 1
|
121
|
+
expect { storage_adapter.find_by(id: uploaded_file.version_id) }.to raise_error Valkyrie::StorageAdapter::FileNotFound
|
122
|
+
end
|
123
|
+
current_length = storage_adapter.find_versions(id: new_version.id).length
|
124
|
+
|
125
|
+
# Restoring a previous version is just pumping its file into upload_version
|
126
|
+
newest_version = storage_adapter.upload_version(file: new_version, id: new_version.id)
|
127
|
+
expect(newest_version.version_id).not_to eq new_version.id
|
128
|
+
expect(storage_adapter.find_by(id: newest_version.id).version_id).to eq newest_version.version_id
|
129
|
+
|
130
|
+
# I can restore a version twice
|
131
|
+
newest_version = storage_adapter.upload_version(file: new_version, id: new_version.id)
|
132
|
+
expect(newest_version.version_id).not_to eq new_version.id
|
133
|
+
expect(storage_adapter.find_by(id: newest_version.id).version_id).to eq newest_version.version_id
|
134
|
+
expect(storage_adapter.find_versions(id: newest_version.id).length).to eq current_length + 2
|
135
|
+
|
136
|
+
# NOTE: We originally wanted deleting the current record to push it into the
|
137
|
+
# versions history, but FCRepo 4/5/6 doesn't work that way, so we changed to
|
138
|
+
# instead make deleting delete everything.
|
139
|
+
storage_adapter.delete(id: new_version.id)
|
140
|
+
expect { storage_adapter.find_by(id: new_version.id) }.to raise_error Valkyrie::StorageAdapter::FileNotFound
|
141
|
+
expect(storage_adapter.find_versions(id: new_version.id).length).to eq 0
|
142
|
+
ensure
|
143
|
+
f&.close
|
144
|
+
end
|
80
145
|
end
|
@@ -27,6 +27,12 @@ module Valkyrie::Storage
|
|
27
27
|
id.to_s.start_with?("disk://#{base_path}")
|
28
28
|
end
|
29
29
|
|
30
|
+
# @param feature [Symbol] Feature to test for.
|
31
|
+
# @return [Boolean] true if the adapter supports the given feature
|
32
|
+
def supports?(_feature)
|
33
|
+
false
|
34
|
+
end
|
35
|
+
|
30
36
|
def file_path(id)
|
31
37
|
id.to_s.gsub(/^disk:\/\//, '')
|
32
38
|
end
|
@@ -19,12 +19,21 @@ module Valkyrie::Storage
|
|
19
19
|
id.to_s.start_with?(PROTOCOL)
|
20
20
|
end
|
21
21
|
|
22
|
+
# @param feature [Symbol] Feature to test for.
|
23
|
+
# @return [Boolean] true if the adapter supports the given feature
|
24
|
+
def supports?(feature)
|
25
|
+
return true if feature == :versions
|
26
|
+
# Fedora 6 auto versions and you can't delete versions.
|
27
|
+
return true if feature == :version_deletion && fedora_version != 6
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
22
31
|
# Return the file associated with the given identifier
|
23
32
|
# @param id [Valkyrie::ID]
|
24
33
|
# @return [Valkyrie::StorageAdapter::StreamFile]
|
25
34
|
# @raise Valkyrie::StorageAdapter::FileNotFound if nothing is found
|
26
35
|
def find_by(id:)
|
27
|
-
|
36
|
+
perform_find(id: id)
|
28
37
|
end
|
29
38
|
|
30
39
|
# @param file [IO]
|
@@ -37,24 +46,123 @@ module Valkyrie::Storage
|
|
37
46
|
def upload(file:, original_filename:, resource:, content_type: "application/octet-stream", # rubocop:disable Metrics/ParameterLists
|
38
47
|
resource_uri_transformer: default_resource_uri_transformer, **_extra_arguments)
|
39
48
|
identifier = resource_uri_transformer.call(resource, base_url) + '/original'
|
49
|
+
upload_file(fedora_uri: identifier, io: file, content_type: content_type, original_filename: original_filename)
|
50
|
+
# Fedora 6 auto versions, so check to see if there's a version for this
|
51
|
+
# initial upload. If not, then mint one (fedora 4/5)
|
52
|
+
version_id = current_version_id(id: valkyrie_identifier(uri: identifier)) || mint_version(identifier, latest_version(identifier))
|
53
|
+
perform_find(id: Valkyrie::ID.new(identifier.to_s.sub(/^.+\/\//, PROTOCOL)), version_id: version_id)
|
54
|
+
end
|
55
|
+
|
56
|
+
# @param id [Valkyrie::ID] ID of the Valkyrie::StorageAdapter::StreamFile to
|
57
|
+
# version.
|
58
|
+
# @param file [IO]
|
59
|
+
def upload_version(id:, file:)
|
60
|
+
uri = fedora_identifier(id: id)
|
61
|
+
# Fedora 6 has auto versioning, so have to sleep if it's too soon after last
|
62
|
+
# upload.
|
63
|
+
if fedora_version == 6 && current_version_id(id: id).to_s.split("/").last == Time.current.utc.strftime("%Y%m%d%H%M%S")
|
64
|
+
sleep(0.5)
|
65
|
+
return upload_version(id: id, file: file)
|
66
|
+
end
|
67
|
+
upload_file(fedora_uri: uri, io: file)
|
68
|
+
version_id = mint_version(uri, latest_version(uri))
|
69
|
+
perform_find(id: Valkyrie::ID.new(uri.to_s.sub(/^.+\/\//, PROTOCOL)), version_id: version_id)
|
70
|
+
end
|
71
|
+
|
72
|
+
# @param id [Valkyrie::ID]
|
73
|
+
# @return [Array<Valkyrie::StorageAdapter::StreamFile>]
|
74
|
+
def find_versions(id:)
|
75
|
+
uri = fedora_identifier(id: id)
|
76
|
+
version_list = version_list(uri)
|
77
|
+
version_list.map do |version|
|
78
|
+
id = valkyrie_identifier(uri: version["@id"])
|
79
|
+
perform_find(id: id, version_id: id)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# Delete the file in Fedora associated with the given identifier.
|
84
|
+
# @param id [Valkyrie::ID]
|
85
|
+
def delete(id:)
|
86
|
+
connection.http.delete(fedora_identifier(id: id))
|
87
|
+
end
|
88
|
+
|
89
|
+
def version_list(fedora_uri)
|
90
|
+
version_list = connection.http.get do |request|
|
91
|
+
request.url "#{fedora_uri}/fcr:versions"
|
92
|
+
request.headers["Accept"] = "application/ld+json"
|
93
|
+
end
|
94
|
+
return [] unless version_list.success?
|
95
|
+
version_graph = JSON.parse(version_list.body)&.first
|
96
|
+
if fedora_version == 4
|
97
|
+
version_graph&.fetch("http://fedora.info/definitions/v4/repository#hasVersion", [])
|
98
|
+
else
|
99
|
+
# Fedora 5/6 use Memento.
|
100
|
+
version_graph&.fetch("http://www.w3.org/ns/ldp#contains", [])&.sort_by { |x| x["@id"] }&.reverse
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def upload_file(fedora_uri:, io:, content_type: "application/octet-stream", original_filename: "default")
|
40
105
|
sha1 = [5, 6].include?(fedora_version) ? "sha" : "sha1"
|
41
106
|
connection.http.put do |request|
|
42
|
-
request.url
|
107
|
+
request.url fedora_uri
|
43
108
|
request.headers['Content-Type'] = content_type
|
44
|
-
request.headers['Content-Length'] =
|
109
|
+
request.headers['Content-Length'] = io.length.to_s if io.respond_to?(:length)
|
45
110
|
request.headers['Content-Disposition'] = "attachment; filename=\"#{original_filename}\""
|
46
|
-
request.headers['digest'] = "#{sha1}=#{Digest::SHA1.file(
|
111
|
+
request.headers['digest'] = "#{sha1}=#{Digest::SHA1.file(io)}" if io.respond_to?(:to_str)
|
47
112
|
request.headers['link'] = "<http://www.w3.org/ns/ldp#NonRDFSource>; rel=\"type\""
|
48
|
-
io = Faraday::UploadIO.new(
|
113
|
+
io = Faraday::UploadIO.new(io, content_type, original_filename)
|
49
114
|
request.body = io
|
50
115
|
end
|
51
|
-
find_by(id: Valkyrie::ID.new(identifier.to_s.sub(/^.+\/\//, PROTOCOL)))
|
52
116
|
end
|
53
117
|
|
54
|
-
#
|
55
|
-
#
|
56
|
-
def
|
57
|
-
|
118
|
+
# Returns a new version identifier to mint. Defaults to version1, but will
|
119
|
+
# increment to version2 etc if one found. Only for Fedora 4.
|
120
|
+
def latest_version(identifier)
|
121
|
+
# Only version 4 needs a version ID, 5/6 both mint using timestamps.
|
122
|
+
return :not_applicable if fedora_version != 4
|
123
|
+
version_list = version_list(identifier)
|
124
|
+
return "version1" if version_list.blank?
|
125
|
+
last_version = version_list.first["@id"]
|
126
|
+
last_version_number = last_version.split("/").last.gsub("version", "").to_i
|
127
|
+
"version#{last_version_number + 1}"
|
128
|
+
end
|
129
|
+
|
130
|
+
# @param [Valkyrie::ID] id A storage ID that's not a version, to get the
|
131
|
+
# version ID of.
|
132
|
+
def current_version_id(id:)
|
133
|
+
version_list = version_list(fedora_identifier(id: id))
|
134
|
+
return nil if version_list.blank?
|
135
|
+
valkyrie_identifier(uri: version_list.first["@id"])
|
136
|
+
end
|
137
|
+
|
138
|
+
def perform_find(id:, version_id: nil)
|
139
|
+
current_id = Valkyrie::ID.new(id.to_s.split("/fcr:versions").first)
|
140
|
+
version_id ||= id if id != current_id
|
141
|
+
# No version got passed and we're asking for a current_id, gotta get the
|
142
|
+
# version ID
|
143
|
+
return perform_find(id: current_id, version_id: (current_version_id(id: id) || :empty)) if version_id.nil?
|
144
|
+
Valkyrie::StorageAdapter::StreamFile.new(id: current_id, io: response(id: id), version_id: version_id)
|
145
|
+
end
|
146
|
+
|
147
|
+
# @param identifier [String] Fedora URI to mint a version for.
|
148
|
+
# @return [Valkyrie::ID] version_id of the minted version.
|
149
|
+
# Versions are created AFTER content is uploaded, except for Fedora 6 which
|
150
|
+
# auto versions.
|
151
|
+
def mint_version(identifier, version_name = "version1")
|
152
|
+
response = connection.http.post do |request|
|
153
|
+
request.url "#{identifier}/fcr:versions"
|
154
|
+
request.headers['Slug'] = version_name if fedora_version == 4
|
155
|
+
end
|
156
|
+
# If there's a deletion marker, don't return anything. (Fedora 4)
|
157
|
+
return nil if response.status == 410
|
158
|
+
# This is awful, but versioning is locked to per-second increments,
|
159
|
+
# returns a 409 in Fedora 5 if there's a conflict.
|
160
|
+
if response.status == 409
|
161
|
+
sleep(0.5)
|
162
|
+
return mint_version(identifier, version_name)
|
163
|
+
end
|
164
|
+
raise "Version unable to be created" unless response.status == 201
|
165
|
+
valkyrie_identifier(uri: response.headers["location"].gsub("/fcr:metadata", ""))
|
58
166
|
end
|
59
167
|
|
60
168
|
class IOProxy
|
@@ -81,6 +189,11 @@ module Valkyrie::Storage
|
|
81
189
|
RDF::URI(identifier)
|
82
190
|
end
|
83
191
|
|
192
|
+
def valkyrie_identifier(uri:)
|
193
|
+
id = uri.to_s.sub("http://", PROTOCOL)
|
194
|
+
Valkyrie::ID.new(id)
|
195
|
+
end
|
196
|
+
|
84
197
|
private
|
85
198
|
|
86
199
|
# @return [IOProxy]
|
@@ -17,7 +17,32 @@ module Valkyrie::Storage
|
|
17
17
|
# @return [Valkyrie::StorageAdapter::StreamFile]
|
18
18
|
def upload(file:, original_filename:, resource: nil, **_extra_arguments)
|
19
19
|
identifier = Valkyrie::ID.new("memory://#{resource.id}")
|
20
|
-
|
20
|
+
version_id = Valkyrie::ID.new("#{identifier}##{SecureRandom.uuid}")
|
21
|
+
cache[identifier] ||= {}
|
22
|
+
cache[identifier][:current] = Valkyrie::StorageAdapter::StreamFile.new(id: identifier, io: file, version_id: version_id)
|
23
|
+
end
|
24
|
+
|
25
|
+
# @param file [IO]
|
26
|
+
# @param original_filename [String]
|
27
|
+
# @param previous_version_id [Valkyrie::ID]
|
28
|
+
# @param _extra_arguments [Hash] additional arguments which may be passed to
|
29
|
+
# other adapters.
|
30
|
+
# @return [Valkyrie::StorageAdapter::StreamFile]
|
31
|
+
def upload_version(id:, file:)
|
32
|
+
# Get previous file and add a UUID to the end of it.
|
33
|
+
new_file = Valkyrie::StorageAdapter::StreamFile.new(id: id, io: file, version_id: Valkyrie::ID.new("#{id}##{SecureRandom.uuid}"))
|
34
|
+
current_file = cache[id][:current]
|
35
|
+
cache[id][:current] = new_file
|
36
|
+
cache[id][:versions] ||= []
|
37
|
+
cache[id][:versions].prepend(current_file) if current_file
|
38
|
+
new_file
|
39
|
+
end
|
40
|
+
|
41
|
+
# @param id [Valkyrie::ID]
|
42
|
+
# @return [Array<Valkyrie::StorageAdapter::StreamFile>]
|
43
|
+
def find_versions(id:)
|
44
|
+
return [] if cache[id].nil?
|
45
|
+
[cache[id][:current] || nil].compact + cache[id].fetch(:versions, [])
|
21
46
|
end
|
22
47
|
|
23
48
|
# Return the file associated with the given identifier
|
@@ -25,8 +50,18 @@ module Valkyrie::Storage
|
|
25
50
|
# @return [Valkyrie::StorageAdapter::StreamFile]
|
26
51
|
# @raise Valkyrie::StorageAdapter::FileNotFound if nothing is found
|
27
52
|
def find_by(id:)
|
28
|
-
|
29
|
-
cache[
|
53
|
+
no_version_id, _version = id_and_version(id)
|
54
|
+
raise Valkyrie::StorageAdapter::FileNotFound unless cache[no_version_id]
|
55
|
+
version =
|
56
|
+
if id == no_version_id
|
57
|
+
cache[id][:current]
|
58
|
+
else
|
59
|
+
find_versions(id: no_version_id).find do |file|
|
60
|
+
file.version_id == id
|
61
|
+
end
|
62
|
+
end
|
63
|
+
raise Valkyrie::StorageAdapter::FileNotFound unless version
|
64
|
+
version
|
30
65
|
end
|
31
66
|
|
32
67
|
# @param id [Valkyrie::ID]
|
@@ -35,10 +70,35 @@ module Valkyrie::Storage
|
|
35
70
|
id.to_s.start_with?("memory://")
|
36
71
|
end
|
37
72
|
|
73
|
+
# @param feature [Symbol] Feature to test for.
|
74
|
+
# @return [Boolean] true if the adapter supports the given feature
|
75
|
+
def supports?(feature)
|
76
|
+
case feature
|
77
|
+
when :versions
|
78
|
+
true
|
79
|
+
when :version_deletion
|
80
|
+
true
|
81
|
+
else
|
82
|
+
false
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def id_and_version(id)
|
87
|
+
id, version = id.to_s.split("#")
|
88
|
+
[Valkyrie::ID.new(id), version]
|
89
|
+
end
|
90
|
+
|
38
91
|
# Delete the file on disk associated with the given identifier.
|
39
92
|
# @param id [Valkyrie::ID]
|
40
93
|
def delete(id:)
|
41
|
-
|
94
|
+
base_id, version = id_and_version(id)
|
95
|
+
if version && cache[base_id][:current]&.version_id != id
|
96
|
+
cache[base_id][:versions].reject! do |file|
|
97
|
+
file.version_id == id
|
98
|
+
end
|
99
|
+
else
|
100
|
+
cache.delete(base_id)
|
101
|
+
end
|
42
102
|
nil
|
43
103
|
end
|
44
104
|
end
|
@@ -0,0 +1,182 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Valkyrie::Storage
|
3
|
+
# The VersionedDisk adapter implements versioned storage on disk by storing
|
4
|
+
# the timestamp of the file's creation as part of the file name
|
5
|
+
# (v-timestamp-filename.jpg). If the
|
6
|
+
# current file is deleted it creates a DeletionMarker, which is an empty file
|
7
|
+
# with "deletionmarker" in the name of the file.
|
8
|
+
class VersionedDisk
|
9
|
+
attr_reader :base_path, :path_generator, :file_mover
|
10
|
+
def initialize(base_path:, path_generator: ::Valkyrie::Storage::Disk::BucketedStorage, file_mover: FileUtils.method(:cp))
|
11
|
+
@base_path = Pathname.new(base_path.to_s)
|
12
|
+
@path_generator = path_generator.new(base_path: base_path)
|
13
|
+
@file_mover = file_mover
|
14
|
+
end
|
15
|
+
|
16
|
+
# @param file [IO]
|
17
|
+
# @param original_filename [String]
|
18
|
+
# @param resource [Valkyrie::Resource]
|
19
|
+
# @param _extra_arguments [Hash] additional arguments which may be passed to other adapters
|
20
|
+
# @return [Valkyrie::StorageAdapter::File]
|
21
|
+
def upload(file:, original_filename:, resource: nil, paused: false, **extra_arguments)
|
22
|
+
version_timestamp = current_timestamp
|
23
|
+
new_path = path_generator.generate(resource: resource, file: file, original_filename: "v-#{version_timestamp}-#{original_filename}")
|
24
|
+
# If we've gone faster than milliseconds here, pause a millisecond and
|
25
|
+
# re-call. Probably only an issue for test suites.
|
26
|
+
return sleep(0.001) && upload(file: file, original_filename: original_filename, resource: resource, paused: true, **extra_arguments) if !paused && File.exist?(new_path)
|
27
|
+
FileUtils.mkdir_p(new_path.parent)
|
28
|
+
file_mover.call(file.try(:path) || file.try(:disk_path), new_path)
|
29
|
+
find_by(id: Valkyrie::ID.new("versiondisk://#{new_path}"))
|
30
|
+
end
|
31
|
+
|
32
|
+
def current_timestamp
|
33
|
+
Time.now.strftime("%s%L")
|
34
|
+
end
|
35
|
+
|
36
|
+
# @param id [Valkyrie::ID] ID of the Valkyrie::StorageAdapter::File to
|
37
|
+
# version.
|
38
|
+
# @param file [IO]
|
39
|
+
# @param paused [Boolean] set to true when upload_version had to pause for a
|
40
|
+
# millisecond to get a later timestamp. Internal only - do not set.
|
41
|
+
def upload_version(id:, file:, paused: false)
|
42
|
+
version_timestamp = current_timestamp
|
43
|
+
# Get the existing version_id so we can calculate the next path from it.
|
44
|
+
current_version_id = version_id(id)
|
45
|
+
current_version_id = current_version_id.version_files[1] if current_version_id.deletion_marker?
|
46
|
+
existing_path = current_version_id.file_path
|
47
|
+
new_path = Pathname.new(existing_path.gsub(current_version_id.version, version_timestamp.to_s))
|
48
|
+
# If we've gone faster than milliseconds here, pause a millisecond and
|
49
|
+
# re-call.
|
50
|
+
return sleep(0.001) && upload_version(id: id, file: file, paused: true) if !paused && File.exist?(new_path)
|
51
|
+
FileUtils.mkdir_p(new_path.parent)
|
52
|
+
file_mover.call(file.try(:path) || file.try(:disk_path), new_path)
|
53
|
+
find_by(id: Valkyrie::ID.new("versiondisk://#{new_path}"))
|
54
|
+
end
|
55
|
+
|
56
|
+
# @param id [Valkyrie::ID]
|
57
|
+
# @return [Boolean] true if this adapter can handle this type of identifer
|
58
|
+
def handles?(id:)
|
59
|
+
id.to_s.start_with?("versiondisk://#{base_path}")
|
60
|
+
end
|
61
|
+
|
62
|
+
# @param feature [Symbol] Feature to test for.
|
63
|
+
# @return [Boolean] true if the adapter supports the given feature
|
64
|
+
def supports?(feature)
|
65
|
+
return true if feature == :versions || feature == :version_deletion
|
66
|
+
false
|
67
|
+
end
|
68
|
+
|
69
|
+
# Return the file associated with the given identifier
|
70
|
+
# @param id [Valkyrie::ID]
|
71
|
+
# @return [Valkyrie::StorageAdapter::File]
|
72
|
+
# @raise Valkyrie::StorageAdapter::FileNotFound if nothing is found
|
73
|
+
def find_by(id:)
|
74
|
+
version_id = version_id(id)
|
75
|
+
raise Valkyrie::StorageAdapter::FileNotFound if version_id.nil? || version_id&.deletion_marker?
|
76
|
+
Valkyrie::StorageAdapter::File.new(id: version_id.current_reference_id.id, io: ::Valkyrie::Storage::Disk::LazyFile.open(version_id.file_path, 'rb'), version_id: version_id.id)
|
77
|
+
rescue Errno::ENOENT
|
78
|
+
raise Valkyrie::StorageAdapter::FileNotFound
|
79
|
+
end
|
80
|
+
|
81
|
+
# Delete the file on disk associated with the given identifier.
|
82
|
+
# @param id [Valkyrie::ID]
|
83
|
+
def delete(id:)
|
84
|
+
id = version_id(id).resolve_current
|
85
|
+
if id.current?
|
86
|
+
id.version_files.each do |version_id|
|
87
|
+
FileUtils.rm_rf(version_id.file_path)
|
88
|
+
end
|
89
|
+
elsif File.exist?(id.file_path)
|
90
|
+
FileUtils.rm_rf(id.file_path)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
# @param id [Valkyrie::ID]
|
95
|
+
# @return [Array<Valkyrie::StorageAdapter::File>]
|
96
|
+
def find_versions(id:)
|
97
|
+
version_files(id: id).select { |x| !x.to_s.include?("deletionmarker") }.map do |file|
|
98
|
+
find_by(id: Valkyrie::ID.new("versiondisk://#{file}"))
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def version_files(id:)
|
103
|
+
root = Pathname.new(file_path(id))
|
104
|
+
id = VersionId.new(id)
|
105
|
+
root.parent.children.select { |file| file.basename.to_s.end_with?(id.filename) }.sort.reverse
|
106
|
+
end
|
107
|
+
|
108
|
+
def file_path(version_id)
|
109
|
+
version_id.to_s.gsub(/^versiondisk:\/\//, '')
|
110
|
+
end
|
111
|
+
|
112
|
+
# @return VersionId A VersionId value that's resolved a current reference,
|
113
|
+
# so we can access the `version_id` and current reference.
|
114
|
+
def version_id(id)
|
115
|
+
id = VersionId.new(id)
|
116
|
+
return id unless id.versioned?
|
117
|
+
id.resolve_current
|
118
|
+
end
|
119
|
+
|
120
|
+
# A small value class that holds a version id and methods for knowing things about it.
|
121
|
+
# Examples of version ids in this adapter:
|
122
|
+
# * "versiondisk://te/st/test/v-current-filename.jpg" (never actually saved this way on disk, just used as a reference)
|
123
|
+
# * "versiondisk://te/st/test/v-1694195675462560794-filename.jpg" (this timestamped form would be saved on disk)
|
124
|
+
# * "versiondisk://te/st/test/v-1694195675462560794-deletionmarker-filename.jpg" (this file is saved on disk but empty)
|
125
|
+
class VersionId
|
126
|
+
attr_reader :id
|
127
|
+
def initialize(id)
|
128
|
+
@id = id
|
129
|
+
end
|
130
|
+
|
131
|
+
def current_reference_id
|
132
|
+
self.class.new(Valkyrie::ID.new(string_id.gsub(version, "current")))
|
133
|
+
end
|
134
|
+
|
135
|
+
# @return [VersionID] the version_id for the current file
|
136
|
+
def resolve_current
|
137
|
+
return self unless reference?
|
138
|
+
version_files.first
|
139
|
+
end
|
140
|
+
|
141
|
+
def file_path
|
142
|
+
@file_path ||= string_id.gsub(/^versiondisk:\/\//, '')
|
143
|
+
end
|
144
|
+
|
145
|
+
def version_files
|
146
|
+
root = Pathname.new(file_path)
|
147
|
+
root.parent.children.select { |file| file.basename.to_s.end_with?(filename) }.sort.reverse.map do |file|
|
148
|
+
VersionId.new(Valkyrie::ID.new("versiondisk://#{file}"))
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def deletion_marker?
|
153
|
+
string_id.include?("deletionmarker")
|
154
|
+
end
|
155
|
+
|
156
|
+
def current?
|
157
|
+
version_files.first.id == id
|
158
|
+
end
|
159
|
+
|
160
|
+
# @return [Boolean] Whether this id is referential (e.g. "current") or absolute (e.g. a timestamp)
|
161
|
+
def reference?
|
162
|
+
version == "current"
|
163
|
+
end
|
164
|
+
|
165
|
+
def versioned?
|
166
|
+
string_id.include?("v-")
|
167
|
+
end
|
168
|
+
|
169
|
+
def version
|
170
|
+
string_id.split("v-").last.split("-", 2).first
|
171
|
+
end
|
172
|
+
|
173
|
+
def filename
|
174
|
+
string_id.split("v-").last.split("-", 2).last.gsub("deletionmarker-", "")
|
175
|
+
end
|
176
|
+
|
177
|
+
def string_id
|
178
|
+
id.to_s
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
data/lib/valkyrie/storage.rb
CHANGED
data/lib/valkyrie/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: valkyrie
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Trey Pendragon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-09-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dry-struct
|
@@ -535,6 +535,7 @@ files:
|
|
535
535
|
- lib/valkyrie/storage/disk.rb
|
536
536
|
- lib/valkyrie/storage/fedora.rb
|
537
537
|
- lib/valkyrie/storage/memory.rb
|
538
|
+
- lib/valkyrie/storage/versioned_disk.rb
|
538
539
|
- lib/valkyrie/storage_adapter.rb
|
539
540
|
- lib/valkyrie/types.rb
|
540
541
|
- lib/valkyrie/value_mapper.rb
|