shrine 3.4.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +96 -0
- data/README.md +11 -11
- data/doc/advantages.md +1 -1
- data/doc/attacher.md +1 -1
- data/doc/carrierwave.md +2 -2
- data/doc/changing_derivatives.md +2 -1
- data/doc/changing_location.md +19 -6
- data/doc/design.md +4 -4
- data/doc/external/articles.md +16 -16
- data/doc/external/extensions.md +1 -1
- data/doc/getting_started.md +73 -30
- data/doc/multiple_files.md +57 -22
- data/doc/plugins/activerecord.md +91 -0
- data/doc/plugins/backgrounding.md +30 -4
- data/doc/plugins/data_uri.md +15 -0
- data/doc/plugins/derivation_endpoint.md +39 -1
- data/doc/plugins/derivatives.md +13 -2
- data/doc/plugins/download_endpoint.md +16 -4
- data/doc/plugins/dynamic_storage.md +15 -0
- data/doc/plugins/fallback_storage.md +90 -0
- data/doc/plugins/keep_files.md +6 -4
- data/doc/plugins/rack_response.md +9 -0
- data/doc/plugins/refresh_metadata.md +20 -0
- data/doc/plugins/sequel.md +57 -0
- data/doc/plugins/signature.md +8 -6
- data/doc/plugins/tempfile.md +14 -6
- data/doc/plugins/upload_options.md +12 -0
- data/doc/plugins/url_options.md +12 -0
- data/doc/plugins/validation_helpers.md +1 -1
- data/doc/processing.md +5 -3
- data/doc/refile.md +3 -3
- data/doc/release_notes/2.1.0.md +1 -1
- data/doc/release_notes/3.10.0.md +30 -0
- data/doc/release_notes/3.5.0.md +63 -0
- data/doc/release_notes/3.6.0.md +23 -0
- data/doc/release_notes/3.7.0.md +75 -0
- data/doc/release_notes/3.7.1.md +31 -0
- data/doc/release_notes/3.8.0.md +37 -0
- data/doc/release_notes/3.9.0.md +42 -0
- data/doc/retrieving_uploads.md +1 -1
- data/doc/storage/s3.md +10 -0
- data/doc/testing.md +45 -17
- data/doc/upgrading_to_3.md +3 -5
- data/lib/shrine/attacher.rb +41 -29
- data/lib/shrine/attachment.rb +2 -2
- data/lib/shrine/plugins/_persistence.rb +1 -1
- data/lib/shrine/plugins/_urlsafe_serialization.rb +4 -4
- data/lib/shrine/plugins/activerecord.rb +25 -4
- data/lib/shrine/plugins/add_metadata.rb +2 -4
- data/lib/shrine/plugins/atomic_helpers.rb +7 -7
- data/lib/shrine/plugins/backgrounding.rb +10 -10
- data/lib/shrine/plugins/column.rb +7 -5
- data/lib/shrine/plugins/data_uri.rb +28 -3
- data/lib/shrine/plugins/default_url.rb +4 -4
- data/lib/shrine/plugins/delete_raw.rb +2 -2
- data/lib/shrine/plugins/derivation_endpoint.rb +65 -59
- data/lib/shrine/plugins/derivatives.rb +35 -21
- data/lib/shrine/plugins/download_endpoint.rb +72 -11
- data/lib/shrine/plugins/entity.rb +15 -9
- data/lib/shrine/plugins/fallback_storage.rb +51 -0
- data/lib/shrine/plugins/infer_extension.rb +5 -1
- data/lib/shrine/plugins/instrumentation.rb +13 -9
- data/lib/shrine/plugins/metadata_attributes.rb +1 -1
- data/lib/shrine/plugins/mirroring.rb +10 -10
- data/lib/shrine/plugins/model.rb +12 -10
- data/lib/shrine/plugins/presign_endpoint.rb +13 -10
- data/lib/shrine/plugins/pretty_location.rb +2 -2
- data/lib/shrine/plugins/processing.rb +3 -3
- data/lib/shrine/plugins/rack_file.rb +2 -2
- data/lib/shrine/plugins/rack_response.rb +12 -6
- data/lib/shrine/plugins/refresh_metadata.rb +6 -6
- data/lib/shrine/plugins/remote_url.rb +4 -4
- data/lib/shrine/plugins/remove_attachment.rb +2 -0
- data/lib/shrine/plugins/remove_invalid.rb +4 -0
- data/lib/shrine/plugins/restore_cached_data.rb +3 -3
- data/lib/shrine/plugins/sequel.rb +15 -1
- data/lib/shrine/plugins/signature.rb +2 -2
- data/lib/shrine/plugins/store_dimensions.rb +2 -2
- data/lib/shrine/plugins/tempfile.rb +0 -2
- data/lib/shrine/plugins/upload_endpoint.rb +7 -5
- data/lib/shrine/plugins/upload_options.rb +18 -4
- data/lib/shrine/plugins/url_options.rb +17 -3
- data/lib/shrine/plugins/validation.rb +8 -8
- data/lib/shrine/plugins/validation_helpers.rb +2 -2
- data/lib/shrine/plugins/versions.rb +10 -10
- data/lib/shrine/plugins.rb +6 -14
- data/lib/shrine/storage/file_system.rb +15 -19
- data/lib/shrine/storage/linter.rb +8 -8
- data/lib/shrine/storage/memory.rb +1 -3
- data/lib/shrine/storage/s3.rb +60 -40
- data/lib/shrine/uploaded_file.rb +23 -19
- data/lib/shrine/version.rb +1 -1
- data/lib/shrine.rb +24 -20
- data/shrine.gemspec +10 -8
- metadata +52 -26
|
@@ -26,12 +26,12 @@ class Shrine
|
|
|
26
26
|
# The `storage_key` needs to be one of the registered Shrine storages.
|
|
27
27
|
# Additional options can be given to override the options given on
|
|
28
28
|
# plugin initialization.
|
|
29
|
-
def upload_endpoint(storage_key, **
|
|
29
|
+
def upload_endpoint(storage_key, **)
|
|
30
30
|
Shrine::UploadEndpoint.new(
|
|
31
31
|
shrine_class: self,
|
|
32
32
|
storage_key: storage_key,
|
|
33
33
|
**opts[:upload_endpoint],
|
|
34
|
-
|
|
34
|
+
**,
|
|
35
35
|
)
|
|
36
36
|
end
|
|
37
37
|
|
|
@@ -41,7 +41,7 @@ class Shrine
|
|
|
41
41
|
# It performs the same mounting logic that Rack and other web
|
|
42
42
|
# frameworks use, and is meant for cases where statically mounting the
|
|
43
43
|
# endpoint in the router isn't enough.
|
|
44
|
-
def upload_response(storage_key, env, **
|
|
44
|
+
def upload_response(storage_key, env, **)
|
|
45
45
|
script_name = env["SCRIPT_NAME"]
|
|
46
46
|
path_info = env["PATH_INFO"]
|
|
47
47
|
|
|
@@ -49,7 +49,7 @@ class Shrine
|
|
|
49
49
|
env["SCRIPT_NAME"] += path_info
|
|
50
50
|
env["PATH_INFO"] = ""
|
|
51
51
|
|
|
52
|
-
upload_endpoint(storage_key, **
|
|
52
|
+
upload_endpoint(storage_key, **).call(env)
|
|
53
53
|
ensure
|
|
54
54
|
env["SCRIPT_NAME"] = script_name
|
|
55
55
|
env["PATH_INFO"] = path_info
|
|
@@ -91,7 +91,9 @@ class Shrine
|
|
|
91
91
|
handle_request(request)
|
|
92
92
|
end
|
|
93
93
|
|
|
94
|
-
headers[
|
|
94
|
+
headers = Rack::Headers[headers] if Rack.release >= "3"
|
|
95
|
+
headers["Content-Length"] ||= body.respond_to?(:bytesize) ? body.bytesize.to_s :
|
|
96
|
+
body.map(&:bytesize).inject(0, :+).to_s
|
|
95
97
|
|
|
96
98
|
[status, headers, body]
|
|
97
99
|
end
|
|
@@ -4,9 +4,9 @@ class Shrine
|
|
|
4
4
|
module Plugins
|
|
5
5
|
# Documentation can be found on https://shrinerb.com/docs/plugins/upload_options
|
|
6
6
|
module UploadOptions
|
|
7
|
-
def self.configure(uploader,
|
|
7
|
+
def self.configure(uploader, options = {})
|
|
8
8
|
uploader.opts[:upload_options] ||= {}
|
|
9
|
-
uploader.opts[:upload_options].merge!(
|
|
9
|
+
uploader.opts[:upload_options].merge!(options)
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
module InstanceMethods
|
|
@@ -15,15 +15,29 @@ class Shrine
|
|
|
15
15
|
def _upload(io, **options)
|
|
16
16
|
upload_options = get_upload_options(io, options)
|
|
17
17
|
|
|
18
|
-
super(io, **options, upload_options:
|
|
18
|
+
super(io, **options, upload_options:)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
def get_upload_options(io, options)
|
|
22
|
-
upload_options =
|
|
22
|
+
upload_options = find_upload_options || {}
|
|
23
23
|
upload_options = upload_options.call(io, options) if upload_options.respond_to?(:call)
|
|
24
24
|
upload_options = upload_options.merge(options[:upload_options]) if options[:upload_options]
|
|
25
25
|
upload_options
|
|
26
26
|
end
|
|
27
|
+
|
|
28
|
+
# Matches the storage key exactly first, then falls back to any
|
|
29
|
+
# registered regex that matches the storage key. The regex form is
|
|
30
|
+
# useful when storage keys are generated dynamically (e.g. via the
|
|
31
|
+
# `dynamic_storage` plugin), since it's not possible to list every
|
|
32
|
+
# storage key upfront.
|
|
33
|
+
def find_upload_options
|
|
34
|
+
upload_options = opts[:upload_options]
|
|
35
|
+
|
|
36
|
+
return upload_options[storage_key] if upload_options.key?(storage_key)
|
|
37
|
+
|
|
38
|
+
_, options = upload_options.find { |key, _| key.is_a?(Regexp) && key.match?(storage_key.to_s) }
|
|
39
|
+
options
|
|
40
|
+
end
|
|
27
41
|
end
|
|
28
42
|
end
|
|
29
43
|
|
|
@@ -4,9 +4,9 @@ class Shrine
|
|
|
4
4
|
module Plugins
|
|
5
5
|
# Documentation can be found on https://shrinerb.com/docs/plugins/url_options
|
|
6
6
|
module UrlOptions
|
|
7
|
-
def self.configure(uploader,
|
|
7
|
+
def self.configure(uploader, options = {})
|
|
8
8
|
uploader.opts[:url_options] ||= {}
|
|
9
|
-
uploader.opts[:url_options].merge!(
|
|
9
|
+
uploader.opts[:url_options].merge!(options)
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
module FileMethods
|
|
@@ -19,10 +19,24 @@ class Shrine
|
|
|
19
19
|
private
|
|
20
20
|
|
|
21
21
|
def url_options(options)
|
|
22
|
-
default_options =
|
|
22
|
+
default_options = find_url_options
|
|
23
23
|
default_options = default_options.call(self, options) if default_options.respond_to?(:call)
|
|
24
24
|
default_options || {}
|
|
25
25
|
end
|
|
26
|
+
|
|
27
|
+
# Matches the storage key exactly first, then falls back to any
|
|
28
|
+
# registered regex that matches the storage key. The regex form is
|
|
29
|
+
# useful when storage keys are generated dynamically (e.g. via the
|
|
30
|
+
# `dynamic_storage` plugin), since it's not possible to list every
|
|
31
|
+
# storage key upfront.
|
|
32
|
+
def find_url_options
|
|
33
|
+
url_options = shrine_class.opts[:url_options]
|
|
34
|
+
|
|
35
|
+
return url_options[storage_key] if url_options.key?(storage_key)
|
|
36
|
+
|
|
37
|
+
_, options = url_options.find { |key, _| key.is_a?(Regexp) && key.match?(storage_key.to_s) }
|
|
38
|
+
options
|
|
39
|
+
end
|
|
26
40
|
end
|
|
27
41
|
end
|
|
28
42
|
|
|
@@ -24,21 +24,21 @@ class Shrine
|
|
|
24
24
|
attr_reader :errors
|
|
25
25
|
|
|
26
26
|
# Initializes validation errors to an empty array.
|
|
27
|
-
def initialize(**
|
|
27
|
+
def initialize(**)
|
|
28
28
|
super
|
|
29
29
|
@errors = []
|
|
30
30
|
end
|
|
31
31
|
|
|
32
32
|
# Performs validations after attaching cached file.
|
|
33
|
-
def attach_cached(value, validate: nil, **
|
|
34
|
-
result = super(value, validate: false, **
|
|
33
|
+
def attach_cached(value, validate: nil, **)
|
|
34
|
+
result = super(value, validate: false, **)
|
|
35
35
|
validation(validate)
|
|
36
36
|
result
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
# Performs validations after attaching file.
|
|
40
|
-
def attach(io, validate: nil, **
|
|
41
|
-
result = super(io, **
|
|
40
|
+
def attach(io, validate: nil, **)
|
|
41
|
+
result = super(io, **)
|
|
42
42
|
validation(validate)
|
|
43
43
|
result
|
|
44
44
|
end
|
|
@@ -61,16 +61,16 @@ class Shrine
|
|
|
61
61
|
end
|
|
62
62
|
|
|
63
63
|
# Calls #validate_block, passing it accepted parameters.
|
|
64
|
-
def _validate(**
|
|
64
|
+
def _validate(**)
|
|
65
65
|
if method(:validate_block).arity.zero?
|
|
66
66
|
validate_block
|
|
67
67
|
else
|
|
68
|
-
validate_block(**
|
|
68
|
+
validate_block(**)
|
|
69
69
|
end
|
|
70
70
|
end
|
|
71
71
|
|
|
72
72
|
# Overridden by the `Attacher.validate` block.
|
|
73
|
-
def validate_block(**
|
|
73
|
+
def validate_block(**)
|
|
74
74
|
end
|
|
75
75
|
end
|
|
76
76
|
end
|
|
@@ -32,7 +32,7 @@ class Shrine
|
|
|
32
32
|
"%.1f %s" % [bytes.to_f / 1024 ** exp, FILESIZE_UNITS[exp]]
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
def self.load_dependencies(uploader,
|
|
35
|
+
def self.load_dependencies(uploader, **)
|
|
36
36
|
uploader.plugin :validation
|
|
37
37
|
end
|
|
38
38
|
|
|
@@ -145,7 +145,7 @@ class Shrine
|
|
|
145
145
|
|
|
146
146
|
# Validates that the dimensions are not smaller than specified.
|
|
147
147
|
#
|
|
148
|
-
#
|
|
148
|
+
# validate_min_dimensions [100, 100]
|
|
149
149
|
def validate_min_dimensions((min_width, min_height), message: nil)
|
|
150
150
|
fail Error, "width and/or height metadata is missing" unless file["width"] && file["height"]
|
|
151
151
|
|
|
@@ -59,16 +59,16 @@ class Shrine
|
|
|
59
59
|
|
|
60
60
|
# Smart versioned URLs, which include the version name in the default
|
|
61
61
|
# URL, and properly forwards any options to the underlying storage.
|
|
62
|
-
def url(version = nil, **
|
|
62
|
+
def url(version = nil, **)
|
|
63
63
|
if file.is_a?(Hash)
|
|
64
64
|
if version
|
|
65
65
|
version = version.to_sym
|
|
66
66
|
if file.key?(version)
|
|
67
|
-
file[version].url(**
|
|
67
|
+
file[version].url(**)
|
|
68
68
|
elsif fallback = shrine_class.version_fallbacks[version]
|
|
69
|
-
url(fallback, **
|
|
69
|
+
url(fallback, **)
|
|
70
70
|
else
|
|
71
|
-
default_url(
|
|
71
|
+
default_url(**, version:)
|
|
72
72
|
end
|
|
73
73
|
else
|
|
74
74
|
raise Error, "must call Shrine::Attacher#url with the name of the version"
|
|
@@ -76,12 +76,12 @@ class Shrine
|
|
|
76
76
|
else
|
|
77
77
|
if version
|
|
78
78
|
if file && shrine_class.opts[:versions][:fallback_to_original]
|
|
79
|
-
file.url(**
|
|
79
|
+
file.url(**)
|
|
80
80
|
else
|
|
81
|
-
default_url(
|
|
81
|
+
default_url(**, version:)
|
|
82
82
|
end
|
|
83
83
|
else
|
|
84
|
-
super(**
|
|
84
|
+
super(**)
|
|
85
85
|
end
|
|
86
86
|
end
|
|
87
87
|
end
|
|
@@ -130,7 +130,7 @@ class Shrine
|
|
|
130
130
|
|
|
131
131
|
def map_file(object, transform_keys: :to_sym)
|
|
132
132
|
if object.is_a?(Hash) || object.is_a?(Array)
|
|
133
|
-
deep_map(object, transform_keys:
|
|
133
|
+
deep_map(object, transform_keys:) do |path, value|
|
|
134
134
|
yield path, value unless value.is_a?(Hash) || value.is_a?(Array)
|
|
135
135
|
end
|
|
136
136
|
elsif object
|
|
@@ -150,7 +150,7 @@ class Shrine
|
|
|
150
150
|
key = key.send(transform_keys)
|
|
151
151
|
result = yield [*path, key], value
|
|
152
152
|
|
|
153
|
-
hash.merge! key => (result || deep_map(value, [*path, key], transform_keys
|
|
153
|
+
hash.merge! key => (result || deep_map(value, [*path, key], transform_keys:, &block))
|
|
154
154
|
end
|
|
155
155
|
elsif object.is_a?(Array)
|
|
156
156
|
result = yield path, object
|
|
@@ -160,7 +160,7 @@ class Shrine
|
|
|
160
160
|
object.map.with_index do |value, idx|
|
|
161
161
|
result = yield [*path, idx], value
|
|
162
162
|
|
|
163
|
-
result || deep_map(value, [*path, idx], transform_keys
|
|
163
|
+
result || deep_map(value, [*path, idx], transform_keys:, &block)
|
|
164
164
|
end
|
|
165
165
|
else
|
|
166
166
|
result = yield path, object
|
data/lib/shrine/plugins.rb
CHANGED
|
@@ -18,26 +18,18 @@ class Shrine
|
|
|
18
18
|
plugin
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
# Delegate
|
|
22
|
-
def self.load_dependencies(plugin, uploader,
|
|
21
|
+
# Delegate to the plugin's `load_dependencies` method.
|
|
22
|
+
def self.load_dependencies(plugin, uploader, ...)
|
|
23
23
|
return unless plugin.respond_to?(:load_dependencies)
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
plugin.load_dependencies(uploader, *args, **kwargs, &block)
|
|
27
|
-
else
|
|
28
|
-
plugin.load_dependencies(uploader, *args, &block)
|
|
29
|
-
end
|
|
25
|
+
plugin.load_dependencies(uploader, ...)
|
|
30
26
|
end
|
|
31
27
|
|
|
32
|
-
# Delegate
|
|
33
|
-
def self.configure(plugin, uploader,
|
|
28
|
+
# Delegate to the plugin's `load_dependencies` method.
|
|
29
|
+
def self.configure(plugin, uploader, ...)
|
|
34
30
|
return unless plugin.respond_to?(:configure)
|
|
35
31
|
|
|
36
|
-
|
|
37
|
-
plugin.configure(uploader, *args, **kwargs, &block)
|
|
38
|
-
else
|
|
39
|
-
plugin.configure(uploader, *args, &block)
|
|
40
|
-
end
|
|
32
|
+
plugin.configure(uploader, ...)
|
|
41
33
|
end
|
|
42
34
|
|
|
43
35
|
# Register the given plugin with Shrine, so that it can be loaded using
|
|
@@ -59,8 +59,8 @@ class Shrine
|
|
|
59
59
|
|
|
60
60
|
# Opens the file on the given location in read mode. Accepts additional
|
|
61
61
|
# `File.open` arguments.
|
|
62
|
-
def open(id, **
|
|
63
|
-
path(id).open(binmode: true, **
|
|
62
|
+
def open(id, **)
|
|
63
|
+
path(id).open(binmode: true, **)
|
|
64
64
|
rescue Errno::ENOENT
|
|
65
65
|
raise Shrine::FileNotFound, "file #{id.inspect} not found on storage"
|
|
66
66
|
end
|
|
@@ -75,7 +75,7 @@ class Shrine
|
|
|
75
75
|
# from the returned path (e.g. #directory can be set to "public" folder).
|
|
76
76
|
# Both cases accept a `:host` value which will be prefixed to the
|
|
77
77
|
# generated path.
|
|
78
|
-
def url(id, host: nil, **
|
|
78
|
+
def url(id, host: nil, **)
|
|
79
79
|
path = (prefix ? relative_path(id) : path(id)).to_s
|
|
80
80
|
host ? host + path : path
|
|
81
81
|
end
|
|
@@ -113,9 +113,18 @@ class Shrine
|
|
|
113
113
|
end
|
|
114
114
|
end
|
|
115
115
|
|
|
116
|
-
# Returns the full path to the file.
|
|
116
|
+
# Returns the full path to the file. Raises Shrine::Error if the id would
|
|
117
|
+
# resolve to a location outside of the storage #directory (e.g. an id
|
|
118
|
+
# containing `../` path traversal sequences in attacker-controlled data).
|
|
117
119
|
def path(id)
|
|
118
|
-
directory.join(id.gsub("/", File::SEPARATOR))
|
|
120
|
+
path = directory.join(id.gsub("/", File::SEPARATOR))
|
|
121
|
+
expanded = path.expand_path
|
|
122
|
+
|
|
123
|
+
unless expanded == directory || expanded.to_s.start_with?("#{directory}#{File::SEPARATOR}")
|
|
124
|
+
raise Shrine::Error, "path #{id.inspect} resolves outside of the storage directory"
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
path
|
|
119
128
|
end
|
|
120
129
|
|
|
121
130
|
protected
|
|
@@ -123,7 +132,7 @@ class Shrine
|
|
|
123
132
|
# Cleans all empty subdirectories up the hierarchy.
|
|
124
133
|
def clean(path)
|
|
125
134
|
path.dirname.ascend do |pathname|
|
|
126
|
-
if
|
|
135
|
+
if Dir.empty?(pathname) && pathname != directory
|
|
127
136
|
pathname.rmdir
|
|
128
137
|
else
|
|
129
138
|
break
|
|
@@ -175,19 +184,6 @@ class Shrine
|
|
|
175
184
|
.find
|
|
176
185
|
.each { |path| yield path if path.file? }
|
|
177
186
|
end
|
|
178
|
-
|
|
179
|
-
if RUBY_VERSION >= "2.4"
|
|
180
|
-
def dir_empty?(path)
|
|
181
|
-
Dir.empty?(path)
|
|
182
|
-
end
|
|
183
|
-
else
|
|
184
|
-
# :nocov:
|
|
185
|
-
def dir_empty?(path)
|
|
186
|
-
Dir.foreach(path) { |x| return false unless [".", ".."].include?(x) }
|
|
187
|
-
true
|
|
188
|
-
end
|
|
189
|
-
# :nocov:
|
|
190
|
-
end
|
|
191
187
|
end
|
|
192
188
|
end
|
|
193
189
|
end
|
|
@@ -24,18 +24,18 @@ class Shrine
|
|
|
24
24
|
#
|
|
25
25
|
# Shrine::Storage::Linter.new(storage).call(->{File.open("test/fixtures/image.jpg")})
|
|
26
26
|
class Linter
|
|
27
|
-
def self.call(*
|
|
28
|
-
new(*
|
|
27
|
+
def self.call(*)
|
|
28
|
+
new(*).call
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
def initialize(storage, action: :error, nonexisting: "nonexisting")
|
|
31
|
+
def initialize(storage, action: :error, nonexisting: String.new("nonexisting"))
|
|
32
32
|
@storage = storage
|
|
33
33
|
@action = action
|
|
34
34
|
@nonexisting = nonexisting
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def call(io_factory = default_io_factory)
|
|
38
|
-
storage.upload(io_factory.call, id = "foo", shrine_metadata: { "foo" => "bar" })
|
|
38
|
+
storage.upload(io_factory.call, id = String.new("foo"), shrine_metadata: { "foo" => "bar" })
|
|
39
39
|
|
|
40
40
|
lint_open(id)
|
|
41
41
|
lint_exists(id)
|
|
@@ -43,9 +43,9 @@ class Shrine
|
|
|
43
43
|
lint_delete(id)
|
|
44
44
|
|
|
45
45
|
if storage.respond_to?(:delete_prefixed)
|
|
46
|
-
storage.upload(io_factory.call, id1 = "a/a/a")
|
|
47
|
-
storage.upload(io_factory.call, id2 = "a/a/b")
|
|
48
|
-
storage.upload(io_factory.call, id3 = "a/aaa/a")
|
|
46
|
+
storage.upload(io_factory.call, id1 = String.new("a/a/a"))
|
|
47
|
+
storage.upload(io_factory.call, id2 = String.new("a/a/b"))
|
|
48
|
+
storage.upload(io_factory.call, id3 = String.new("a/aaa/a"))
|
|
49
49
|
|
|
50
50
|
lint_delete_prefixed(prefix: "a/a/",
|
|
51
51
|
expect_deleted: [id1, id2],
|
|
@@ -55,7 +55,7 @@ class Shrine
|
|
|
55
55
|
end
|
|
56
56
|
|
|
57
57
|
if storage.respond_to?(:clear!)
|
|
58
|
-
storage.upload(io_factory.call, id = "quux"
|
|
58
|
+
storage.upload(io_factory.call, id = String.new("quux"))
|
|
59
59
|
lint_clear(id)
|
|
60
60
|
end
|
|
61
61
|
|
|
@@ -17,9 +17,7 @@ class Shrine
|
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def open(id, **)
|
|
20
|
-
|
|
21
|
-
io.set_encoding(io.string.encoding) # Ruby 2.7.0 – https://bugs.ruby-lang.org/issues/16497
|
|
22
|
-
io
|
|
20
|
+
StringIO.new(store.fetch(id))
|
|
23
21
|
rescue KeyError
|
|
24
22
|
raise Shrine::FileNotFound, "file #{id.inspect} not found on storage"
|
|
25
23
|
end
|
data/lib/shrine/storage/s3.rb
CHANGED
|
@@ -14,13 +14,15 @@ require "tempfile"
|
|
|
14
14
|
class Shrine
|
|
15
15
|
module Storage
|
|
16
16
|
class S3
|
|
17
|
-
attr_reader :client, :bucket, :prefix, :upload_options, :signer, :public
|
|
17
|
+
attr_reader :client, :bucket, :prefix, :upload_options, :copy_options, :signer, :public
|
|
18
18
|
|
|
19
19
|
MAX_MULTIPART_PARTS = 10_000
|
|
20
20
|
MIN_PART_SIZE = 5*1024*1024
|
|
21
21
|
|
|
22
22
|
MULTIPART_THRESHOLD = { upload: 15*1024*1024, copy: 100*1024*1024 }
|
|
23
23
|
|
|
24
|
+
COPY_OPTIONS = { tagging_directive: "REPLACE" }
|
|
25
|
+
|
|
24
26
|
# Initializes a storage for uploading to S3. All options are forwarded to
|
|
25
27
|
# [`Aws::S3::Client#initialize`], except the following:
|
|
26
28
|
#
|
|
@@ -41,12 +43,20 @@ class Shrine
|
|
|
41
43
|
# be passed to [`Aws::S3::Object#put`], [`Aws::S3::Object#copy_from`]
|
|
42
44
|
# and [`Aws::S3::Bucket#presigned_post`].
|
|
43
45
|
#
|
|
46
|
+
# :copy_options
|
|
47
|
+
# : Additional options that will be used for copying files, they will
|
|
48
|
+
# be passed to [`Aws::S3::Object#copy_from`].
|
|
49
|
+
#
|
|
44
50
|
# :multipart_threshold
|
|
45
51
|
# : If the input file is larger than the specified size, a parallelized
|
|
46
52
|
# multipart will be used for the upload/copy. Defaults to
|
|
47
53
|
# `{upload: 15*1024*1024, copy: 100*1024*1024}` (15MB for upload
|
|
48
54
|
# requests, 100MB for copy requests).
|
|
49
55
|
#
|
|
56
|
+
# :max_multipart_parts
|
|
57
|
+
# : Limits the number of parts if parellized multipart upload/copy is used.
|
|
58
|
+
# Defaults to 10_000.
|
|
59
|
+
#
|
|
50
60
|
# In addition to specifying the `:bucket`, you'll also need to provide
|
|
51
61
|
# AWS credentials. The most common way is to provide them directly via
|
|
52
62
|
# `:access_key_id`, `:secret_access_key`, and `:region` options. But you
|
|
@@ -58,14 +68,17 @@ class Shrine
|
|
|
58
68
|
# [`Aws::S3::Bucket#presigned_post`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#presigned_post-instance_method
|
|
59
69
|
# [`Aws::S3::Client#initialize`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#initialize-instance_method
|
|
60
70
|
# [configuring AWS SDK]: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html
|
|
61
|
-
def initialize(bucket:, client: nil, prefix: nil, upload_options: {}, multipart_threshold: {}, signer: nil, public: nil, **
|
|
71
|
+
def initialize(bucket:, client: nil, prefix: nil, upload_options: {}, multipart_threshold: {}, max_multipart_parts: nil, signer: nil, public: nil, copy_options: COPY_OPTIONS, **)
|
|
62
72
|
raise ArgumentError, "the :bucket option is nil" unless bucket
|
|
63
73
|
|
|
64
|
-
@client = client || Aws::S3::Client.new(**
|
|
74
|
+
@client = client || Aws::S3::Client.new(**)
|
|
75
|
+
@transfer_manager = Aws::S3::TransferManager.new(client: @client) if defined?(Aws::S3::TransferManager)
|
|
65
76
|
@bucket = Aws::S3::Bucket.new(name: bucket, client: @client)
|
|
66
77
|
@prefix = prefix
|
|
67
78
|
@upload_options = upload_options
|
|
79
|
+
@copy_options = copy_options
|
|
68
80
|
@multipart_threshold = MULTIPART_THRESHOLD.merge(multipart_threshold)
|
|
81
|
+
@max_multipart_parts = max_multipart_parts || MAX_MULTIPART_PARTS
|
|
69
82
|
@signer = signer
|
|
70
83
|
@public = public
|
|
71
84
|
end
|
|
@@ -85,8 +98,7 @@ class Shrine
|
|
|
85
98
|
options[:content_disposition] = ContentDisposition.inline(filename) if filename
|
|
86
99
|
options[:acl] = "public-read" if public
|
|
87
100
|
|
|
88
|
-
options.merge!(@upload_options)
|
|
89
|
-
options.merge!(upload_options)
|
|
101
|
+
options.merge!(@upload_options, upload_options)
|
|
90
102
|
|
|
91
103
|
if copyable?(io)
|
|
92
104
|
copy(io, id, **options)
|
|
@@ -98,15 +110,16 @@ class Shrine
|
|
|
98
110
|
# Returns a `Down::ChunkedIO` object that downloads S3 object content
|
|
99
111
|
# on-demand. By default, read content will be cached onto disk so that
|
|
100
112
|
# it can be rewinded, but if you don't need that you can pass
|
|
101
|
-
# `rewindable: false`.
|
|
113
|
+
# `rewindable: false`. A required character encoding can be passed in
|
|
114
|
+
# `encoding`; the default is `Encoding::BINARY` via `Down::ChunkedIO`.
|
|
102
115
|
#
|
|
103
116
|
# Any additional options are forwarded to [`Aws::S3::Object#get`].
|
|
104
117
|
#
|
|
105
118
|
# [`Aws::S3::Object#get`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#get-instance_method
|
|
106
|
-
def open(id, rewindable: true, **
|
|
107
|
-
chunks, length = get(id, **
|
|
119
|
+
def open(id, rewindable: true, encoding: nil, **)
|
|
120
|
+
chunks, length = get(id, **)
|
|
108
121
|
|
|
109
|
-
Down::ChunkedIO.new(chunks: chunks, rewindable: rewindable, size: length)
|
|
122
|
+
Down::ChunkedIO.new(chunks: chunks, rewindable: rewindable, size: length, encoding: encoding)
|
|
110
123
|
rescue Aws::S3::Errors::NoSuchKey
|
|
111
124
|
raise Shrine::FileNotFound, "file #{id.inspect} not found on storage"
|
|
112
125
|
end
|
|
@@ -132,11 +145,11 @@ class Shrine
|
|
|
132
145
|
#
|
|
133
146
|
# [`Aws::S3::Object#presigned_url`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#presigned_url-instance_method
|
|
134
147
|
# [`Aws::S3::Object#public_url`]: http://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Object.html#public_url-instance_method
|
|
135
|
-
def url(id, public: self.public, host: nil, **
|
|
148
|
+
def url(id, public: self.public, host: nil, **)
|
|
136
149
|
if public || signer
|
|
137
|
-
url = object(id).public_url(**
|
|
150
|
+
url = object(id).public_url(**)
|
|
138
151
|
else
|
|
139
|
-
url = object(id).presigned_url(:get, **
|
|
152
|
+
url = object(id).presigned_url(:get, **)
|
|
140
153
|
end
|
|
141
154
|
|
|
142
155
|
if host
|
|
@@ -146,7 +159,7 @@ class Shrine
|
|
|
146
159
|
end
|
|
147
160
|
|
|
148
161
|
if signer
|
|
149
|
-
url = signer.call(url, **
|
|
162
|
+
url = signer.call(url, **)
|
|
150
163
|
end
|
|
151
164
|
|
|
152
165
|
url
|
|
@@ -177,8 +190,7 @@ class Shrine
|
|
|
177
190
|
options = {}
|
|
178
191
|
options[:acl] = "public-read" if public
|
|
179
192
|
|
|
180
|
-
options.merge!(@upload_options)
|
|
181
|
-
options.merge!(presign_options)
|
|
193
|
+
options.merge!(@upload_options, presign_options)
|
|
182
194
|
|
|
183
195
|
send(:"presign_#{method}", id, options)
|
|
184
196
|
end
|
|
@@ -219,11 +231,15 @@ class Shrine
|
|
|
219
231
|
private
|
|
220
232
|
|
|
221
233
|
# Uploads the file to S3. Uses multipart upload for large files.
|
|
222
|
-
def put(io, id, **
|
|
234
|
+
def put(io, id, **)
|
|
223
235
|
if io.respond_to?(:size) && io.size && io.size <= @multipart_threshold[:upload]
|
|
224
|
-
object(id).put(body: io, **
|
|
225
|
-
|
|
226
|
-
|
|
236
|
+
object(id).put(body: io, **)
|
|
237
|
+
elsif @transfer_manager # multipart upload - transfer manager
|
|
238
|
+
@transfer_manager.upload_stream(bucket: bucket.name, key: object_key(id), part_size: part_size(io), **) do |write_stream|
|
|
239
|
+
IO.copy_stream(io, write_stream)
|
|
240
|
+
end
|
|
241
|
+
else # multipart upload - before transfer manager
|
|
242
|
+
object(id).upload_stream(part_size: part_size(io), **) do |write_stream|
|
|
227
243
|
IO.copy_stream(io, write_stream)
|
|
228
244
|
end
|
|
229
245
|
end
|
|
@@ -232,14 +248,17 @@ class Shrine
|
|
|
232
248
|
# Copies an existing S3 object to a new location. Uses multipart copy for
|
|
233
249
|
# large files.
|
|
234
250
|
def copy(io, id, **copy_options)
|
|
235
|
-
|
|
251
|
+
# don't inherit source object metadata or AWS tags
|
|
252
|
+
options = {
|
|
253
|
+
metadata_directive: "REPLACE",
|
|
254
|
+
}
|
|
236
255
|
|
|
237
256
|
if io.size && io.size >= @multipart_threshold[:copy]
|
|
238
257
|
# pass :content_length on multipart copy to avoid an additional HEAD request
|
|
239
258
|
options.merge!(multipart_copy: true, content_length: io.size)
|
|
240
259
|
end
|
|
241
260
|
|
|
242
|
-
options.merge!(copy_options)
|
|
261
|
+
options.merge!(@copy_options, copy_options)
|
|
243
262
|
|
|
244
263
|
object(id).copy_from(io.storage.object(io.id), **options)
|
|
245
264
|
end
|
|
@@ -257,13 +276,14 @@ class Shrine
|
|
|
257
276
|
|
|
258
277
|
# When any of these options are specified, the corresponding request
|
|
259
278
|
# headers must be included in the upload request.
|
|
260
|
-
headers = {
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
279
|
+
headers = {
|
|
280
|
+
"Content-Length" => options[:content_length],
|
|
281
|
+
"Content-Type" => options[:content_type],
|
|
282
|
+
"Content-Disposition" => options[:content_disposition],
|
|
283
|
+
"Content-Encoding" => options[:content_encoding],
|
|
284
|
+
"Content-Language" => options[:content_language],
|
|
285
|
+
"Content-MD5" => options[:content_md5],
|
|
286
|
+
}.compact
|
|
267
287
|
|
|
268
288
|
{ method: :put, url: url, headers: headers }
|
|
269
289
|
end
|
|
@@ -273,10 +293,10 @@ class Shrine
|
|
|
273
293
|
def part_size(io)
|
|
274
294
|
return unless io.respond_to?(:size) && io.size
|
|
275
295
|
|
|
276
|
-
if io.size <= MIN_PART_SIZE *
|
|
296
|
+
if io.size <= MIN_PART_SIZE * @max_multipart_parts # <= 50 GB
|
|
277
297
|
MIN_PART_SIZE
|
|
278
298
|
else # > 50 GB
|
|
279
|
-
(io.size.to_f /
|
|
299
|
+
(io.size.to_f / @max_multipart_parts).ceil
|
|
280
300
|
end
|
|
281
301
|
end
|
|
282
302
|
|
|
@@ -284,8 +304,8 @@ class Shrine
|
|
|
284
304
|
# object before all content is downloaded, so we hack our way around it.
|
|
285
305
|
# This way get the content length without an additional HEAD request.
|
|
286
306
|
if Gem::Version.new(Aws::CORE_GEM_VERSION) >= Gem::Version.new("3.104.0")
|
|
287
|
-
def get(id, **
|
|
288
|
-
enum = object(id).enum_for(:get, **
|
|
307
|
+
def get(id, **)
|
|
308
|
+
enum = object(id).enum_for(:get, **)
|
|
289
309
|
|
|
290
310
|
begin
|
|
291
311
|
content_length = Integer(enum.peek.last["content-length"])
|
|
@@ -298,8 +318,8 @@ class Shrine
|
|
|
298
318
|
[chunks, content_length]
|
|
299
319
|
end
|
|
300
320
|
else
|
|
301
|
-
def get(id, **
|
|
302
|
-
req = client.build_request(:get_object, bucket: bucket.name, key: object_key(id), **
|
|
321
|
+
def get(id, **)
|
|
322
|
+
req = client.build_request(:get_object, bucket: bucket.name, key: object_key(id), **)
|
|
303
323
|
|
|
304
324
|
body = req.enum_for(:send_request)
|
|
305
325
|
begin
|
|
@@ -342,10 +362,10 @@ class Shrine
|
|
|
342
362
|
|
|
343
363
|
# Save the encryption client and continue initialization with normal
|
|
344
364
|
# client.
|
|
345
|
-
def initialize(client: nil, **
|
|
365
|
+
def initialize(client: nil, **)
|
|
346
366
|
return super unless client.class.name.start_with?("Aws::S3::Encryption")
|
|
347
367
|
|
|
348
|
-
super(client: client.client, **
|
|
368
|
+
super(client: client.client, **)
|
|
349
369
|
@encryption_client = client
|
|
350
370
|
end
|
|
351
371
|
|
|
@@ -353,19 +373,19 @@ class Shrine
|
|
|
353
373
|
|
|
354
374
|
# Encryption client doesn't support multipart uploads, so we always use
|
|
355
375
|
# #put_object.
|
|
356
|
-
def put(io, id, **
|
|
376
|
+
def put(io, id, **)
|
|
357
377
|
return super unless encryption_client
|
|
358
378
|
|
|
359
|
-
encryption_client.put_object(body: io, bucket: bucket.name, key: object_key(id), **
|
|
379
|
+
encryption_client.put_object(body: io, bucket: bucket.name, key: object_key(id), **)
|
|
360
380
|
end
|
|
361
381
|
|
|
362
|
-
def get(id, **
|
|
382
|
+
def get(id, **)
|
|
363
383
|
return super unless encryption_client
|
|
364
384
|
|
|
365
385
|
# Encryption client v2 warns against streaming download, so we first
|
|
366
386
|
# download all content into a file.
|
|
367
387
|
tempfile = Tempfile.new("shrine-s3", binmode: true)
|
|
368
|
-
response = encryption_client.get_object(response_target: tempfile, bucket: bucket.name, key: object_key(id), **
|
|
388
|
+
response = encryption_client.get_object(response_target: tempfile, bucket: bucket.name, key: object_key(id), **)
|
|
369
389
|
tempfile.rewind
|
|
370
390
|
|
|
371
391
|
chunks = Enumerator.new do |yielder|
|