activestorage 5.2.7.1 → 6.1.4.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of activestorage might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +225 -93
- data/MIT-LICENSE +1 -1
- data/README.md +43 -8
- data/app/assets/javascripts/activestorage.js +5 -2
- data/app/controllers/active_storage/base_controller.rb +13 -4
- data/app/controllers/active_storage/blobs/proxy_controller.rb +14 -0
- data/app/controllers/active_storage/{blobs_controller.rb → blobs/redirect_controller.rb} +3 -3
- data/app/controllers/active_storage/direct_uploads_controller.rb +2 -2
- data/app/controllers/active_storage/disk_controller.rb +13 -22
- data/app/controllers/active_storage/representations/base_controller.rb +14 -0
- data/app/controllers/active_storage/representations/proxy_controller.rb +13 -0
- data/app/controllers/active_storage/{representations_controller.rb → representations/redirect_controller.rb} +3 -5
- data/app/controllers/concerns/active_storage/file_server.rb +18 -0
- data/app/controllers/concerns/active_storage/set_blob.rb +1 -1
- data/app/controllers/concerns/active_storage/set_current.rb +15 -0
- data/app/controllers/concerns/active_storage/set_headers.rb +12 -0
- data/app/javascript/activestorage/blob_record.js +7 -2
- data/app/jobs/active_storage/analyze_job.rb +5 -0
- data/app/jobs/active_storage/base_job.rb +0 -1
- data/app/jobs/active_storage/mirror_job.rb +15 -0
- data/app/jobs/active_storage/purge_job.rb +3 -0
- data/app/models/active_storage/attachment.rb +35 -16
- data/app/models/active_storage/blob/analyzable.rb +6 -2
- data/app/models/active_storage/blob/identifiable.rb +7 -6
- data/app/models/active_storage/blob/representable.rb +36 -6
- data/app/models/active_storage/blob.rb +186 -68
- data/app/models/active_storage/filename.rb +0 -6
- data/app/models/active_storage/preview.rb +37 -12
- data/app/models/active_storage/record.rb +7 -0
- data/app/models/active_storage/variant.rb +53 -67
- data/app/models/active_storage/variant_record.rb +8 -0
- data/app/models/active_storage/variant_with_record.rb +54 -0
- data/app/models/active_storage/variation.rb +30 -94
- data/config/routes.rb +66 -15
- data/db/migrate/20170806125915_create_active_storage_tables.rb +14 -5
- data/db/update_migrate/20190112182829_add_service_name_to_active_storage_blobs.rb +17 -0
- data/db/update_migrate/20191206030411_create_active_storage_variant_records.rb +11 -0
- data/lib/active_storage/analyzer/image_analyzer.rb +14 -4
- data/lib/active_storage/analyzer/null_analyzer.rb +4 -0
- data/lib/active_storage/analyzer/video_analyzer.rb +17 -8
- data/lib/active_storage/analyzer.rb +15 -4
- data/lib/active_storage/attached/changes/create_many.rb +47 -0
- data/lib/active_storage/attached/changes/create_one.rb +82 -0
- data/lib/active_storage/attached/changes/create_one_of_many.rb +10 -0
- data/lib/active_storage/attached/changes/delete_many.rb +27 -0
- data/lib/active_storage/attached/changes/delete_one.rb +19 -0
- data/lib/active_storage/attached/changes.rb +16 -0
- data/lib/active_storage/attached/many.rb +19 -12
- data/lib/active_storage/attached/model.rb +212 -0
- data/lib/active_storage/attached/one.rb +19 -21
- data/lib/active_storage/attached.rb +7 -22
- data/lib/active_storage/downloader.rb +43 -0
- data/lib/active_storage/engine.rb +60 -38
- data/lib/active_storage/errors.rb +25 -3
- data/lib/active_storage/gem_version.rb +4 -4
- data/lib/active_storage/log_subscriber.rb +6 -0
- data/lib/active_storage/previewer/mupdf_previewer.rb +3 -3
- data/lib/active_storage/previewer/poppler_pdf_previewer.rb +3 -3
- data/lib/active_storage/previewer/video_previewer.rb +17 -10
- data/lib/active_storage/previewer.rb +34 -14
- data/lib/active_storage/reflection.rb +64 -0
- data/lib/active_storage/service/azure_storage_service.rb +65 -44
- data/lib/active_storage/service/configurator.rb +6 -2
- data/lib/active_storage/service/disk_service.rb +57 -44
- data/lib/active_storage/service/gcs_service.rb +68 -64
- data/lib/active_storage/service/mirror_service.rb +31 -7
- data/lib/active_storage/service/registry.rb +32 -0
- data/lib/active_storage/service/s3_service.rb +56 -24
- data/lib/active_storage/service.rb +44 -12
- data/lib/active_storage/transformers/image_processing_transformer.rb +45 -0
- data/lib/active_storage/transformers/transformer.rb +39 -0
- data/lib/active_storage.rb +31 -296
- data/lib/tasks/activestorage.rake +11 -0
- metadata +82 -16
- data/app/models/active_storage/filename/parameters.rb +0 -36
- data/lib/active_storage/attached/macros.rb +0 -110
- data/lib/active_storage/downloading.rb +0 -39
@@ -1,28 +1,30 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
gem "azure-storage-blob", ">= 1.1"
|
4
|
+
|
3
5
|
require "active_support/core_ext/numeric/bytes"
|
4
|
-
require "azure/storage"
|
5
|
-
require "azure/storage/core/auth/shared_access_signature"
|
6
|
+
require "azure/storage/blob"
|
7
|
+
require "azure/storage/common/core/auth/shared_access_signature"
|
6
8
|
|
7
9
|
module ActiveStorage
|
8
10
|
# Wraps the Microsoft Azure Storage Blob Service as an Active Storage service.
|
9
11
|
# See ActiveStorage::Service for the generic API documentation that applies to all services.
|
10
12
|
class Service::AzureStorageService < Service
|
11
|
-
attr_reader :client, :
|
13
|
+
attr_reader :client, :container, :signer
|
12
14
|
|
13
|
-
def initialize(storage_account_name:, storage_access_key:, container:, **options)
|
14
|
-
@client = Azure::Storage::
|
15
|
-
@signer = Azure::Storage::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key)
|
16
|
-
@blobs = client.blob_client
|
15
|
+
def initialize(storage_account_name:, storage_access_key:, container:, public: false, **options)
|
16
|
+
@client = Azure::Storage::Blob::BlobService.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key, **options)
|
17
|
+
@signer = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key)
|
17
18
|
@container = container
|
19
|
+
@public = public
|
18
20
|
end
|
19
21
|
|
20
|
-
def upload(key, io, checksum: nil, **)
|
22
|
+
def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, **)
|
21
23
|
instrument :upload, key: key, checksum: checksum do
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
24
|
+
handle_errors do
|
25
|
+
content_disposition = content_disposition_with(filename: filename, type: disposition) if disposition && filename
|
26
|
+
|
27
|
+
client.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition)
|
26
28
|
end
|
27
29
|
end
|
28
30
|
end
|
@@ -34,26 +36,29 @@ module ActiveStorage
|
|
34
36
|
end
|
35
37
|
else
|
36
38
|
instrument :download, key: key do
|
37
|
-
|
38
|
-
|
39
|
+
handle_errors do
|
40
|
+
_, io = client.get_blob(container, key)
|
41
|
+
io.force_encoding(Encoding::BINARY)
|
42
|
+
end
|
39
43
|
end
|
40
44
|
end
|
41
45
|
end
|
42
46
|
|
43
47
|
def download_chunk(key, range)
|
44
48
|
instrument :download_chunk, key: key, range: range do
|
45
|
-
|
46
|
-
|
49
|
+
handle_errors do
|
50
|
+
_, io = client.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end)
|
51
|
+
io.force_encoding(Encoding::BINARY)
|
52
|
+
end
|
47
53
|
end
|
48
54
|
end
|
49
55
|
|
50
56
|
def delete(key)
|
51
57
|
instrument :delete, key: key do
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
end
|
58
|
+
client.delete_blob(container, key)
|
59
|
+
rescue Azure::Core::Http::HTTPError => e
|
60
|
+
raise unless e.type == "BlobNotFound"
|
61
|
+
# Ignore files already deleted
|
57
62
|
end
|
58
63
|
end
|
59
64
|
|
@@ -62,10 +67,10 @@ module ActiveStorage
|
|
62
67
|
marker = nil
|
63
68
|
|
64
69
|
loop do
|
65
|
-
results =
|
70
|
+
results = client.list_blobs(container, prefix: prefix, marker: marker)
|
66
71
|
|
67
72
|
results.each do |blob|
|
68
|
-
|
73
|
+
client.delete_blob(container, blob.name)
|
69
74
|
end
|
70
75
|
|
71
76
|
break unless marker = results.continuation_token.presence
|
@@ -81,15 +86,13 @@ module ActiveStorage
|
|
81
86
|
end
|
82
87
|
end
|
83
88
|
|
84
|
-
def
|
89
|
+
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
|
85
90
|
instrument :url, key: key do |payload|
|
86
91
|
generated_url = signer.signed_uri(
|
87
92
|
uri_for(key), false,
|
88
93
|
service: "b",
|
89
|
-
permissions: "
|
90
|
-
expiry: format_expiry(expires_in)
|
91
|
-
content_disposition: content_disposition_with(type: disposition, filename: filename),
|
92
|
-
content_type: content_type
|
94
|
+
permissions: "rw",
|
95
|
+
expiry: format_expiry(expires_in)
|
93
96
|
).to_s
|
94
97
|
|
95
98
|
payload[:url] = generated_url
|
@@ -98,32 +101,35 @@ module ActiveStorage
|
|
98
101
|
end
|
99
102
|
end
|
100
103
|
|
101
|
-
def
|
102
|
-
|
103
|
-
|
104
|
+
def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, **)
|
105
|
+
content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
|
106
|
+
|
107
|
+
{ "Content-Type" => content_type, "Content-MD5" => checksum, "x-ms-blob-content-disposition" => content_disposition, "x-ms-blob-type" => "BlockBlob" }
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
def private_url(key, expires_in:, filename:, disposition:, content_type:, **)
|
112
|
+
signer.signed_uri(
|
104
113
|
uri_for(key), false,
|
105
114
|
service: "b",
|
106
|
-
permissions: "
|
107
|
-
expiry: format_expiry(expires_in)
|
115
|
+
permissions: "r",
|
116
|
+
expiry: format_expiry(expires_in),
|
117
|
+
content_disposition: content_disposition_with(type: disposition, filename: filename),
|
118
|
+
content_type: content_type
|
108
119
|
).to_s
|
120
|
+
end
|
109
121
|
|
110
|
-
|
111
|
-
|
112
|
-
generated_url
|
122
|
+
def public_url(key, **)
|
123
|
+
uri_for(key).to_s
|
113
124
|
end
|
114
|
-
end
|
115
125
|
|
116
|
-
def headers_for_direct_upload(key, content_type:, checksum:, **)
|
117
|
-
{ "Content-Type" => content_type, "Content-MD5" => checksum, "x-ms-blob-type" => "BlockBlob" }
|
118
|
-
end
|
119
126
|
|
120
|
-
private
|
121
127
|
def uri_for(key)
|
122
|
-
|
128
|
+
client.generate_uri("#{container}/#{key}")
|
123
129
|
end
|
124
130
|
|
125
131
|
def blob_for(key)
|
126
|
-
|
132
|
+
client.get_blob_properties(container, key)
|
127
133
|
rescue Azure::Core::Http::HTTPError
|
128
134
|
false
|
129
135
|
end
|
@@ -139,11 +145,26 @@ module ActiveStorage
|
|
139
145
|
chunk_size = 5.megabytes
|
140
146
|
offset = 0
|
141
147
|
|
148
|
+
raise ActiveStorage::FileNotFoundError unless blob.present?
|
149
|
+
|
142
150
|
while offset < blob.properties[:content_length]
|
143
|
-
_, chunk =
|
151
|
+
_, chunk = client.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1)
|
144
152
|
yield chunk.force_encoding(Encoding::BINARY)
|
145
153
|
offset += chunk_size
|
146
154
|
end
|
147
155
|
end
|
156
|
+
|
157
|
+
def handle_errors
|
158
|
+
yield
|
159
|
+
rescue Azure::Core::Http::HTTPError => e
|
160
|
+
case e.type
|
161
|
+
when "BlobNotFound"
|
162
|
+
raise ActiveStorage::FileNotFoundError
|
163
|
+
when "Md5Mismatch"
|
164
|
+
raise ActiveStorage::IntegrityError
|
165
|
+
else
|
166
|
+
raise
|
167
|
+
end
|
168
|
+
end
|
148
169
|
end
|
149
170
|
end
|
@@ -14,7 +14,9 @@ module ActiveStorage
|
|
14
14
|
|
15
15
|
def build(service_name)
|
16
16
|
config = config_for(service_name.to_sym)
|
17
|
-
resolve(config.fetch(:service)).build(
|
17
|
+
resolve(config.fetch(:service)).build(
|
18
|
+
**config, configurator: self, name: service_name
|
19
|
+
)
|
18
20
|
end
|
19
21
|
|
20
22
|
private
|
@@ -26,7 +28,9 @@ module ActiveStorage
|
|
26
28
|
|
27
29
|
def resolve(class_name)
|
28
30
|
require "active_storage/service/#{class_name.to_s.underscore}_service"
|
29
|
-
ActiveStorage::Service.const_get(:"#{class_name}Service")
|
31
|
+
ActiveStorage::Service.const_get(:"#{class_name.camelize}Service")
|
32
|
+
rescue LoadError
|
33
|
+
raise "Missing service adapter for #{class_name.inspect}"
|
30
34
|
end
|
31
35
|
end
|
32
36
|
end
|
@@ -11,8 +11,9 @@ module ActiveStorage
|
|
11
11
|
class Service::DiskService < Service
|
12
12
|
attr_reader :root
|
13
13
|
|
14
|
-
def initialize(root:)
|
14
|
+
def initialize(root:, public: false, **options)
|
15
15
|
@root = root
|
16
|
+
@public = public
|
16
17
|
end
|
17
18
|
|
18
19
|
def upload(key, io, checksum: nil, **)
|
@@ -22,18 +23,16 @@ module ActiveStorage
|
|
22
23
|
end
|
23
24
|
end
|
24
25
|
|
25
|
-
def download(key)
|
26
|
+
def download(key, &block)
|
26
27
|
if block_given?
|
27
28
|
instrument :streaming_download, key: key do
|
28
|
-
|
29
|
-
while data = file.read(5.megabytes)
|
30
|
-
yield data
|
31
|
-
end
|
32
|
-
end
|
29
|
+
stream key, &block
|
33
30
|
end
|
34
31
|
else
|
35
32
|
instrument :download, key: key do
|
36
33
|
File.binread path_for(key)
|
34
|
+
rescue Errno::ENOENT
|
35
|
+
raise ActiveStorage::FileNotFoundError
|
37
36
|
end
|
38
37
|
end
|
39
38
|
end
|
@@ -44,16 +43,16 @@ module ActiveStorage
|
|
44
43
|
file.seek range.begin
|
45
44
|
file.read range.size
|
46
45
|
end
|
46
|
+
rescue Errno::ENOENT
|
47
|
+
raise ActiveStorage::FileNotFoundError
|
47
48
|
end
|
48
49
|
end
|
49
50
|
|
50
51
|
def delete(key)
|
51
52
|
instrument :delete, key: key do
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
# Ignore files already deleted
|
56
|
-
end
|
53
|
+
File.delete path_for(key)
|
54
|
+
rescue Errno::ENOENT
|
55
|
+
# Ignore files already deleted
|
57
56
|
end
|
58
57
|
end
|
59
58
|
|
@@ -73,35 +72,6 @@ module ActiveStorage
|
|
73
72
|
end
|
74
73
|
end
|
75
74
|
|
76
|
-
def url(key, expires_in:, filename:, disposition:, content_type:)
|
77
|
-
instrument :url, key: key do |payload|
|
78
|
-
content_disposition = content_disposition_with(type: disposition, filename: filename)
|
79
|
-
verified_key_with_expiration = ActiveStorage.verifier.generate(
|
80
|
-
{
|
81
|
-
key: key,
|
82
|
-
disposition: content_disposition,
|
83
|
-
content_type: content_type
|
84
|
-
},
|
85
|
-
{ expires_in: expires_in,
|
86
|
-
purpose: :blob_key }
|
87
|
-
)
|
88
|
-
|
89
|
-
current_uri = URI.parse(current_host)
|
90
|
-
|
91
|
-
generated_url = url_helpers.rails_disk_service_url(verified_key_with_expiration,
|
92
|
-
protocol: current_uri.scheme,
|
93
|
-
host: current_uri.host,
|
94
|
-
port: current_uri.port,
|
95
|
-
disposition: content_disposition,
|
96
|
-
content_type: content_type,
|
97
|
-
filename: filename
|
98
|
-
)
|
99
|
-
payload[:url] = generated_url
|
100
|
-
|
101
|
-
generated_url
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
75
|
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
|
106
76
|
instrument :url, key: key do |payload|
|
107
77
|
verified_token_with_expiration = ActiveStorage.verifier.generate(
|
@@ -109,10 +79,11 @@ module ActiveStorage
|
|
109
79
|
key: key,
|
110
80
|
content_type: content_type,
|
111
81
|
content_length: content_length,
|
112
|
-
checksum: checksum
|
82
|
+
checksum: checksum,
|
83
|
+
service_name: name
|
113
84
|
},
|
114
|
-
|
115
|
-
purpose: :blob_token
|
85
|
+
expires_in: expires_in,
|
86
|
+
purpose: :blob_token
|
116
87
|
)
|
117
88
|
|
118
89
|
generated_url = url_helpers.update_rails_disk_service_url(verified_token_with_expiration, host: current_host)
|
@@ -132,6 +103,48 @@ module ActiveStorage
|
|
132
103
|
end
|
133
104
|
|
134
105
|
private
|
106
|
+
def private_url(key, expires_in:, filename:, content_type:, disposition:, **)
|
107
|
+
generate_url(key, expires_in: expires_in, filename: filename, content_type: content_type, disposition: disposition)
|
108
|
+
end
|
109
|
+
|
110
|
+
def public_url(key, filename:, content_type: nil, disposition: :attachment, **)
|
111
|
+
generate_url(key, expires_in: nil, filename: filename, content_type: content_type, disposition: disposition)
|
112
|
+
end
|
113
|
+
|
114
|
+
def generate_url(key, expires_in:, filename:, content_type:, disposition:)
|
115
|
+
content_disposition = content_disposition_with(type: disposition, filename: filename)
|
116
|
+
verified_key_with_expiration = ActiveStorage.verifier.generate(
|
117
|
+
{
|
118
|
+
key: key,
|
119
|
+
disposition: content_disposition,
|
120
|
+
content_type: content_type,
|
121
|
+
service_name: name
|
122
|
+
},
|
123
|
+
expires_in: expires_in,
|
124
|
+
purpose: :blob_key
|
125
|
+
)
|
126
|
+
|
127
|
+
current_uri = URI.parse(current_host)
|
128
|
+
|
129
|
+
url_helpers.rails_disk_service_url(verified_key_with_expiration,
|
130
|
+
protocol: current_uri.scheme,
|
131
|
+
host: current_uri.host,
|
132
|
+
port: current_uri.port,
|
133
|
+
filename: filename
|
134
|
+
)
|
135
|
+
end
|
136
|
+
|
137
|
+
|
138
|
+
def stream(key)
|
139
|
+
File.open(path_for(key), "rb") do |file|
|
140
|
+
while data = file.read(5.megabytes)
|
141
|
+
yield data
|
142
|
+
end
|
143
|
+
end
|
144
|
+
rescue Errno::ENOENT
|
145
|
+
raise ActiveStorage::FileNotFoundError
|
146
|
+
end
|
147
|
+
|
135
148
|
def folder_for(key)
|
136
149
|
[ key[0..1], key[2..3] ].join("/")
|
137
150
|
end
|
@@ -1,31 +1,40 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
gem "google-cloud-storage", "~> 1.
|
4
|
-
|
3
|
+
gem "google-cloud-storage", "~> 1.11"
|
5
4
|
require "google/cloud/storage"
|
6
|
-
require "net/http"
|
7
|
-
|
8
|
-
require "active_support/core_ext/object/to_query"
|
9
5
|
|
10
6
|
module ActiveStorage
|
11
7
|
# Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API
|
12
8
|
# documentation that applies to all services.
|
13
9
|
class Service::GCSService < Service
|
14
|
-
def initialize(**config)
|
10
|
+
def initialize(public: false, **config)
|
15
11
|
@config = config
|
12
|
+
@public = public
|
16
13
|
end
|
17
14
|
|
18
15
|
def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil)
|
19
16
|
instrument :upload, key: key, checksum: checksum do
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
17
|
+
# GCS's signed URLs don't include params such as response-content-type response-content_disposition
|
18
|
+
# in the signature, which means an attacker can modify them and bypass our effort to force these to
|
19
|
+
# binary and attachment when the file's content type requires it. The only way to force them is to
|
20
|
+
# store them as object's metadata.
|
21
|
+
content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
|
22
|
+
bucket.create_file(io, key, md5: checksum, content_type: content_type, content_disposition: content_disposition)
|
23
|
+
rescue Google::Cloud::InvalidArgumentError
|
24
|
+
raise ActiveStorage::IntegrityError
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def download(key, &block)
|
29
|
+
if block_given?
|
30
|
+
instrument :streaming_download, key: key do
|
31
|
+
stream(key, &block)
|
32
|
+
end
|
33
|
+
else
|
34
|
+
instrument :download, key: key do
|
35
|
+
file_for(key).download.string
|
36
|
+
rescue Google::Cloud::NotFoundError
|
37
|
+
raise ActiveStorage::FileNotFoundError
|
29
38
|
end
|
30
39
|
end
|
31
40
|
end
|
@@ -39,49 +48,28 @@ module ActiveStorage
|
|
39
48
|
end
|
40
49
|
end
|
41
50
|
|
42
|
-
# FIXME: Download in chunks when given a block.
|
43
|
-
def download(key)
|
44
|
-
instrument :download, key: key do
|
45
|
-
io = file_for(key).download
|
46
|
-
io.rewind
|
47
|
-
|
48
|
-
if block_given?
|
49
|
-
yield io.string
|
50
|
-
else
|
51
|
-
io.string
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
51
|
def download_chunk(key, range)
|
57
52
|
instrument :download_chunk, key: key, range: range do
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |client|
|
62
|
-
client.get(uri, "Range" => "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body
|
63
|
-
end
|
53
|
+
file_for(key).download(range: range).string
|
54
|
+
rescue Google::Cloud::NotFoundError
|
55
|
+
raise ActiveStorage::FileNotFoundError
|
64
56
|
end
|
65
57
|
end
|
66
58
|
|
67
59
|
def delete(key)
|
68
60
|
instrument :delete, key: key do
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
# Ignore files already deleted
|
73
|
-
end
|
61
|
+
file_for(key).delete
|
62
|
+
rescue Google::Cloud::NotFoundError
|
63
|
+
# Ignore files already deleted
|
74
64
|
end
|
75
65
|
end
|
76
66
|
|
77
67
|
def delete_prefixed(prefix)
|
78
68
|
instrument :delete_prefixed, prefix: prefix do
|
79
69
|
bucket.files(prefix: prefix).all do |file|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
# Ignore concurrently-deleted files
|
84
|
-
end
|
70
|
+
file.delete
|
71
|
+
rescue Google::Cloud::NotFoundError
|
72
|
+
# Ignore concurrently-deleted files
|
85
73
|
end
|
86
74
|
end
|
87
75
|
end
|
@@ -94,19 +82,6 @@ module ActiveStorage
|
|
94
82
|
end
|
95
83
|
end
|
96
84
|
|
97
|
-
def url(key, expires_in:, filename:, content_type:, disposition:)
|
98
|
-
instrument :url, key: key do |payload|
|
99
|
-
generated_url = file_for(key).signed_url expires: expires_in, query: {
|
100
|
-
"response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
|
101
|
-
"response-content-type" => content_type
|
102
|
-
}
|
103
|
-
|
104
|
-
payload[:url] = generated_url
|
105
|
-
|
106
|
-
generated_url
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
85
|
def url_for_direct_upload(key, expires_in:, checksum:, **)
|
111
86
|
instrument :url, key: key do |payload|
|
112
87
|
generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, content_md5: checksum
|
@@ -117,23 +92,52 @@ module ActiveStorage
|
|
117
92
|
end
|
118
93
|
end
|
119
94
|
|
120
|
-
def headers_for_direct_upload(key, checksum:, **)
|
121
|
-
|
95
|
+
def headers_for_direct_upload(key, checksum:, filename: nil, disposition: nil, **)
|
96
|
+
content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
|
97
|
+
|
98
|
+
{ "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
|
122
99
|
end
|
123
100
|
|
124
101
|
private
|
102
|
+
def private_url(key, expires_in:, filename:, content_type:, disposition:, **)
|
103
|
+
file_for(key).signed_url expires: expires_in, query: {
|
104
|
+
"response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
|
105
|
+
"response-content-type" => content_type
|
106
|
+
}
|
107
|
+
end
|
108
|
+
|
109
|
+
def public_url(key, **)
|
110
|
+
file_for(key).public_url
|
111
|
+
end
|
112
|
+
|
113
|
+
|
125
114
|
attr_reader :config
|
126
115
|
|
127
|
-
def file_for(key)
|
128
|
-
bucket.file(key, skip_lookup:
|
116
|
+
def file_for(key, skip_lookup: true)
|
117
|
+
bucket.file(key, skip_lookup: skip_lookup)
|
118
|
+
end
|
119
|
+
|
120
|
+
# Reads the file for the given key in chunks, yielding each to the block.
|
121
|
+
def stream(key)
|
122
|
+
file = file_for(key, skip_lookup: false)
|
123
|
+
|
124
|
+
chunk_size = 5.megabytes
|
125
|
+
offset = 0
|
126
|
+
|
127
|
+
raise ActiveStorage::FileNotFoundError unless file.present?
|
128
|
+
|
129
|
+
while offset < file.size
|
130
|
+
yield file.download(range: offset..(offset + chunk_size - 1)).string
|
131
|
+
offset += chunk_size
|
132
|
+
end
|
129
133
|
end
|
130
134
|
|
131
135
|
def bucket
|
132
|
-
@bucket ||= client.bucket(config.fetch(:bucket))
|
136
|
+
@bucket ||= client.bucket(config.fetch(:bucket), skip_lookup: true)
|
133
137
|
end
|
134
138
|
|
135
139
|
def client
|
136
|
-
@client ||= Google::Cloud::Storage.new(config.except(:bucket))
|
140
|
+
@client ||= Google::Cloud::Storage.new(**config.except(:bucket))
|
137
141
|
end
|
138
142
|
end
|
139
143
|
end
|
@@ -4,18 +4,26 @@ require "active_support/core_ext/module/delegation"
|
|
4
4
|
|
5
5
|
module ActiveStorage
|
6
6
|
# Wraps a set of mirror services and provides a single ActiveStorage::Service object that will all
|
7
|
-
# have the files uploaded to them. A +primary+ service is designated to answer calls to
|
8
|
-
#
|
7
|
+
# have the files uploaded to them. A +primary+ service is designated to answer calls to:
|
8
|
+
# * +download+
|
9
|
+
# * +exists?+
|
10
|
+
# * +url+
|
11
|
+
# * +url_for_direct_upload+
|
12
|
+
# * +headers_for_direct_upload+
|
9
13
|
class Service::MirrorService < Service
|
10
14
|
attr_reader :primary, :mirrors
|
11
15
|
|
12
|
-
delegate :download, :download_chunk, :exist?, :url,
|
16
|
+
delegate :download, :download_chunk, :exist?, :url,
|
17
|
+
:url_for_direct_upload, :headers_for_direct_upload, :path_for, to: :primary
|
13
18
|
|
14
19
|
# Stitch together from named services.
|
15
|
-
def self.build(primary:, mirrors:, configurator:, **options) #:nodoc:
|
16
|
-
new
|
20
|
+
def self.build(primary:, mirrors:, name:, configurator:, **options) #:nodoc:
|
21
|
+
new(
|
17
22
|
primary: configurator.build(primary),
|
18
|
-
mirrors: mirrors.collect { |
|
23
|
+
mirrors: mirrors.collect { |mirror_name| configurator.build mirror_name }
|
24
|
+
).tap do |service_instance|
|
25
|
+
service_instance.name = name
|
26
|
+
end
|
19
27
|
end
|
20
28
|
|
21
29
|
def initialize(primary:, mirrors:)
|
@@ -26,7 +34,8 @@ module ActiveStorage
|
|
26
34
|
# ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError.
|
27
35
|
def upload(key, io, checksum: nil, **options)
|
28
36
|
each_service.collect do |service|
|
29
|
-
|
37
|
+
io.rewind
|
38
|
+
service.upload key, io, checksum: checksum, **options
|
30
39
|
end
|
31
40
|
end
|
32
41
|
|
@@ -40,6 +49,21 @@ module ActiveStorage
|
|
40
49
|
perform_across_services :delete_prefixed, prefix
|
41
50
|
end
|
42
51
|
|
52
|
+
|
53
|
+
# Copy the file at the +key+ from the primary service to each of the mirrors where it doesn't already exist.
|
54
|
+
def mirror(key, checksum:)
|
55
|
+
instrument :mirror, key: key, checksum: checksum do
|
56
|
+
if (mirrors_in_need_of_mirroring = mirrors.select { |service| !service.exist?(key) }).any?
|
57
|
+
primary.open(key, checksum: checksum) do |io|
|
58
|
+
mirrors_in_need_of_mirroring.each do |service|
|
59
|
+
io.rewind
|
60
|
+
service.upload key, io, checksum: checksum
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
43
67
|
private
|
44
68
|
def each_service(&block)
|
45
69
|
[ primary, *mirrors ].each(&block)
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveStorage
|
4
|
+
class Service::Registry #:nodoc:
|
5
|
+
def initialize(configurations)
|
6
|
+
@configurations = configurations.deep_symbolize_keys
|
7
|
+
@services = {}
|
8
|
+
end
|
9
|
+
|
10
|
+
def fetch(name)
|
11
|
+
services.fetch(name.to_sym) do |key|
|
12
|
+
if configurations.include?(key)
|
13
|
+
services[key] = configurator.build(key)
|
14
|
+
else
|
15
|
+
if block_given?
|
16
|
+
yield key
|
17
|
+
else
|
18
|
+
raise KeyError, "Missing configuration for the #{key} Active Storage service. " \
|
19
|
+
"Configurations available for the #{configurations.keys.to_sentence} services."
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
attr_reader :configurations, :services
|
27
|
+
|
28
|
+
def configurator
|
29
|
+
@configurator ||= ActiveStorage::Service::Configurator.new(configurations)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|