activestorage 6.0.6.1 → 6.1.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of activestorage might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +137 -248
- data/MIT-LICENSE +1 -1
- data/README.md +36 -4
- data/app/controllers/active_storage/base_controller.rb +11 -0
- data/app/controllers/active_storage/blobs/proxy_controller.rb +14 -0
- data/app/controllers/active_storage/{blobs_controller.rb → blobs/redirect_controller.rb} +2 -2
- data/app/controllers/active_storage/disk_controller.rb +8 -20
- data/app/controllers/active_storage/representations/proxy_controller.rb +19 -0
- data/app/controllers/active_storage/{representations_controller.rb → representations/redirect_controller.rb} +2 -2
- data/app/controllers/concerns/active_storage/file_server.rb +18 -0
- data/app/controllers/concerns/active_storage/set_blob.rb +1 -1
- data/app/controllers/concerns/active_storage/set_current.rb +2 -2
- data/app/controllers/concerns/active_storage/set_headers.rb +12 -0
- data/app/jobs/active_storage/mirror_job.rb +15 -0
- data/app/models/active_storage/attachment.rb +18 -10
- data/app/models/active_storage/blob/analyzable.rb +6 -2
- data/app/models/active_storage/blob/identifiable.rb +7 -6
- data/app/models/active_storage/blob/representable.rb +34 -4
- data/app/models/active_storage/blob.rb +114 -57
- data/app/models/active_storage/preview.rb +31 -10
- data/app/models/active_storage/record.rb +7 -0
- data/app/models/active_storage/variant.rb +28 -41
- data/app/models/active_storage/variant_record.rb +8 -0
- data/app/models/active_storage/variant_with_record.rb +54 -0
- data/app/models/active_storage/variation.rb +25 -20
- data/config/routes.rb +58 -8
- data/db/migrate/20170806125915_create_active_storage_tables.rb +14 -5
- data/db/update_migrate/20190112182829_add_service_name_to_active_storage_blobs.rb +17 -0
- data/db/update_migrate/20191206030411_create_active_storage_variant_records.rb +11 -0
- data/lib/active_storage/analyzer/image_analyzer.rb +3 -0
- data/lib/active_storage/analyzer/null_analyzer.rb +4 -0
- data/lib/active_storage/analyzer/video_analyzer.rb +14 -3
- data/lib/active_storage/analyzer.rb +6 -0
- data/lib/active_storage/attached/changes/create_many.rb +1 -0
- data/lib/active_storage/attached/changes/create_one.rb +17 -4
- data/lib/active_storage/attached/many.rb +4 -3
- data/lib/active_storage/attached/model.rb +49 -10
- data/lib/active_storage/attached/one.rb +4 -3
- data/lib/active_storage/engine.rb +25 -43
- data/lib/active_storage/gem_version.rb +3 -3
- data/lib/active_storage/log_subscriber.rb +6 -0
- data/lib/active_storage/previewer/mupdf_previewer.rb +3 -3
- data/lib/active_storage/previewer/poppler_pdf_previewer.rb +2 -2
- data/lib/active_storage/previewer/video_previewer.rb +2 -2
- data/lib/active_storage/previewer.rb +3 -2
- data/lib/active_storage/service/azure_storage_service.rb +40 -35
- data/lib/active_storage/service/configurator.rb +3 -1
- data/lib/active_storage/service/disk_service.rb +36 -31
- data/lib/active_storage/service/gcs_service.rb +18 -16
- data/lib/active_storage/service/mirror_service.rb +31 -7
- data/lib/active_storage/service/registry.rb +32 -0
- data/lib/active_storage/service/s3_service.rb +51 -23
- data/lib/active_storage/service.rb +35 -7
- data/lib/active_storage/transformers/image_processing_transformer.rb +13 -365
- data/lib/active_storage/transformers/transformer.rb +0 -3
- data/lib/active_storage.rb +9 -8
- metadata +60 -25
- data/db/update_migrate/20180723000244_add_foreign_key_constraint_to_active_storage_attachments_for_blob_id.rb +0 -9
- data/lib/active_storage/downloading.rb +0 -47
- data/lib/active_storage/transformers/mini_magick_transformer.rb +0 -38
@@ -7,8 +7,9 @@ module ActiveStorage
|
|
7
7
|
# Wraps the Google Cloud Storage as an Active Storage service. See ActiveStorage::Service for the generic API
|
8
8
|
# documentation that applies to all services.
|
9
9
|
class Service::GCSService < Service
|
10
|
-
def initialize(**config)
|
10
|
+
def initialize(public: false, **config)
|
11
11
|
@config = config
|
12
|
+
@public = public
|
12
13
|
end
|
13
14
|
|
14
15
|
def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil)
|
@@ -81,19 +82,6 @@ module ActiveStorage
|
|
81
82
|
end
|
82
83
|
end
|
83
84
|
|
84
|
-
def url(key, expires_in:, filename:, content_type:, disposition:)
|
85
|
-
instrument :url, key: key do |payload|
|
86
|
-
generated_url = file_for(key).signed_url expires: expires_in, query: {
|
87
|
-
"response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
|
88
|
-
"response-content-type" => content_type
|
89
|
-
}
|
90
|
-
|
91
|
-
payload[:url] = generated_url
|
92
|
-
|
93
|
-
generated_url
|
94
|
-
end
|
95
|
-
end
|
96
|
-
|
97
85
|
def url_for_direct_upload(key, expires_in:, checksum:, **)
|
98
86
|
instrument :url, key: key do |payload|
|
99
87
|
generated_url = bucket.signed_url key, method: "PUT", expires: expires_in, content_md5: checksum
|
@@ -104,11 +92,25 @@ module ActiveStorage
|
|
104
92
|
end
|
105
93
|
end
|
106
94
|
|
107
|
-
def headers_for_direct_upload(key, checksum:, **)
|
108
|
-
|
95
|
+
def headers_for_direct_upload(key, checksum:, filename: nil, disposition: nil, **)
|
96
|
+
content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
|
97
|
+
|
98
|
+
{ "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
|
109
99
|
end
|
110
100
|
|
111
101
|
private
|
102
|
+
def private_url(key, expires_in:, filename:, content_type:, disposition:, **)
|
103
|
+
file_for(key).signed_url expires: expires_in, query: {
|
104
|
+
"response-content-disposition" => content_disposition_with(type: disposition, filename: filename),
|
105
|
+
"response-content-type" => content_type
|
106
|
+
}
|
107
|
+
end
|
108
|
+
|
109
|
+
def public_url(key, **)
|
110
|
+
file_for(key).public_url
|
111
|
+
end
|
112
|
+
|
113
|
+
|
112
114
|
attr_reader :config
|
113
115
|
|
114
116
|
def file_for(key, skip_lookup: true)
|
@@ -4,18 +4,26 @@ require "active_support/core_ext/module/delegation"
|
|
4
4
|
|
5
5
|
module ActiveStorage
|
6
6
|
# Wraps a set of mirror services and provides a single ActiveStorage::Service object that will all
|
7
|
-
# have the files uploaded to them. A +primary+ service is designated to answer calls to
|
8
|
-
#
|
7
|
+
# have the files uploaded to them. A +primary+ service is designated to answer calls to:
|
8
|
+
# * +download+
|
9
|
+
# * +exists?+
|
10
|
+
# * +url+
|
11
|
+
# * +url_for_direct_upload+
|
12
|
+
# * +headers_for_direct_upload+
|
9
13
|
class Service::MirrorService < Service
|
10
14
|
attr_reader :primary, :mirrors
|
11
15
|
|
12
|
-
delegate :download, :download_chunk, :exist?, :url,
|
16
|
+
delegate :download, :download_chunk, :exist?, :url,
|
17
|
+
:url_for_direct_upload, :headers_for_direct_upload, :path_for, to: :primary
|
13
18
|
|
14
19
|
# Stitch together from named services.
|
15
|
-
def self.build(primary:, mirrors:, configurator:, **options) #:nodoc:
|
16
|
-
new
|
20
|
+
def self.build(primary:, mirrors:, name:, configurator:, **options) #:nodoc:
|
21
|
+
new(
|
17
22
|
primary: configurator.build(primary),
|
18
|
-
mirrors: mirrors.collect { |
|
23
|
+
mirrors: mirrors.collect { |mirror_name| configurator.build mirror_name }
|
24
|
+
).tap do |service_instance|
|
25
|
+
service_instance.name = name
|
26
|
+
end
|
19
27
|
end
|
20
28
|
|
21
29
|
def initialize(primary:, mirrors:)
|
@@ -26,7 +34,8 @@ module ActiveStorage
|
|
26
34
|
# ensure a match when the upload has completed or raise an ActiveStorage::IntegrityError.
|
27
35
|
def upload(key, io, checksum: nil, **options)
|
28
36
|
each_service.collect do |service|
|
29
|
-
|
37
|
+
io.rewind
|
38
|
+
service.upload key, io, checksum: checksum, **options
|
30
39
|
end
|
31
40
|
end
|
32
41
|
|
@@ -40,6 +49,21 @@ module ActiveStorage
|
|
40
49
|
perform_across_services :delete_prefixed, prefix
|
41
50
|
end
|
42
51
|
|
52
|
+
|
53
|
+
# Copy the file at the +key+ from the primary service to each of the mirrors where it doesn't already exist.
|
54
|
+
def mirror(key, checksum:)
|
55
|
+
instrument :mirror, key: key, checksum: checksum do
|
56
|
+
if (mirrors_in_need_of_mirroring = mirrors.select { |service| !service.exist?(key) }).any?
|
57
|
+
primary.open(key, checksum: checksum) do |io|
|
58
|
+
mirrors_in_need_of_mirroring.each do |service|
|
59
|
+
io.rewind
|
60
|
+
service.upload key, io, checksum: checksum
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
43
67
|
private
|
44
68
|
def each_service(&block)
|
45
69
|
[ primary, *mirrors ].each(&block)
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveStorage
|
4
|
+
class Service::Registry #:nodoc:
|
5
|
+
def initialize(configurations)
|
6
|
+
@configurations = configurations.deep_symbolize_keys
|
7
|
+
@services = {}
|
8
|
+
end
|
9
|
+
|
10
|
+
def fetch(name)
|
11
|
+
services.fetch(name.to_sym) do |key|
|
12
|
+
if configurations.include?(key)
|
13
|
+
services[key] = configurator.build(key)
|
14
|
+
else
|
15
|
+
if block_given?
|
16
|
+
yield key
|
17
|
+
else
|
18
|
+
raise KeyError, "Missing configuration for the #{key} Active Storage service. " \
|
19
|
+
"Configurations available for the #{configurations.keys.to_sentence} services."
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
attr_reader :configurations, :services
|
27
|
+
|
28
|
+
def configurator
|
29
|
+
@configurator ||= ActiveStorage::Service::Configurator.new(configurations)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -9,20 +9,29 @@ module ActiveStorage
|
|
9
9
|
# Wraps the Amazon Simple Storage Service (S3) as an Active Storage service.
|
10
10
|
# See ActiveStorage::Service for the generic API documentation that applies to all services.
|
11
11
|
class Service::S3Service < Service
|
12
|
-
attr_reader :client, :bucket
|
12
|
+
attr_reader :client, :bucket
|
13
|
+
attr_reader :multipart_upload_threshold, :upload_options
|
13
14
|
|
14
|
-
def initialize(bucket:, upload: {}, **options)
|
15
|
+
def initialize(bucket:, upload: {}, public: false, **options)
|
15
16
|
@client = Aws::S3::Resource.new(**options)
|
16
17
|
@bucket = @client.bucket(bucket)
|
17
18
|
|
19
|
+
@multipart_upload_threshold = upload.fetch(:multipart_threshold, 100.megabytes)
|
20
|
+
@public = public
|
21
|
+
|
18
22
|
@upload_options = upload
|
23
|
+
@upload_options[:acl] = "public-read" if public?
|
19
24
|
end
|
20
25
|
|
21
|
-
def upload(key, io, checksum: nil, content_type: nil, **)
|
26
|
+
def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, **)
|
22
27
|
instrument :upload, key: key, checksum: checksum do
|
23
|
-
|
24
|
-
|
25
|
-
|
28
|
+
content_disposition = content_disposition_with(filename: filename, type: disposition) if disposition && filename
|
29
|
+
|
30
|
+
if io.size < multipart_upload_threshold
|
31
|
+
upload_with_single_part key, io, checksum: checksum, content_type: content_type, content_disposition: content_disposition
|
32
|
+
else
|
33
|
+
upload_with_multipart key, io, content_type: content_type, content_disposition: content_disposition
|
34
|
+
end
|
26
35
|
end
|
27
36
|
end
|
28
37
|
|
@@ -42,7 +51,7 @@ module ActiveStorage
|
|
42
51
|
|
43
52
|
def download_chunk(key, range)
|
44
53
|
instrument :download_chunk, key: key, range: range do
|
45
|
-
object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.
|
54
|
+
object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}").body.string.force_encoding(Encoding::BINARY)
|
46
55
|
rescue Aws::S3::Errors::NoSuchKey
|
47
56
|
raise ActiveStorage::FileNotFoundError
|
48
57
|
end
|
@@ -68,23 +77,11 @@ module ActiveStorage
|
|
68
77
|
end
|
69
78
|
end
|
70
79
|
|
71
|
-
def url(key, expires_in:, filename:, disposition:, content_type:)
|
72
|
-
instrument :url, key: key do |payload|
|
73
|
-
generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i,
|
74
|
-
response_content_disposition: content_disposition_with(type: disposition, filename: filename),
|
75
|
-
response_content_type: content_type
|
76
|
-
|
77
|
-
payload[:url] = generated_url
|
78
|
-
|
79
|
-
generated_url
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
80
|
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
|
84
81
|
instrument :url, key: key do |payload|
|
85
82
|
generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i,
|
86
83
|
content_type: content_type, content_length: content_length, content_md5: checksum,
|
87
|
-
whitelist_headers: [
|
84
|
+
whitelist_headers: ["content-length"], **upload_options
|
88
85
|
|
89
86
|
payload[:url] = generated_url
|
90
87
|
|
@@ -92,11 +89,42 @@ module ActiveStorage
|
|
92
89
|
end
|
93
90
|
end
|
94
91
|
|
95
|
-
def headers_for_direct_upload(key, content_type:, checksum:, **)
|
96
|
-
|
92
|
+
def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, **)
|
93
|
+
content_disposition = content_disposition_with(type: disposition, filename: filename) if filename
|
94
|
+
|
95
|
+
{ "Content-Type" => content_type, "Content-MD5" => checksum, "Content-Disposition" => content_disposition }
|
97
96
|
end
|
98
97
|
|
99
98
|
private
|
99
|
+
def private_url(key, expires_in:, filename:, disposition:, content_type:, **)
|
100
|
+
object_for(key).presigned_url :get, expires_in: expires_in.to_i,
|
101
|
+
response_content_disposition: content_disposition_with(type: disposition, filename: filename),
|
102
|
+
response_content_type: content_type
|
103
|
+
end
|
104
|
+
|
105
|
+
def public_url(key, **)
|
106
|
+
object_for(key).public_url
|
107
|
+
end
|
108
|
+
|
109
|
+
|
110
|
+
MAXIMUM_UPLOAD_PARTS_COUNT = 10000
|
111
|
+
MINIMUM_UPLOAD_PART_SIZE = 5.megabytes
|
112
|
+
|
113
|
+
def upload_with_single_part(key, io, checksum: nil, content_type: nil, content_disposition: nil)
|
114
|
+
object_for(key).put(body: io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition, **upload_options)
|
115
|
+
rescue Aws::S3::Errors::BadDigest
|
116
|
+
raise ActiveStorage::IntegrityError
|
117
|
+
end
|
118
|
+
|
119
|
+
def upload_with_multipart(key, io, content_type: nil, content_disposition: nil)
|
120
|
+
part_size = [ io.size.fdiv(MAXIMUM_UPLOAD_PARTS_COUNT).ceil, MINIMUM_UPLOAD_PART_SIZE ].max
|
121
|
+
|
122
|
+
object_for(key).upload_stream(content_type: content_type, content_disposition: content_disposition, part_size: part_size, **upload_options) do |out|
|
123
|
+
IO.copy_stream(io, out)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
|
100
128
|
def object_for(key)
|
101
129
|
bucket.object(key)
|
102
130
|
end
|
@@ -111,7 +139,7 @@ module ActiveStorage
|
|
111
139
|
raise ActiveStorage::FileNotFoundError unless object.exists?
|
112
140
|
|
113
141
|
while offset < object.content_length
|
114
|
-
yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.
|
142
|
+
yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}").body.string.force_encoding(Encoding::BINARY)
|
115
143
|
offset += chunk_size
|
116
144
|
end
|
117
145
|
end
|
@@ -41,6 +41,7 @@ module ActiveStorage
|
|
41
41
|
class Service
|
42
42
|
extend ActiveSupport::Autoload
|
43
43
|
autoload :Configurator
|
44
|
+
attr_accessor :name
|
44
45
|
|
45
46
|
class << self
|
46
47
|
# Configure an Active Storage service by name from a set of configurations,
|
@@ -56,8 +57,10 @@ module ActiveStorage
|
|
56
57
|
# Passes the configurator and all of the service's config as keyword args.
|
57
58
|
#
|
58
59
|
# See MirrorService for an example.
|
59
|
-
def build(configurator:, service: nil, **service_config) #:nodoc:
|
60
|
-
new(**service_config)
|
60
|
+
def build(configurator:, name:, service: nil, **service_config) #:nodoc:
|
61
|
+
new(**service_config).tap do |service_instance|
|
62
|
+
service_instance.name = name
|
63
|
+
end
|
61
64
|
end
|
62
65
|
end
|
63
66
|
|
@@ -102,11 +105,23 @@ module ActiveStorage
|
|
102
105
|
raise NotImplementedError
|
103
106
|
end
|
104
107
|
|
105
|
-
# Returns
|
106
|
-
#
|
107
|
-
# +filename+, and +content_type+ that you wish the file to be served with on request.
|
108
|
-
|
109
|
-
|
108
|
+
# Returns the URL for the file at the +key+. This returns a permanent URL for public files, and returns a
|
109
|
+
# short-lived URL for private files. For private files you can provide the +disposition+ (+:inline+ or +:attachment+),
|
110
|
+
# +filename+, and +content_type+ that you wish the file to be served with on request. Additionally, you can also provide
|
111
|
+
# the amount of seconds the URL will be valid for, specified in +expires_in+.
|
112
|
+
def url(key, **options)
|
113
|
+
instrument :url, key: key do |payload|
|
114
|
+
generated_url =
|
115
|
+
if public?
|
116
|
+
public_url(key, **options)
|
117
|
+
else
|
118
|
+
private_url(key, **options)
|
119
|
+
end
|
120
|
+
|
121
|
+
payload[:url] = generated_url
|
122
|
+
|
123
|
+
generated_url
|
124
|
+
end
|
110
125
|
end
|
111
126
|
|
112
127
|
# Returns a signed, temporary URL that a direct upload file can be PUT to on the +key+.
|
@@ -122,7 +137,20 @@ module ActiveStorage
|
|
122
137
|
{}
|
123
138
|
end
|
124
139
|
|
140
|
+
def public?
|
141
|
+
@public
|
142
|
+
end
|
143
|
+
|
125
144
|
private
|
145
|
+
def private_url(key, expires_in:, filename:, disposition:, content_type:, **)
|
146
|
+
raise NotImplementedError
|
147
|
+
end
|
148
|
+
|
149
|
+
def public_url(key, **)
|
150
|
+
raise NotImplementedError
|
151
|
+
end
|
152
|
+
|
153
|
+
|
126
154
|
def instrument(operation, payload = {}, &block)
|
127
155
|
ActiveSupport::Notifications.instrument(
|
128
156
|
"service_#{operation}.active_storage",
|