aws-sdk-s3 1.9.0 → 1.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/aws-sdk-s3/bucket.rb +22 -3
- data/lib/aws-sdk-s3/bucket_acl.rb +1 -0
- data/lib/aws-sdk-s3/bucket_cors.rb +1 -0
- data/lib/aws-sdk-s3/bucket_lifecycle.rb +3 -2
- data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +3 -2
- data/lib/aws-sdk-s3/bucket_notification.rb +9 -5
- data/lib/aws-sdk-s3/bucket_tagging.rb +1 -0
- data/lib/aws-sdk-s3/bucket_website.rb +4 -0
- data/lib/aws-sdk-s3/client.rb +1303 -351
- data/lib/aws-sdk-s3/client_api.rb +352 -2
- data/lib/aws-sdk-s3/customizations/bucket.rb +2 -2
- data/lib/aws-sdk-s3/customizations/object.rb +60 -0
- data/lib/aws-sdk-s3/customizations/object_summary.rb +7 -0
- data/lib/aws-sdk-s3/customizations.rb +1 -0
- data/lib/aws-sdk-s3/encryption/client.rb +1 -1
- data/lib/aws-sdk-s3/encryption/io_auth_decrypter.rb +9 -3
- data/lib/aws-sdk-s3/event_streams.rb +62 -0
- data/lib/aws-sdk-s3/file_downloader.rb +10 -9
- data/lib/aws-sdk-s3/file_part.rb +5 -7
- data/lib/aws-sdk-s3/file_uploader.rb +1 -3
- data/lib/aws-sdk-s3/multipart_stream_uploader.rb +162 -0
- data/lib/aws-sdk-s3/multipart_upload.rb +1 -0
- data/lib/aws-sdk-s3/multipart_upload_part.rb +2 -2
- data/lib/aws-sdk-s3/object.rb +71 -7
- data/lib/aws-sdk-s3/object_acl.rb +1 -0
- data/lib/aws-sdk-s3/object_summary.rb +54 -7
- data/lib/aws-sdk-s3/object_version.rb +11 -0
- data/lib/aws-sdk-s3/plugins/accelerate.rb +4 -0
- data/lib/aws-sdk-s3/plugins/md5s.rb +4 -1
- data/lib/aws-sdk-s3/plugins/redirects.rb +3 -1
- data/lib/aws-sdk-s3/plugins/s3_signer.rb +6 -1
- data/lib/aws-sdk-s3/presigner.rb +4 -0
- data/lib/aws-sdk-s3/resource.rb +4 -0
- data/lib/aws-sdk-s3/types.rb +1539 -227
- data/lib/aws-sdk-s3.rb +2 -1
- metadata +12 -4
@@ -61,8 +61,8 @@ module Aws
|
|
61
61
|
# You can pass `virtual_host: true` to use the bucket name as the
|
62
62
|
# host name.
|
63
63
|
#
|
64
|
-
# bucket = s3.bucket('my.bucket.com'
|
65
|
-
# bucket.url
|
64
|
+
# bucket = s3.bucket('my.bucket.com')
|
65
|
+
# bucket.url(virtual_host: true)
|
66
66
|
# #=> "http://my.bucket.com"
|
67
67
|
#
|
68
68
|
# @option options [Boolean] :virtual_host (false) When `true`,
|
@@ -214,6 +214,62 @@ module Aws
|
|
214
214
|
url.to_s
|
215
215
|
end
|
216
216
|
|
217
|
+
# Uploads a stream in a streaming fashion to the current object in S3.
|
218
|
+
#
|
219
|
+
# # Passed chunks automatically split into multipart upload parts
|
220
|
+
# # and the parts are uploaded in parallel. This allows for streaming uploads
|
221
|
+
# # that never touch the disk.
|
222
|
+
#
|
223
|
+
# Note that this is known to have issues in JRuby until jruby-9.1.15.0, so avoid using this with older versions of JRuby.
|
224
|
+
#
|
225
|
+
# @example Streaming chunks of data
|
226
|
+
# obj.upload_stream do |write_stream|
|
227
|
+
# 10.times { write_stream << 'foo' }
|
228
|
+
# end
|
229
|
+
# @example Streaming chunks of data
|
230
|
+
# obj.upload_stream do |write_stream|
|
231
|
+
# IO.copy_stream(IO.popen('ls'), write_stream)
|
232
|
+
# end
|
233
|
+
# @example Streaming chunks of data
|
234
|
+
# obj.upload_stream do |write_stream|
|
235
|
+
# IO.copy_stream(STDIN, write_stream)
|
236
|
+
# end
|
237
|
+
#
|
238
|
+
# @option options [Integer] :thread_count
|
239
|
+
# The number of parallel multipart uploads
|
240
|
+
# Default `:thread_count` is `10`.
|
241
|
+
#
|
242
|
+
# @option options [Boolean] :tempfile
|
243
|
+
# Normally read data is stored in memory when building the parts in order to complete
|
244
|
+
# the underlying multipart upload. By passing `:tempfile => true` data read will be
|
245
|
+
# temporarily stored on disk reducing the memory footprint vastly.
|
246
|
+
# Default `:tempfile` is `false`.
|
247
|
+
#
|
248
|
+
# @option options [Integer] :part_size
|
249
|
+
# Define how big each part size but the last should be.
|
250
|
+
# Default `:part_size` is `5 * 1024 * 1024`.
|
251
|
+
#
|
252
|
+
# @raise [MultipartUploadError] If an object is being uploaded in
|
253
|
+
# parts, and the upload can not be completed, then the upload is
|
254
|
+
# aborted and this error is raised. The raised error has a `#errors`
|
255
|
+
# method that returns the failures that caused the upload to be
|
256
|
+
# aborted.
|
257
|
+
#
|
258
|
+
# @return [Boolean] Returns `true` when the object is uploaded
|
259
|
+
# without any errors.
|
260
|
+
#
|
261
|
+
def upload_stream(options = {}, &block)
|
262
|
+
uploading_options = options.dup
|
263
|
+
uploader = MultipartStreamUploader.new(
|
264
|
+
client: client,
|
265
|
+
thread_count: uploading_options.delete(:thread_count),
|
266
|
+
tempfile: uploading_options.delete(:tempfile),
|
267
|
+
part_size: uploading_options.delete(:part_size),
|
268
|
+
)
|
269
|
+
uploader.upload(uploading_options.merge(bucket: bucket_name, key: key), &block)
|
270
|
+
true
|
271
|
+
end
|
272
|
+
|
217
273
|
# Uploads a file from disk to the current object in S3.
|
218
274
|
#
|
219
275
|
# # small files are uploaded in a single API call
|
@@ -277,6 +333,10 @@ module Aws
|
|
277
333
|
# @option options [String] thread_count Customize threads used in multipart
|
278
334
|
# download, if not provided, 10 is default value
|
279
335
|
#
|
336
|
+
# @option options [String] version_id The object version id used to retrieve
|
337
|
+
# the object, to know more about object versioning, see:
|
338
|
+
# https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
|
339
|
+
#
|
280
340
|
# @return [Boolean] Returns `true` when the file is downloaded
|
281
341
|
# without any errors.
|
282
342
|
def download_file(destination, options = {})
|
@@ -60,6 +60,13 @@ module Aws
|
|
60
60
|
object.upload_file(source, options)
|
61
61
|
end
|
62
62
|
|
63
|
+
# @options (see Object#upload_stream)
|
64
|
+
# @return (see Object#upload_stream)
|
65
|
+
# @see Object#upload_stream
|
66
|
+
def upload_stream(options = {}, &block)
|
67
|
+
object.upload_stream(options, &block)
|
68
|
+
end
|
69
|
+
|
63
70
|
# @param (see Object#download_file)
|
64
71
|
# @options (see Object#download_file)
|
65
72
|
# @return (see Object#download_file)
|
@@ -6,6 +6,7 @@ require 'aws-sdk-s3/file_uploader'
|
|
6
6
|
require 'aws-sdk-s3/file_downloader'
|
7
7
|
require 'aws-sdk-s3/legacy_signer'
|
8
8
|
require 'aws-sdk-s3/multipart_file_uploader'
|
9
|
+
require 'aws-sdk-s3/multipart_stream_uploader'
|
9
10
|
require 'aws-sdk-s3/multipart_upload_error'
|
10
11
|
require 'aws-sdk-s3/object_copier'
|
11
12
|
require 'aws-sdk-s3/object_multipart_copier'
|
@@ -22,8 +22,10 @@ module Aws
|
|
22
22
|
|
23
23
|
def write(chunk)
|
24
24
|
chunk = truncate_chunk(chunk)
|
25
|
-
|
26
|
-
|
25
|
+
if chunk.bytesize > 0
|
26
|
+
@bytes_written += chunk.bytesize
|
27
|
+
@decrypter.write(chunk)
|
28
|
+
end
|
27
29
|
end
|
28
30
|
|
29
31
|
def finalize
|
@@ -39,8 +41,12 @@ module Aws
|
|
39
41
|
def truncate_chunk(chunk)
|
40
42
|
if chunk.bytesize + @bytes_written <= @max_bytes
|
41
43
|
chunk
|
42
|
-
|
44
|
+
elsif @bytes_written < @max_bytes
|
43
45
|
chunk[0..(@max_bytes - @bytes_written - 1)]
|
46
|
+
else
|
47
|
+
# If the tag was sent over after the full body has been read,
|
48
|
+
# we don't want to accidentally append it.
|
49
|
+
""
|
44
50
|
end
|
45
51
|
end
|
46
52
|
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# WARNING ABOUT GENERATED CODE
|
2
|
+
#
|
3
|
+
# This file is generated. See the contributing guide for more information:
|
4
|
+
# https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md
|
5
|
+
#
|
6
|
+
# WARNING ABOUT GENERATED CODE
|
7
|
+
|
8
|
+
module Aws::S3
|
9
|
+
module EventStreams
|
10
|
+
class SelectObjectContentEventStream
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@event_emitter = Aws::EventEmitter.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def on_records_event(&block)
|
17
|
+
@event_emitter.on(:records, Proc.new)
|
18
|
+
end
|
19
|
+
|
20
|
+
def on_stats_event(&block)
|
21
|
+
@event_emitter.on(:stats, Proc.new)
|
22
|
+
end
|
23
|
+
|
24
|
+
def on_progress_event(&block)
|
25
|
+
@event_emitter.on(:progress, Proc.new)
|
26
|
+
end
|
27
|
+
|
28
|
+
def on_cont_event(&block)
|
29
|
+
@event_emitter.on(:cont, Proc.new)
|
30
|
+
end
|
31
|
+
|
32
|
+
def on_end_event(&block)
|
33
|
+
@event_emitter.on(:end, Proc.new)
|
34
|
+
end
|
35
|
+
|
36
|
+
def on_error_event(&block)
|
37
|
+
@event_emitter.on(:error, Proc.new)
|
38
|
+
end
|
39
|
+
|
40
|
+
def on_initial_response_event(&block)
|
41
|
+
@event_emitter.on(:initial_response, Proc.new)
|
42
|
+
end
|
43
|
+
|
44
|
+
def on_event(&block)
|
45
|
+
on_records_event(&block)
|
46
|
+
on_stats_event(&block)
|
47
|
+
on_progress_event(&block)
|
48
|
+
on_cont_event(&block)
|
49
|
+
on_end_event(&block)
|
50
|
+
on_error_event(&block)
|
51
|
+
on_initial_response_event(&block)
|
52
|
+
end
|
53
|
+
|
54
|
+
# @api private
|
55
|
+
# @return Aws::EventEmitter
|
56
|
+
attr_reader :event_emitter
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
@@ -24,15 +24,18 @@ module Aws
|
|
24
24
|
@mode = options[:mode] || "auto"
|
25
25
|
@thread_count = options[:thread_count] || THREAD_COUNT
|
26
26
|
@chunk_size = options[:chunk_size]
|
27
|
-
@
|
28
|
-
|
27
|
+
@params = {
|
28
|
+
bucket: options[:bucket],
|
29
|
+
key: options[:key],
|
30
|
+
}
|
31
|
+
@params[:version_id] = options[:version_id] if options[:version_id]
|
29
32
|
|
30
33
|
case @mode
|
31
34
|
when "auto" then multipart_download
|
32
35
|
when "single_request" then single_request
|
33
36
|
when "get_range"
|
34
37
|
if @chunk_size
|
35
|
-
resp = @client.head_object(
|
38
|
+
resp = @client.head_object(@params)
|
36
39
|
multithreaded_get_by_ranges(construct_chunks(resp.content_length))
|
37
40
|
else
|
38
41
|
msg = "In :get_range mode, :chunk_size must be provided"
|
@@ -48,7 +51,7 @@ module Aws
|
|
48
51
|
private
|
49
52
|
|
50
53
|
def multipart_download
|
51
|
-
resp = @client.head_object(
|
54
|
+
resp = @client.head_object(@params.merge(part_number: 1))
|
52
55
|
count = resp.parts_count
|
53
56
|
if count.nil? || count <= 1
|
54
57
|
resp.content_length < MIN_CHUNK_SIZE ?
|
@@ -56,7 +59,7 @@ module Aws
|
|
56
59
|
multithreaded_get_by_ranges(construct_chunks(resp.content_length))
|
57
60
|
else
|
58
61
|
# partNumber is an option
|
59
|
-
resp = @client.head_object(
|
62
|
+
resp = @client.head_object(@params)
|
60
63
|
resp.content_length < MIN_CHUNK_SIZE ?
|
61
64
|
single_request :
|
62
65
|
compute_mode(resp.content_length, count)
|
@@ -112,9 +115,7 @@ module Aws
|
|
112
115
|
batch.each do |chunk|
|
113
116
|
threads << Thread.new do
|
114
117
|
resp = @client.get_object(
|
115
|
-
|
116
|
-
:key => @key,
|
117
|
-
param.to_sym => chunk
|
118
|
+
@params.merge(param.to_sym => chunk)
|
118
119
|
)
|
119
120
|
write(resp)
|
120
121
|
end
|
@@ -131,7 +132,7 @@ module Aws
|
|
131
132
|
|
132
133
|
def single_request
|
133
134
|
@client.get_object(
|
134
|
-
|
135
|
+
@params.merge(response_target: @path)
|
135
136
|
)
|
136
137
|
end
|
137
138
|
end
|
data/lib/aws-sdk-s3/file_part.rb
CHANGED
@@ -56,14 +56,12 @@ module Aws
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def read_from_file(bytes, output_buffer)
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
else
|
63
|
-
data = @file.read(remaining_bytes)
|
64
|
-
end
|
59
|
+
length = [remaining_bytes, *bytes].min
|
60
|
+
data = @file.read(length, output_buffer)
|
61
|
+
|
65
62
|
@position += data ? data.bytesize : 0
|
66
|
-
|
63
|
+
|
64
|
+
data.to_s unless bytes && (data.nil? || data.empty?)
|
67
65
|
end
|
68
66
|
|
69
67
|
def remaining_bytes
|
@@ -0,0 +1,162 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'set'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'stringio'
|
5
|
+
|
6
|
+
module Aws
|
7
|
+
module S3
|
8
|
+
# @api private
|
9
|
+
class MultipartStreamUploader
|
10
|
+
# api private
|
11
|
+
PART_SIZE = 5 * 1024 * 1024 # 5MB
|
12
|
+
|
13
|
+
# api private
|
14
|
+
THREAD_COUNT = 10
|
15
|
+
|
16
|
+
# api private
|
17
|
+
TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze
|
18
|
+
|
19
|
+
# @api private
|
20
|
+
CREATE_OPTIONS =
|
21
|
+
Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
|
22
|
+
|
23
|
+
# @api private
|
24
|
+
UPLOAD_PART_OPTIONS =
|
25
|
+
Set.new(Client.api.operation(:upload_part).input.shape.member_names)
|
26
|
+
|
27
|
+
# @option options [Client] :client
|
28
|
+
def initialize(options = {})
|
29
|
+
@client = options[:client] || Client.new
|
30
|
+
@tempfile = options[:tempfile]
|
31
|
+
@part_size = options[:part_size] || PART_SIZE
|
32
|
+
@thread_count = options[:thread_count] || THREAD_COUNT
|
33
|
+
end
|
34
|
+
|
35
|
+
# @return [Client]
|
36
|
+
attr_reader :client
|
37
|
+
|
38
|
+
# @option options [required,String] :bucket
|
39
|
+
# @option options [required,String] :key
|
40
|
+
# @return [void]
|
41
|
+
def upload(options = {}, &block)
|
42
|
+
upload_id = initiate_upload(options)
|
43
|
+
parts = upload_parts(upload_id, options, &block)
|
44
|
+
complete_upload(upload_id, parts, options)
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def initiate_upload(options)
|
50
|
+
@client.create_multipart_upload(create_opts(options)).upload_id
|
51
|
+
end
|
52
|
+
|
53
|
+
def complete_upload(upload_id, parts, options)
|
54
|
+
@client.complete_multipart_upload(
|
55
|
+
bucket: options[:bucket],
|
56
|
+
key: options[:key],
|
57
|
+
upload_id: upload_id,
|
58
|
+
multipart_upload: { parts: parts })
|
59
|
+
end
|
60
|
+
|
61
|
+
def upload_parts(upload_id, options, &block)
|
62
|
+
completed = Queue.new
|
63
|
+
errors = IO.pipe do |read_pipe, write_pipe|
|
64
|
+
threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
|
65
|
+
block.call(write_pipe)
|
66
|
+
write_pipe.close
|
67
|
+
threads.map(&:value).compact
|
68
|
+
end
|
69
|
+
if errors.empty?
|
70
|
+
Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] }
|
71
|
+
else
|
72
|
+
abort_upload(upload_id, options, errors)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def abort_upload(upload_id, options, errors)
|
77
|
+
@client.abort_multipart_upload(
|
78
|
+
bucket: options[:bucket],
|
79
|
+
key: options[:key],
|
80
|
+
upload_id: upload_id
|
81
|
+
)
|
82
|
+
msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
|
83
|
+
raise MultipartUploadError.new(msg, errors)
|
84
|
+
rescue MultipartUploadError => error
|
85
|
+
raise error
|
86
|
+
rescue => error
|
87
|
+
msg = "failed to abort multipart upload: #{error.message}"
|
88
|
+
raise MultipartUploadError.new(msg, errors + [error])
|
89
|
+
end
|
90
|
+
|
91
|
+
def create_opts(options)
|
92
|
+
CREATE_OPTIONS.inject({}) do |hash, key|
|
93
|
+
hash[key] = options[key] if options.key?(key)
|
94
|
+
hash
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def upload_part_opts(options)
|
99
|
+
UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
|
100
|
+
hash[key] = options[key] if options.key?(key)
|
101
|
+
hash
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def read_to_part_body(read_pipe)
|
106
|
+
return if read_pipe.closed?
|
107
|
+
temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new
|
108
|
+
temp_io.binmode
|
109
|
+
bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
|
110
|
+
temp_io.rewind
|
111
|
+
if bytes_copied == 0
|
112
|
+
if Tempfile === temp_io
|
113
|
+
temp_io.close
|
114
|
+
temp_io.unlink
|
115
|
+
end
|
116
|
+
nil
|
117
|
+
else
|
118
|
+
temp_io
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def upload_in_threads(read_pipe, completed, options)
|
123
|
+
mutex = Mutex.new
|
124
|
+
part_number = 0
|
125
|
+
@thread_count.times.map do
|
126
|
+
thread = Thread.new do
|
127
|
+
begin
|
128
|
+
loop do
|
129
|
+
body, thread_part_number = mutex.synchronize do
|
130
|
+
[read_to_part_body(read_pipe), part_number += 1]
|
131
|
+
end
|
132
|
+
break unless (body || thread_part_number == 1)
|
133
|
+
begin
|
134
|
+
part = options.merge(
|
135
|
+
body: body,
|
136
|
+
part_number: thread_part_number,
|
137
|
+
)
|
138
|
+
resp = @client.upload_part(part)
|
139
|
+
completed << {etag: resp.etag, part_number: part[:part_number]}
|
140
|
+
ensure
|
141
|
+
if Tempfile === body
|
142
|
+
body.close
|
143
|
+
body.unlink
|
144
|
+
elsif StringIO === body
|
145
|
+
body.string.clear
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
nil
|
150
|
+
rescue => error
|
151
|
+
# keep other threads from uploading other parts
|
152
|
+
mutex.synchronize { read_pipe.close_read }
|
153
|
+
error
|
154
|
+
end
|
155
|
+
end
|
156
|
+
thread.abort_on_exception = true
|
157
|
+
thread
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
@@ -66,7 +66,7 @@ module Aws::S3
|
|
66
66
|
data[:etag]
|
67
67
|
end
|
68
68
|
|
69
|
-
# Size of the uploaded part data.
|
69
|
+
# Size in bytes of the uploaded part data.
|
70
70
|
# @return [Integer]
|
71
71
|
def size
|
72
72
|
data[:size]
|
@@ -235,7 +235,7 @@ module Aws::S3
|
|
235
235
|
# must use the form bytes=first-last, where the first and last are the
|
236
236
|
# zero-based byte offsets to copy. For example, bytes=0-9 indicates that
|
237
237
|
# you want to copy the first ten bytes of the source. You can copy a
|
238
|
-
# range only if the source object is greater than 5
|
238
|
+
# range only if the source object is greater than 5 MB.
|
239
239
|
# @option options [String] :sse_customer_algorithm
|
240
240
|
# Specifies the algorithm to use to when encrypting the object (e.g.,
|
241
241
|
# AES256).
|