aws-sdk-s3 1.9.0 → 1.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/lib/aws-sdk-s3/bucket.rb +22 -3
  3. data/lib/aws-sdk-s3/bucket_acl.rb +1 -0
  4. data/lib/aws-sdk-s3/bucket_cors.rb +1 -0
  5. data/lib/aws-sdk-s3/bucket_lifecycle.rb +3 -2
  6. data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +3 -2
  7. data/lib/aws-sdk-s3/bucket_notification.rb +9 -5
  8. data/lib/aws-sdk-s3/bucket_tagging.rb +1 -0
  9. data/lib/aws-sdk-s3/bucket_website.rb +4 -0
  10. data/lib/aws-sdk-s3/client.rb +1303 -351
  11. data/lib/aws-sdk-s3/client_api.rb +352 -2
  12. data/lib/aws-sdk-s3/customizations/bucket.rb +2 -2
  13. data/lib/aws-sdk-s3/customizations/object.rb +60 -0
  14. data/lib/aws-sdk-s3/customizations/object_summary.rb +7 -0
  15. data/lib/aws-sdk-s3/customizations.rb +1 -0
  16. data/lib/aws-sdk-s3/encryption/client.rb +1 -1
  17. data/lib/aws-sdk-s3/encryption/io_auth_decrypter.rb +9 -3
  18. data/lib/aws-sdk-s3/event_streams.rb +62 -0
  19. data/lib/aws-sdk-s3/file_downloader.rb +10 -9
  20. data/lib/aws-sdk-s3/file_part.rb +5 -7
  21. data/lib/aws-sdk-s3/file_uploader.rb +1 -3
  22. data/lib/aws-sdk-s3/multipart_stream_uploader.rb +162 -0
  23. data/lib/aws-sdk-s3/multipart_upload.rb +1 -0
  24. data/lib/aws-sdk-s3/multipart_upload_part.rb +2 -2
  25. data/lib/aws-sdk-s3/object.rb +71 -7
  26. data/lib/aws-sdk-s3/object_acl.rb +1 -0
  27. data/lib/aws-sdk-s3/object_summary.rb +54 -7
  28. data/lib/aws-sdk-s3/object_version.rb +11 -0
  29. data/lib/aws-sdk-s3/plugins/accelerate.rb +4 -0
  30. data/lib/aws-sdk-s3/plugins/md5s.rb +4 -1
  31. data/lib/aws-sdk-s3/plugins/redirects.rb +3 -1
  32. data/lib/aws-sdk-s3/plugins/s3_signer.rb +6 -1
  33. data/lib/aws-sdk-s3/presigner.rb +4 -0
  34. data/lib/aws-sdk-s3/resource.rb +4 -0
  35. data/lib/aws-sdk-s3/types.rb +1539 -227
  36. data/lib/aws-sdk-s3.rb +2 -1
  37. metadata +12 -4
@@ -61,8 +61,8 @@ module Aws
61
61
  # You can pass `virtual_host: true` to use the bucket name as the
62
62
  # host name.
63
63
  #
64
- # bucket = s3.bucket('my.bucket.com', virtual_host: true)
65
- # bucket.url
64
+ # bucket = s3.bucket('my.bucket.com')
65
+ # bucket.url(virtual_host: true)
66
66
  # #=> "http://my.bucket.com"
67
67
  #
68
68
  # @option options [Boolean] :virtual_host (false) When `true`,
@@ -214,6 +214,62 @@ module Aws
214
214
  url.to_s
215
215
  end
216
216
 
217
+ # Uploads a stream in a streaming fashion to the current object in S3.
218
+ #
219
+ # # Passed chunks automatically split into multipart upload parts
220
+ # # and the parts are uploaded in parallel. This allows for streaming uploads
221
+ # # that never touch the disk.
222
+ #
223
+ # Note that this is known to have issues in JRuby until jruby-9.1.15.0, so avoid using this with older versions of JRuby.
224
+ #
225
+ # @example Streaming chunks of data
226
+ # obj.upload_stream do |write_stream|
227
+ # 10.times { write_stream << 'foo' }
228
+ # end
229
+ # @example Streaming chunks of data
230
+ # obj.upload_stream do |write_stream|
231
+ # IO.copy_stream(IO.popen('ls'), write_stream)
232
+ # end
233
+ # @example Streaming chunks of data
234
+ # obj.upload_stream do |write_stream|
235
+ # IO.copy_stream(STDIN, write_stream)
236
+ # end
237
+ #
238
+ # @option options [Integer] :thread_count
239
+ # The number of parallel multipart uploads
240
+ # Default `:thread_count` is `10`.
241
+ #
242
+ # @option options [Boolean] :tempfile
243
+ # Normally read data is stored in memory when building the parts in order to complete
244
+ # the underlying multipart upload. By passing `:tempfile => true` data read will be
245
+ # temporarily stored on disk reducing the memory footprint vastly.
246
+ # Default `:tempfile` is `false`.
247
+ #
248
+ # @option options [Integer] :part_size
249
+ # Define how big each part size but the last should be.
250
+ # Default `:part_size` is `5 * 1024 * 1024`.
251
+ #
252
+ # @raise [MultipartUploadError] If an object is being uploaded in
253
+ # parts, and the upload can not be completed, then the upload is
254
+ # aborted and this error is raised. The raised error has a `#errors`
255
+ # method that returns the failures that caused the upload to be
256
+ # aborted.
257
+ #
258
+ # @return [Boolean] Returns `true` when the object is uploaded
259
+ # without any errors.
260
+ #
261
+ def upload_stream(options = {}, &block)
262
+ uploading_options = options.dup
263
+ uploader = MultipartStreamUploader.new(
264
+ client: client,
265
+ thread_count: uploading_options.delete(:thread_count),
266
+ tempfile: uploading_options.delete(:tempfile),
267
+ part_size: uploading_options.delete(:part_size),
268
+ )
269
+ uploader.upload(uploading_options.merge(bucket: bucket_name, key: key), &block)
270
+ true
271
+ end
272
+
217
273
  # Uploads a file from disk to the current object in S3.
218
274
  #
219
275
  # # small files are uploaded in a single API call
@@ -277,6 +333,10 @@ module Aws
277
333
  # @option options [String] thread_count Customize threads used in multipart
278
334
  # download, if not provided, 10 is default value
279
335
  #
336
+ # @option options [String] version_id The object version id used to retrieve
337
+ # the object, to know more about object versioning, see:
338
+ # https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
339
+ #
280
340
  # @return [Boolean] Returns `true` when the file is downloaded
281
341
  # without any errors.
282
342
  def download_file(destination, options = {})
@@ -60,6 +60,13 @@ module Aws
60
60
  object.upload_file(source, options)
61
61
  end
62
62
 
63
+ # @options (see Object#upload_stream)
64
+ # @return (see Object#upload_stream)
65
+ # @see Object#upload_stream
66
+ def upload_stream(options = {}, &block)
67
+ object.upload_stream(options, &block)
68
+ end
69
+
63
70
  # @param (see Object#download_file)
64
71
  # @options (see Object#download_file)
65
72
  # @return (see Object#download_file)
@@ -6,6 +6,7 @@ require 'aws-sdk-s3/file_uploader'
6
6
  require 'aws-sdk-s3/file_downloader'
7
7
  require 'aws-sdk-s3/legacy_signer'
8
8
  require 'aws-sdk-s3/multipart_file_uploader'
9
+ require 'aws-sdk-s3/multipart_stream_uploader'
9
10
  require 'aws-sdk-s3/multipart_upload_error'
10
11
  require 'aws-sdk-s3/object_copier'
11
12
  require 'aws-sdk-s3/object_multipart_copier'
@@ -179,7 +179,7 @@ module Aws
179
179
 
180
180
  extend Deprecations
181
181
 
182
- # Creates a new encryption client. You must provide on of the following
182
+ # Creates a new encryption client. You must provide one of the following
183
183
  # options:
184
184
  #
185
185
  # * `:encryption_key`
@@ -22,8 +22,10 @@ module Aws
22
22
 
23
23
  def write(chunk)
24
24
  chunk = truncate_chunk(chunk)
25
- @bytes_written += chunk.bytesize
26
- @decrypter.write(chunk)
25
+ if chunk.bytesize > 0
26
+ @bytes_written += chunk.bytesize
27
+ @decrypter.write(chunk)
28
+ end
27
29
  end
28
30
 
29
31
  def finalize
@@ -39,8 +41,12 @@ module Aws
39
41
  def truncate_chunk(chunk)
40
42
  if chunk.bytesize + @bytes_written <= @max_bytes
41
43
  chunk
42
- else
44
+ elsif @bytes_written < @max_bytes
43
45
  chunk[0..(@max_bytes - @bytes_written - 1)]
46
+ else
47
+ # If the tag was sent over after the full body has been read,
48
+ # we don't want to accidentally append it.
49
+ ""
44
50
  end
45
51
  end
46
52
 
@@ -0,0 +1,62 @@
1
+ # WARNING ABOUT GENERATED CODE
2
+ #
3
+ # This file is generated. See the contributing guide for more information:
4
+ # https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md
5
+ #
6
+ # WARNING ABOUT GENERATED CODE
7
+
8
+ module Aws::S3
9
+ module EventStreams
10
+ class SelectObjectContentEventStream
11
+
12
+ def initialize
13
+ @event_emitter = Aws::EventEmitter.new
14
+ end
15
+
16
+ def on_records_event(&block)
17
+ @event_emitter.on(:records, Proc.new)
18
+ end
19
+
20
+ def on_stats_event(&block)
21
+ @event_emitter.on(:stats, Proc.new)
22
+ end
23
+
24
+ def on_progress_event(&block)
25
+ @event_emitter.on(:progress, Proc.new)
26
+ end
27
+
28
+ def on_cont_event(&block)
29
+ @event_emitter.on(:cont, Proc.new)
30
+ end
31
+
32
+ def on_end_event(&block)
33
+ @event_emitter.on(:end, Proc.new)
34
+ end
35
+
36
+ def on_error_event(&block)
37
+ @event_emitter.on(:error, Proc.new)
38
+ end
39
+
40
+ def on_initial_response_event(&block)
41
+ @event_emitter.on(:initial_response, Proc.new)
42
+ end
43
+
44
+ def on_event(&block)
45
+ on_records_event(&block)
46
+ on_stats_event(&block)
47
+ on_progress_event(&block)
48
+ on_cont_event(&block)
49
+ on_end_event(&block)
50
+ on_error_event(&block)
51
+ on_initial_response_event(&block)
52
+ end
53
+
54
+ # @api private
55
+ # @return Aws::EventEmitter
56
+ attr_reader :event_emitter
57
+
58
+ end
59
+
60
+ end
61
+ end
62
+
@@ -24,15 +24,18 @@ module Aws
24
24
  @mode = options[:mode] || "auto"
25
25
  @thread_count = options[:thread_count] || THREAD_COUNT
26
26
  @chunk_size = options[:chunk_size]
27
- @bucket = options[:bucket]
28
- @key = options[:key]
27
+ @params = {
28
+ bucket: options[:bucket],
29
+ key: options[:key],
30
+ }
31
+ @params[:version_id] = options[:version_id] if options[:version_id]
29
32
 
30
33
  case @mode
31
34
  when "auto" then multipart_download
32
35
  when "single_request" then single_request
33
36
  when "get_range"
34
37
  if @chunk_size
35
- resp = @client.head_object(bucket: @bucket, key: @key)
38
+ resp = @client.head_object(@params)
36
39
  multithreaded_get_by_ranges(construct_chunks(resp.content_length))
37
40
  else
38
41
  msg = "In :get_range mode, :chunk_size must be provided"
@@ -48,7 +51,7 @@ module Aws
48
51
  private
49
52
 
50
53
  def multipart_download
51
- resp = @client.head_object(bucket: @bucket, key: @key, part_number: 1)
54
+ resp = @client.head_object(@params.merge(part_number: 1))
52
55
  count = resp.parts_count
53
56
  if count.nil? || count <= 1
54
57
  resp.content_length < MIN_CHUNK_SIZE ?
@@ -56,7 +59,7 @@ module Aws
56
59
  multithreaded_get_by_ranges(construct_chunks(resp.content_length))
57
60
  else
58
61
  # partNumber is an option
59
- resp = @client.head_object(bucket: @bucket, key: @key)
62
+ resp = @client.head_object(@params)
60
63
  resp.content_length < MIN_CHUNK_SIZE ?
61
64
  single_request :
62
65
  compute_mode(resp.content_length, count)
@@ -112,9 +115,7 @@ module Aws
112
115
  batch.each do |chunk|
113
116
  threads << Thread.new do
114
117
  resp = @client.get_object(
115
- :bucket => @bucket,
116
- :key => @key,
117
- param.to_sym => chunk
118
+ @params.merge(param.to_sym => chunk)
118
119
  )
119
120
  write(resp)
120
121
  end
@@ -131,7 +132,7 @@ module Aws
131
132
 
132
133
  def single_request
133
134
  @client.get_object(
134
- bucket: @bucket, key: @key, response_target: @path
135
+ @params.merge(response_target: @path)
135
136
  )
136
137
  end
137
138
  end
@@ -56,14 +56,12 @@ module Aws
56
56
  end
57
57
 
58
58
  def read_from_file(bytes, output_buffer)
59
- if bytes
60
- data = @file.read([remaining_bytes, bytes].min)
61
- data = nil if data == ''
62
- else
63
- data = @file.read(remaining_bytes)
64
- end
59
+ length = [remaining_bytes, *bytes].min
60
+ data = @file.read(length, output_buffer)
61
+
65
62
  @position += data ? data.bytesize : 0
66
- output_buffer ? output_buffer.replace(data || '') : data
63
+
64
+ data.to_s unless bytes && (data.nil? || data.empty?)
67
65
  end
68
66
 
69
67
  def remaining_bytes
@@ -45,9 +45,7 @@ module Aws
45
45
 
46
46
  def open_file(source)
47
47
  if String === source || Pathname === source
48
- file = File.open(source, 'rb')
49
- yield(file)
50
- file.close
48
+ File.open(source, 'rb') { |file| yield(file) }
51
49
  else
52
50
  yield(source)
53
51
  end
@@ -0,0 +1,162 @@
1
+ require 'thread'
2
+ require 'set'
3
+ require 'tempfile'
4
+ require 'stringio'
5
+
6
+ module Aws
7
+ module S3
8
+ # @api private
9
+ class MultipartStreamUploader
10
+ # api private
11
+ PART_SIZE = 5 * 1024 * 1024 # 5MB
12
+
13
+ # api private
14
+ THREAD_COUNT = 10
15
+
16
+ # api private
17
+ TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze
18
+
19
+ # @api private
20
+ CREATE_OPTIONS =
21
+ Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
22
+
23
+ # @api private
24
+ UPLOAD_PART_OPTIONS =
25
+ Set.new(Client.api.operation(:upload_part).input.shape.member_names)
26
+
27
+ # @option options [Client] :client
28
+ def initialize(options = {})
29
+ @client = options[:client] || Client.new
30
+ @tempfile = options[:tempfile]
31
+ @part_size = options[:part_size] || PART_SIZE
32
+ @thread_count = options[:thread_count] || THREAD_COUNT
33
+ end
34
+
35
+ # @return [Client]
36
+ attr_reader :client
37
+
38
+ # @option options [required,String] :bucket
39
+ # @option options [required,String] :key
40
+ # @return [void]
41
+ def upload(options = {}, &block)
42
+ upload_id = initiate_upload(options)
43
+ parts = upload_parts(upload_id, options, &block)
44
+ complete_upload(upload_id, parts, options)
45
+ end
46
+
47
+ private
48
+
49
+ def initiate_upload(options)
50
+ @client.create_multipart_upload(create_opts(options)).upload_id
51
+ end
52
+
53
+ def complete_upload(upload_id, parts, options)
54
+ @client.complete_multipart_upload(
55
+ bucket: options[:bucket],
56
+ key: options[:key],
57
+ upload_id: upload_id,
58
+ multipart_upload: { parts: parts })
59
+ end
60
+
61
+ def upload_parts(upload_id, options, &block)
62
+ completed = Queue.new
63
+ errors = IO.pipe do |read_pipe, write_pipe|
64
+ threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
65
+ block.call(write_pipe)
66
+ write_pipe.close
67
+ threads.map(&:value).compact
68
+ end
69
+ if errors.empty?
70
+ Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] }
71
+ else
72
+ abort_upload(upload_id, options, errors)
73
+ end
74
+ end
75
+
76
+ def abort_upload(upload_id, options, errors)
77
+ @client.abort_multipart_upload(
78
+ bucket: options[:bucket],
79
+ key: options[:key],
80
+ upload_id: upload_id
81
+ )
82
+ msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
83
+ raise MultipartUploadError.new(msg, errors)
84
+ rescue MultipartUploadError => error
85
+ raise error
86
+ rescue => error
87
+ msg = "failed to abort multipart upload: #{error.message}"
88
+ raise MultipartUploadError.new(msg, errors + [error])
89
+ end
90
+
91
+ def create_opts(options)
92
+ CREATE_OPTIONS.inject({}) do |hash, key|
93
+ hash[key] = options[key] if options.key?(key)
94
+ hash
95
+ end
96
+ end
97
+
98
+ def upload_part_opts(options)
99
+ UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
100
+ hash[key] = options[key] if options.key?(key)
101
+ hash
102
+ end
103
+ end
104
+
105
+ def read_to_part_body(read_pipe)
106
+ return if read_pipe.closed?
107
+ temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new
108
+ temp_io.binmode
109
+ bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
110
+ temp_io.rewind
111
+ if bytes_copied == 0
112
+ if Tempfile === temp_io
113
+ temp_io.close
114
+ temp_io.unlink
115
+ end
116
+ nil
117
+ else
118
+ temp_io
119
+ end
120
+ end
121
+
122
+ def upload_in_threads(read_pipe, completed, options)
123
+ mutex = Mutex.new
124
+ part_number = 0
125
+ @thread_count.times.map do
126
+ thread = Thread.new do
127
+ begin
128
+ loop do
129
+ body, thread_part_number = mutex.synchronize do
130
+ [read_to_part_body(read_pipe), part_number += 1]
131
+ end
132
+ break unless (body || thread_part_number == 1)
133
+ begin
134
+ part = options.merge(
135
+ body: body,
136
+ part_number: thread_part_number,
137
+ )
138
+ resp = @client.upload_part(part)
139
+ completed << {etag: resp.etag, part_number: part[:part_number]}
140
+ ensure
141
+ if Tempfile === body
142
+ body.close
143
+ body.unlink
144
+ elsif StringIO === body
145
+ body.string.clear
146
+ end
147
+ end
148
+ end
149
+ nil
150
+ rescue => error
151
+ # keep other threads from uploading other parts
152
+ mutex.synchronize { read_pipe.close_read }
153
+ error
154
+ end
155
+ end
156
+ thread.abort_on_exception = true
157
+ thread
158
+ end
159
+ end
160
+ end
161
+ end
162
+ end
@@ -70,6 +70,7 @@ module Aws::S3
70
70
  data[:storage_class]
71
71
  end
72
72
 
73
+
73
74
  # @return [Types::Owner]
74
75
  def owner
75
76
  data[:owner]
@@ -66,7 +66,7 @@ module Aws::S3
66
66
  data[:etag]
67
67
  end
68
68
 
69
- # Size of the uploaded part data.
69
+ # Size in bytes of the uploaded part data.
70
70
  # @return [Integer]
71
71
  def size
72
72
  data[:size]
@@ -235,7 +235,7 @@ module Aws::S3
235
235
  # must use the form bytes=first-last, where the first and last are the
236
236
  # zero-based byte offsets to copy. For example, bytes=0-9 indicates that
237
237
  # you want to copy the first ten bytes of the source. You can copy a
238
- # range only if the source object is greater than 5 GB.
238
+ # range only if the source object is greater than 5 MB.
239
239
  # @option options [String] :sse_customer_algorithm
240
240
  # Specifies the algorithm to use to when encrypting the object (e.g.,
241
241
  # AES256).