aws-sdk-s3 1.30.0 → 1.114.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (100) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +903 -0
  3. data/LICENSE.txt +202 -0
  4. data/VERSION +1 -0
  5. data/lib/aws-sdk-s3/arn/access_point_arn.rb +69 -0
  6. data/lib/aws-sdk-s3/arn/multi_region_access_point_arn.rb +68 -0
  7. data/lib/aws-sdk-s3/arn/object_lambda_arn.rb +69 -0
  8. data/lib/aws-sdk-s3/arn/outpost_access_point_arn.rb +74 -0
  9. data/lib/aws-sdk-s3/bucket.rb +377 -77
  10. data/lib/aws-sdk-s3/bucket_acl.rb +57 -14
  11. data/lib/aws-sdk-s3/bucket_cors.rb +67 -13
  12. data/lib/aws-sdk-s3/bucket_lifecycle.rb +54 -15
  13. data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +56 -15
  14. data/lib/aws-sdk-s3/bucket_logging.rb +52 -15
  15. data/lib/aws-sdk-s3/bucket_notification.rb +44 -15
  16. data/lib/aws-sdk-s3/bucket_policy.rb +51 -13
  17. data/lib/aws-sdk-s3/bucket_region_cache.rb +2 -0
  18. data/lib/aws-sdk-s3/bucket_request_payment.rb +51 -12
  19. data/lib/aws-sdk-s3/bucket_tagging.rb +59 -13
  20. data/lib/aws-sdk-s3/bucket_versioning.rb +118 -12
  21. data/lib/aws-sdk-s3/bucket_website.rb +66 -13
  22. data/lib/aws-sdk-s3/client.rb +9016 -669
  23. data/lib/aws-sdk-s3/client_api.rb +786 -2
  24. data/lib/aws-sdk-s3/customizations/bucket.rb +59 -16
  25. data/lib/aws-sdk-s3/customizations/multipart_upload.rb +2 -0
  26. data/lib/aws-sdk-s3/customizations/object.rb +200 -62
  27. data/lib/aws-sdk-s3/customizations/object_summary.rb +5 -0
  28. data/lib/aws-sdk-s3/customizations/types/list_object_versions_output.rb +2 -0
  29. data/lib/aws-sdk-s3/customizations.rb +4 -1
  30. data/lib/aws-sdk-s3/encryption/client.rb +23 -6
  31. data/lib/aws-sdk-s3/encryption/decrypt_handler.rb +71 -29
  32. data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb +43 -5
  33. data/lib/aws-sdk-s3/encryption/default_key_provider.rb +2 -0
  34. data/lib/aws-sdk-s3/encryption/encrypt_handler.rb +13 -2
  35. data/lib/aws-sdk-s3/encryption/errors.rb +2 -0
  36. data/lib/aws-sdk-s3/encryption/io_auth_decrypter.rb +2 -0
  37. data/lib/aws-sdk-s3/encryption/io_decrypter.rb +11 -3
  38. data/lib/aws-sdk-s3/encryption/io_encrypter.rb +2 -0
  39. data/lib/aws-sdk-s3/encryption/key_provider.rb +2 -0
  40. data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +34 -3
  41. data/lib/aws-sdk-s3/encryption/materials.rb +8 -6
  42. data/lib/aws-sdk-s3/encryption/utils.rb +25 -0
  43. data/lib/aws-sdk-s3/encryption.rb +4 -0
  44. data/lib/aws-sdk-s3/encryptionV2/client.rb +566 -0
  45. data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +222 -0
  46. data/lib/aws-sdk-s3/encryptionV2/default_cipher_provider.rb +170 -0
  47. data/lib/aws-sdk-s3/encryptionV2/default_key_provider.rb +40 -0
  48. data/lib/aws-sdk-s3/encryptionV2/encrypt_handler.rb +65 -0
  49. data/lib/aws-sdk-s3/encryptionV2/errors.rb +37 -0
  50. data/lib/aws-sdk-s3/encryptionV2/io_auth_decrypter.rb +58 -0
  51. data/lib/aws-sdk-s3/encryptionV2/io_decrypter.rb +37 -0
  52. data/lib/aws-sdk-s3/encryptionV2/io_encrypter.rb +73 -0
  53. data/lib/aws-sdk-s3/encryptionV2/key_provider.rb +31 -0
  54. data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +169 -0
  55. data/lib/aws-sdk-s3/encryptionV2/materials.rb +60 -0
  56. data/lib/aws-sdk-s3/encryptionV2/utils.rb +103 -0
  57. data/lib/aws-sdk-s3/encryption_v2.rb +23 -0
  58. data/lib/aws-sdk-s3/errors.rb +123 -1
  59. data/lib/aws-sdk-s3/event_streams.rb +20 -7
  60. data/lib/aws-sdk-s3/file_downloader.rb +17 -10
  61. data/lib/aws-sdk-s3/file_part.rb +11 -6
  62. data/lib/aws-sdk-s3/file_uploader.rb +33 -14
  63. data/lib/aws-sdk-s3/legacy_signer.rb +17 -25
  64. data/lib/aws-sdk-s3/multipart_file_uploader.rb +78 -19
  65. data/lib/aws-sdk-s3/multipart_stream_uploader.rb +20 -7
  66. data/lib/aws-sdk-s3/multipart_upload.rb +178 -28
  67. data/lib/aws-sdk-s3/multipart_upload_error.rb +2 -0
  68. data/lib/aws-sdk-s3/multipart_upload_part.rb +236 -43
  69. data/lib/aws-sdk-s3/object.rb +848 -162
  70. data/lib/aws-sdk-s3/object_acl.rb +81 -20
  71. data/lib/aws-sdk-s3/object_copier.rb +2 -0
  72. data/lib/aws-sdk-s3/object_multipart_copier.rb +2 -0
  73. data/lib/aws-sdk-s3/object_summary.rb +615 -144
  74. data/lib/aws-sdk-s3/object_version.rb +161 -68
  75. data/lib/aws-sdk-s3/plugins/accelerate.rb +38 -38
  76. data/lib/aws-sdk-s3/plugins/arn.rb +254 -0
  77. data/lib/aws-sdk-s3/plugins/bucket_dns.rb +8 -8
  78. data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb +25 -3
  79. data/lib/aws-sdk-s3/plugins/dualstack.rb +38 -33
  80. data/lib/aws-sdk-s3/plugins/expect_100_continue.rb +4 -4
  81. data/lib/aws-sdk-s3/plugins/get_bucket_location_fix.rb +3 -1
  82. data/lib/aws-sdk-s3/plugins/http_200_errors.rb +11 -3
  83. data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb +73 -0
  84. data/lib/aws-sdk-s3/plugins/location_constraint.rb +2 -0
  85. data/lib/aws-sdk-s3/plugins/md5s.rb +34 -30
  86. data/lib/aws-sdk-s3/plugins/object_lambda_endpoint.rb +25 -0
  87. data/lib/aws-sdk-s3/plugins/redirects.rb +2 -0
  88. data/lib/aws-sdk-s3/plugins/s3_host_id.rb +2 -0
  89. data/lib/aws-sdk-s3/plugins/s3_signer.rb +95 -36
  90. data/lib/aws-sdk-s3/plugins/skip_whole_multipart_get_checksums.rb +31 -0
  91. data/lib/aws-sdk-s3/plugins/sse_cpk.rb +3 -1
  92. data/lib/aws-sdk-s3/plugins/streaming_retry.rb +139 -0
  93. data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb +2 -0
  94. data/lib/aws-sdk-s3/presigned_post.rb +110 -51
  95. data/lib/aws-sdk-s3/presigner.rb +168 -66
  96. data/lib/aws-sdk-s3/resource.rb +41 -5
  97. data/lib/aws-sdk-s3/types.rb +8652 -1146
  98. data/lib/aws-sdk-s3/waiters.rb +67 -1
  99. data/lib/aws-sdk-s3.rb +12 -6
  100. metadata +38 -13
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'pathname'
2
4
  require 'thread'
3
5
  require 'set'
@@ -21,7 +23,7 @@ module Aws
21
23
 
22
24
  def download(destination, options = {})
23
25
  @path = destination
24
- @mode = options[:mode] || "auto"
26
+ @mode = options[:mode] || 'auto'
25
27
  @thread_count = options[:thread_count] || THREAD_COUNT
26
28
  @chunk_size = options[:chunk_size]
27
29
  @params = {
@@ -31,19 +33,19 @@ module Aws
31
33
  @params[:version_id] = options[:version_id] if options[:version_id]
32
34
 
33
35
  case @mode
34
- when "auto" then multipart_download
35
- when "single_request" then single_request
36
- when "get_range"
36
+ when 'auto' then multipart_download
37
+ when 'single_request' then single_request
38
+ when 'get_range'
37
39
  if @chunk_size
38
40
  resp = @client.head_object(@params)
39
41
  multithreaded_get_by_ranges(construct_chunks(resp.content_length))
40
42
  else
41
- msg = "In :get_range mode, :chunk_size must be provided"
43
+ msg = 'In :get_range mode, :chunk_size must be provided'
42
44
  raise ArgumentError, msg
43
45
  end
44
46
  else
45
47
  msg = "Invalid mode #{@mode} provided, "\
46
- "mode should be :single_request, :get_range or :auto"
48
+ 'mode should be :single_request, :get_range or :auto'
47
49
  raise ArgumentError, msg
48
50
  end
49
51
  end
@@ -92,7 +94,12 @@ module Aws
92
94
  if @chunk_size && @chunk_size > file_size
93
95
  raise ArgumentError, ":chunk_size shouldn't exceed total file size."
94
96
  else
95
- @chunk_size || [(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE].max.to_i
97
+ chunk_size = @chunk_size || [
98
+ (file_size.to_f / MAX_PARTS).ceil,
99
+ MIN_CHUNK_SIZE
100
+ ].max.to_i
101
+ chunk_size -= 1 if file_size % chunk_size == 1
102
+ chunk_size
96
103
  end
97
104
  end
98
105
 
@@ -125,9 +132,9 @@ module Aws
125
132
  end
126
133
 
127
134
  def write(resp)
128
- range, _ = resp.content_range.split(" ").last.split("/")
129
- head, _ = range.split("-").map {|s| s.to_i}
130
- IO.write(@path, resp.body.read, head)
135
+ range, _ = resp.content_range.split(' ').last.split('/')
136
+ head, _ = range.split('-').map {|s| s.to_i}
137
+ File.write(@path, resp.body.read, head)
131
138
  end
132
139
 
133
140
  def single_request
@@ -1,15 +1,20 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Aws
2
4
  module S3
3
5
 
4
- # A utility class that provides an IO-like interface to a portion of
5
- # a file on disk.
6
+ # A utility class that provides an IO-like interface to a portion of a file
7
+ # on disk.
6
8
  # @api private
7
9
  class FilePart
8
10
 
9
- # @option options [required,String,Pathname,File,Tempfile] :source
10
- # @option options [required,Integer] :offset The file part will read
11
+ # @option options [required, String, Pathname, File, Tempfile] :source
12
+ # The file to upload.
13
+ #
14
+ # @option options [required, Integer] :offset The file part will read
11
15
  # starting at this byte offset.
12
- # @option options [required,Integer] :size The maximum number of bytes to
16
+ #
17
+ # @option options [required, Integer] :size The maximum number of bytes to
13
18
  # read from the `:offset`.
14
19
  def initialize(options = {})
15
20
  @source = options[:source]
@@ -19,7 +24,7 @@ module Aws
19
24
  @file = nil
20
25
  end
21
26
 
22
- # @return [String,Pathname,File,Tempfile]
27
+ # @return [String, Pathname, File, Tempfile]
23
28
  attr_reader :source
24
29
 
25
30
  # @return [Integer]
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'pathname'
2
4
 
3
5
  module Aws
@@ -5,44 +7,47 @@ module Aws
5
7
  # @api private
6
8
  class FileUploader
7
9
 
8
- FIFTEEN_MEGABYTES = 15 * 1024 * 1024
10
+ ONE_HUNDRED_MEGABYTES = 100 * 1024 * 1024
9
11
 
12
+ # @param [Hash] options
10
13
  # @option options [Client] :client
11
- # @option options [Integer] :multipart_threshold Files greater than
12
- # `:multipart_threshold` bytes are uploaded using S3 multipart APIs.
14
+ # @option options [Integer] :multipart_threshold (104857600)
13
15
  def initialize(options = {})
14
16
  @options = options
15
17
  @client = options[:client] || Client.new
16
- @multipart_threshold = options[:multipart_threshold] || FIFTEEN_MEGABYTES
18
+ @multipart_threshold = options[:multipart_threshold] ||
19
+ ONE_HUNDRED_MEGABYTES
17
20
  end
18
21
 
19
22
  # @return [Client]
20
23
  attr_reader :client
21
24
 
22
- # @return [Integer] Files larger than this in bytes are uploaded
25
+ # @return [Integer] Files larger than or equal to this in bytes are uploaded
23
26
  # using a {MultipartFileUploader}.
24
27
  attr_reader :multipart_threshold
25
28
 
26
- # @param [String,Pathname,File,Tempfile] source
27
- # @option options [required,String] :bucket
28
- # @option options [required,String] :key
29
+ # @param [String, Pathname, File, Tempfile] source The file to upload.
30
+ # @option options [required, String] :bucket The bucket to upload to.
31
+ # @option options [required, String] :key The key for the object.
32
+ # @option options [Proc] :progress_callback
33
+ # A Proc that will be called when each chunk of the upload is sent.
34
+ # It will be invoked with [bytes_read], [total_sizes]
35
+ # @option options [Integer] :thread_count
36
+ # The thread count to use for multipart uploads. Ignored for
37
+ # objects smaller than the multipart threshold.
29
38
  # @return [void]
30
39
  def upload(source, options = {})
31
40
  if File.size(source) >= multipart_threshold
32
41
  MultipartFileUploader.new(@options).upload(source, options)
33
42
  else
43
+ # remove multipart parameters not supported by put_object
44
+ options.delete(:thread_count)
34
45
  put_object(source, options)
35
46
  end
36
47
  end
37
48
 
38
49
  private
39
50
 
40
- def put_object(source, options)
41
- open_file(source) do |file|
42
- @client.put_object(options.merge(body: file))
43
- end
44
- end
45
-
46
51
  def open_file(source)
47
52
  if String === source || Pathname === source
48
53
  File.open(source, 'rb') { |file| yield(file) }
@@ -51,6 +56,20 @@ module Aws
51
56
  end
52
57
  end
53
58
 
59
+ def put_object(source, options)
60
+ if (callback = options.delete(:progress_callback))
61
+ options[:on_chunk_sent] = single_part_progress(callback)
62
+ end
63
+ open_file(source) do |file|
64
+ @client.put_object(options.merge(body: file))
65
+ end
66
+ end
67
+
68
+ def single_part_progress(progress_callback)
69
+ proc do |_chunk, bytes_read, total_size|
70
+ progress_callback.call([bytes_read], [total_size])
71
+ end
72
+ end
54
73
  end
55
74
  end
56
75
  end
@@ -1,8 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'set'
2
4
  require 'time'
3
5
  require 'openssl'
4
6
  require 'cgi'
5
- require 'webrick/httputils'
6
7
  require 'aws-sdk-core/query'
7
8
 
8
9
  module Aws
@@ -155,33 +156,24 @@ module Aws
155
156
  end
156
157
 
157
158
  def uri_escape(s)
158
-
159
159
  #URI.escape(s)
160
160
 
161
- # URI.escape is deprecated, replacing it with escape from webrick
162
- # to squelch the massive number of warnings generated from Ruby.
163
- # The following script was used to determine the differences
164
- # between the various escape methods available. The webrick
165
- # escape only had two differences and it is available in the
166
- # standard lib.
167
- #
168
- # (0..255).each {|c|
169
- # s = [c].pack("C")
170
- # e = [
171
- # CGI.escape(s),
172
- # ERB::Util.url_encode(s),
173
- # URI.encode_www_form_component(s),
174
- # WEBrick::HTTPUtils.escape_form(s),
175
- # WEBrick::HTTPUtils.escape(s),
176
- # URI.escape(s),
177
- # ]
178
- # next if e.uniq.length == 1
179
- # puts("%5s %5s %5s %5s %5s %5s %5s" % ([s.inspect] + e))
180
- # }
181
- #
182
- WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']')
161
+ # (0..255).each {|c|
162
+ # s = [c].pack("C")
163
+ # e = [
164
+ # CGI.escape(s),
165
+ # ERB::Util.url_encode(s),
166
+ # URI.encode_www_form_component(s),
167
+ # WEBrick::HTTPUtils.escape_form(s),
168
+ # WEBrick::HTTPUtils.escape(s),
169
+ # URI.escape(s),
170
+ # URI::DEFAULT_PARSER.escape(s)
171
+ # ]
172
+ # next if e.uniq.length == 1
173
+ # puts("%5s %5s %5s %5s %5s %5s %5s %5s" % ([s.inspect] + e))
174
+ # }
175
+ URI::DEFAULT_PARSER.escape(s)
183
176
  end
184
-
185
177
  end
186
178
  end
187
179
  end
@@ -1,5 +1,6 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'pathname'
2
- require 'thread'
3
4
  require 'set'
4
5
 
5
6
  module Aws
@@ -16,14 +17,21 @@ module Aws
16
17
  THREAD_COUNT = 10
17
18
 
18
19
  # @api private
19
- CREATE_OPTIONS =
20
- Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
20
+ CREATE_OPTIONS = Set.new(
21
+ Client.api.operation(:create_multipart_upload).input.shape.member_names
22
+ )
23
+
24
+ COMPLETE_OPTIONS = Set.new(
25
+ Client.api.operation(:complete_multipart_upload).input.shape.member_names
26
+ )
21
27
 
22
28
  # @api private
23
- UPLOAD_PART_OPTIONS =
24
- Set.new(Client.api.operation(:upload_part).input.shape.member_names)
29
+ UPLOAD_PART_OPTIONS = Set.new(
30
+ Client.api.operation(:upload_part).input.shape.member_names
31
+ )
25
32
 
26
33
  # @option options [Client] :client
34
+ # @option options [Integer] :thread_count (THREAD_COUNT)
27
35
  def initialize(options = {})
28
36
  @client = options[:client] || Client.new
29
37
  @thread_count = options[:thread_count] || THREAD_COUNT
@@ -32,10 +40,13 @@ module Aws
32
40
  # @return [Client]
33
41
  attr_reader :client
34
42
 
35
- # @param [String,Pathname,File,Tempfile] source
36
- # @option options [required,String] :bucket
37
- # @option options [required,String] :key
38
- # @return [void]
43
+ # @param [String, Pathname, File, Tempfile] source The file to upload.
44
+ # @option options [required, String] :bucket The bucket to upload to.
45
+ # @option options [required, String] :key The key for the object.
46
+ # @option options [Proc] :progress_callback
47
+ # A Proc that will be called when each chunk of the upload is sent.
48
+ # It will be invoked with [bytes_read], [total_sizes]
49
+ # @return [Seahorse::Client::Response] - the CompleteMultipartUploadResponse
39
50
  def upload(source, options = {})
40
51
  if File.size(source) < MIN_PART_SIZE
41
52
  raise ArgumentError, FILE_TOO_SMALL
@@ -54,16 +65,17 @@ module Aws
54
65
 
55
66
  def complete_upload(upload_id, parts, options)
56
67
  @client.complete_multipart_upload(
57
- bucket: options[:bucket],
58
- key: options[:key],
59
- upload_id: upload_id,
60
- multipart_upload: { parts: parts })
68
+ **complete_opts(options).merge(
69
+ upload_id: upload_id,
70
+ multipart_upload: { parts: parts }
71
+ )
72
+ )
61
73
  end
62
74
 
63
75
  def upload_parts(upload_id, source, options)
64
76
  pending = PartList.new(compute_parts(upload_id, source, options))
65
77
  completed = PartList.new
66
- errors = upload_in_threads(pending, completed)
78
+ errors = upload_in_threads(pending, completed, options)
67
79
  if errors.empty?
68
80
  completed.to_a.sort_by { |part| part[:part_number] }
69
81
  else
@@ -93,7 +105,7 @@ module Aws
93
105
  part_number = 1
94
106
  parts = []
95
107
  while offset < size
96
- parts << upload_part_opts(options).merge({
108
+ parts << upload_part_opts(options).merge(
97
109
  upload_id: upload_id,
98
110
  part_number: part_number,
99
111
  body: FilePart.new(
@@ -101,7 +113,7 @@ module Aws
101
113
  offset: offset,
102
114
  size: part_size(size, default_part_size, offset)
103
115
  )
104
- })
116
+ )
105
117
  part_number += 1
106
118
  offset += default_part_size
107
119
  end
@@ -115,6 +127,13 @@ module Aws
115
127
  end
116
128
  end
117
129
 
130
+ def complete_opts(options)
131
+ COMPLETE_OPTIONS.inject({}) do |hash, key|
132
+ hash[key] = options[key] if options.key?(key)
133
+ hash
134
+ end
135
+ end
136
+
118
137
  def upload_part_opts(options)
119
138
  UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
120
139
  hash[key] = options[key] if options.key?(key)
@@ -122,15 +141,32 @@ module Aws
122
141
  end
123
142
  end
124
143
 
125
- def upload_in_threads(pending, completed)
144
+ def upload_in_threads(pending, completed, options)
126
145
  threads = []
146
+ if (callback = options[:progress_callback])
147
+ progress = MultipartProgress.new(pending, callback)
148
+ end
127
149
  @thread_count.times do
128
150
  thread = Thread.new do
129
151
  begin
130
152
  while part = pending.shift
153
+ if progress
154
+ part[:on_chunk_sent] =
155
+ proc do |_chunk, bytes, _total|
156
+ progress.call(part[:part_number], bytes)
157
+ end
158
+ end
131
159
  resp = @client.upload_part(part)
132
160
  part[:body].close
133
- completed.push(etag: resp.etag, part_number: part[:part_number])
161
+ completed_part = {etag: resp.etag, part_number: part[:part_number]}
162
+
163
+ # get the requested checksum from the response
164
+ if part[:checksum_algorithm]
165
+ k = "checksum_#{part[:checksum_algorithm].downcase}".to_sym
166
+ completed_part[k] = resp[k]
167
+ end
168
+
169
+ completed.push(completed_part)
134
170
  end
135
171
  nil
136
172
  rescue => error
@@ -177,11 +213,34 @@ module Aws
177
213
  @mutex.synchronize { @parts.clear }
178
214
  end
179
215
 
216
+ def size
217
+ @mutex.synchronize { @parts.size }
218
+ end
219
+
220
+ def part_sizes
221
+ @mutex.synchronize { @parts.map { |p| p[:body].size } }
222
+ end
223
+
180
224
  def to_a
181
225
  @mutex.synchronize { @parts.dup }
182
226
  end
183
227
 
184
228
  end
229
+
230
+ # @api private
231
+ class MultipartProgress
232
+ def initialize(parts, progress_callback)
233
+ @bytes_sent = Array.new(parts.size, 0)
234
+ @total_sizes = parts.part_sizes
235
+ @progress_callback = progress_callback
236
+ end
237
+
238
+ def call(part_number, bytes_read)
239
+ # part numbers start at 1
240
+ @bytes_sent[part_number - 1] = bytes_read
241
+ @progress_callback.call(@bytes_sent, @total_sizes)
242
+ end
243
+ end
185
244
  end
186
245
  end
187
- end
246
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'thread'
2
4
  require 'set'
3
5
  require 'tempfile'
@@ -60,12 +62,21 @@ module Aws
60
62
 
61
63
  def upload_parts(upload_id, options, &block)
62
64
  completed = Queue.new
63
- errors = IO.pipe do |read_pipe, write_pipe|
64
- threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
65
- block.call(write_pipe)
66
- write_pipe.close
67
- threads.map(&:value).compact
65
+ errors = begin
66
+ IO.pipe do |read_pipe, write_pipe|
67
+ threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
68
+ begin
69
+ block.call(write_pipe)
70
+ ensure
71
+ # Ensure the pipe is closed to avoid https://github.com/jruby/jruby/issues/6111
72
+ write_pipe.close
73
+ end
74
+ threads.map(&:value).compact
75
+ end
76
+ rescue => e
77
+ [e]
68
78
  end
79
+
69
80
  if errors.empty?
70
81
  Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] }
71
82
  else
@@ -104,7 +115,7 @@ module Aws
104
115
 
105
116
  def read_to_part_body(read_pipe)
106
117
  return if read_pipe.closed?
107
- temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new
118
+ temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new(String.new)
108
119
  temp_io.binmode
109
120
  bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
110
121
  temp_io.rewind
@@ -141,13 +152,15 @@ module Aws
141
152
  if Tempfile === body
142
153
  body.close
143
154
  body.unlink
155
+ elsif StringIO === body
156
+ body.string.clear
144
157
  end
145
158
  end
146
159
  end
147
160
  nil
148
161
  rescue => error
149
162
  # keep other threads from uploading other parts
150
- mutex.synchronize { read_pipe.close_read }
163
+ mutex.synchronize { read_pipe.close_read unless read_pipe.closed? }
151
164
  error
152
165
  end
153
166
  end