aws-sdk-s3 1.232.3 → 1.233.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-s3/client.rb +4 -4
- data/lib/aws-sdk-s3/customizations/object.rb +5 -2
- data/lib/aws-sdk-s3/default_executor.rb +65 -19
- data/lib/aws-sdk-s3/directory_downloader.rb +3 -2
- data/lib/aws-sdk-s3/directory_uploader.rb +4 -0
- data/lib/aws-sdk-s3/file_downloader.rb +21 -14
- data/lib/aws-sdk-s3/multipart_file_uploader.rb +22 -15
- data/lib/aws-sdk-s3/multipart_stream_uploader.rb +32 -17
- data/lib/aws-sdk-s3/transfer_manager.rb +5 -2
- data/lib/aws-sdk-s3.rb +1 -1
- data/sig/client.rbs +1 -1
- data/sig/types.rbs +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7dd065c16e48d0c0f7d227b772a6d6716478341cc697d4b552ba6098a6cfab06
|
|
4
|
+
data.tar.gz: b3322ba1c97612a0796f149b92c22d232f385480d7be7bd091125c1818a8f0c5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2591b76c9a1118cc31cfd6c85ddbf29767ad50ac197349f93f1a33ae31b0138f74abf455aef08ab2f8f12f75337b8bc5daba1bf858657a3f93fe2f8f87818f85
|
|
7
|
+
data.tar.gz: 232fa0a2eaf103bf1738acea96860f9f59657f13c1507fa14f4dc469fadf052de744e58ff9f62648bbb1d9779e0913919731d114ee71ba503d8b0afe014ff177
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,25 @@
|
|
|
1
1
|
Unreleased Changes
|
|
2
2
|
------------------
|
|
3
3
|
|
|
4
|
+
1.233.2 (2026-10-06)
|
|
5
|
+
------------------
|
|
6
|
+
|
|
7
|
+
* Issue - Fix `download_file` on `TransferManager` and `Aws::S3::Object` leaving a corrupt file at the destination when a part raises a non-`StandardError`, and leaving a temp file behind when a caller-provided executor rejects a part.
|
|
8
|
+
|
|
9
|
+
* Issue - Prevent `upload_directory` and `download_directory` on `TransferManager` from hanging when listing files or objects raises a non-`StandardError`, such as from a `:filter_callback`.
|
|
10
|
+
|
|
11
|
+
1.233.1 (2026-10-01)
|
|
12
|
+
------------------
|
|
13
|
+
|
|
14
|
+
* Issue - Fix unbounded memory growth in `upload_stream` on `TransferManager` and `Aws::S3::Object` when the data source outpaces the upload (#3407).
|
|
15
|
+
|
|
16
|
+
* Issue - Prevent `upload_file`, `upload_stream` and `download_file` on `TransferManager` and `Aws::S3::Object` from hanging when a worker thread is terminated by a non-`StandardError`.
|
|
17
|
+
|
|
18
|
+
1.233.0 (2026-09-30)
|
|
19
|
+
------------------
|
|
20
|
+
|
|
21
|
+
* Feature - Amazon S3 adds a new optional S3 Inventory field, IntelligentTieringReferenceDate, reporting the reference date S3 Intelligent-Tiering uses to evaluate an object's tier-transition eligibility. The value is populated for objects in the Intelligent-Tiering storage class and left blank for others.
|
|
22
|
+
|
|
4
23
|
1.232.3 (2026-09-28)
|
|
5
24
|
------------------
|
|
6
25
|
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.
|
|
1
|
+
1.233.2
|
data/lib/aws-sdk-s3/client.rb
CHANGED
|
@@ -7723,7 +7723,7 @@ module Aws::S3
|
|
|
7723
7723
|
# resp.inventory_configuration.id #=> String
|
|
7724
7724
|
# resp.inventory_configuration.included_object_versions #=> String, one of "All", "Current"
|
|
7725
7725
|
# resp.inventory_configuration.optional_fields #=> Array
|
|
7726
|
-
# resp.inventory_configuration.optional_fields[0] #=> String, one of "Size", "LastModifiedDate", "StorageClass", "ETag", "IsMultipartUploaded", "ReplicationStatus", "EncryptionStatus", "ObjectLockRetainUntilDate", "ObjectLockMode", "ObjectLockLegalHoldStatus", "ObjectLockEventHoldStatus", "ObjectLockEventHoldDuration", "IntelligentTieringAccessTier", "BucketKeyStatus", "ChecksumAlgorithm", "ObjectAccessControlList", "ObjectOwner", "LifecycleExpirationDate"
|
|
7726
|
+
# resp.inventory_configuration.optional_fields[0] #=> String, one of "Size", "LastModifiedDate", "StorageClass", "ETag", "IsMultipartUploaded", "ReplicationStatus", "EncryptionStatus", "ObjectLockRetainUntilDate", "ObjectLockMode", "ObjectLockLegalHoldStatus", "ObjectLockEventHoldStatus", "ObjectLockEventHoldDuration", "IntelligentTieringAccessTier", "BucketKeyStatus", "ChecksumAlgorithm", "ObjectAccessControlList", "ObjectOwner", "LifecycleExpirationDate", "IntelligentTieringReferenceDate"
|
|
7727
7727
|
# resp.inventory_configuration.schedule.frequency #=> String, one of "Daily", "Weekly"
|
|
7728
7728
|
#
|
|
7729
7729
|
# @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/GetBucketInventoryConfiguration AWS API Documentation
|
|
@@ -12760,7 +12760,7 @@ module Aws::S3
|
|
|
12760
12760
|
# resp.inventory_configuration_list[0].id #=> String
|
|
12761
12761
|
# resp.inventory_configuration_list[0].included_object_versions #=> String, one of "All", "Current"
|
|
12762
12762
|
# resp.inventory_configuration_list[0].optional_fields #=> Array
|
|
12763
|
-
# resp.inventory_configuration_list[0].optional_fields[0] #=> String, one of "Size", "LastModifiedDate", "StorageClass", "ETag", "IsMultipartUploaded", "ReplicationStatus", "EncryptionStatus", "ObjectLockRetainUntilDate", "ObjectLockMode", "ObjectLockLegalHoldStatus", "ObjectLockEventHoldStatus", "ObjectLockEventHoldDuration", "IntelligentTieringAccessTier", "BucketKeyStatus", "ChecksumAlgorithm", "ObjectAccessControlList", "ObjectOwner", "LifecycleExpirationDate"
|
|
12763
|
+
# resp.inventory_configuration_list[0].optional_fields[0] #=> String, one of "Size", "LastModifiedDate", "StorageClass", "ETag", "IsMultipartUploaded", "ReplicationStatus", "EncryptionStatus", "ObjectLockRetainUntilDate", "ObjectLockMode", "ObjectLockLegalHoldStatus", "ObjectLockEventHoldStatus", "ObjectLockEventHoldDuration", "IntelligentTieringAccessTier", "BucketKeyStatus", "ChecksumAlgorithm", "ObjectAccessControlList", "ObjectOwner", "LifecycleExpirationDate", "IntelligentTieringReferenceDate"
|
|
12764
12764
|
# resp.inventory_configuration_list[0].schedule.frequency #=> String, one of "Daily", "Weekly"
|
|
12765
12765
|
# resp.is_truncated #=> Boolean
|
|
12766
12766
|
# resp.next_continuation_token #=> String
|
|
@@ -16319,7 +16319,7 @@ module Aws::S3
|
|
|
16319
16319
|
# },
|
|
16320
16320
|
# id: "InventoryId", # required
|
|
16321
16321
|
# included_object_versions: "All", # required, accepts All, Current
|
|
16322
|
-
# optional_fields: ["Size"], # accepts Size, LastModifiedDate, StorageClass, ETag, IsMultipartUploaded, ReplicationStatus, EncryptionStatus, ObjectLockRetainUntilDate, ObjectLockMode, ObjectLockLegalHoldStatus, ObjectLockEventHoldStatus, ObjectLockEventHoldDuration, IntelligentTieringAccessTier, BucketKeyStatus, ChecksumAlgorithm, ObjectAccessControlList, ObjectOwner, LifecycleExpirationDate
|
|
16322
|
+
# optional_fields: ["Size"], # accepts Size, LastModifiedDate, StorageClass, ETag, IsMultipartUploaded, ReplicationStatus, EncryptionStatus, ObjectLockRetainUntilDate, ObjectLockMode, ObjectLockLegalHoldStatus, ObjectLockEventHoldStatus, ObjectLockEventHoldDuration, IntelligentTieringAccessTier, BucketKeyStatus, ChecksumAlgorithm, ObjectAccessControlList, ObjectOwner, LifecycleExpirationDate, IntelligentTieringReferenceDate
|
|
16323
16323
|
# schedule: { # required
|
|
16324
16324
|
# frequency: "Daily", # required, accepts Daily, Weekly
|
|
16325
16325
|
# },
|
|
@@ -24209,7 +24209,7 @@ module Aws::S3
|
|
|
24209
24209
|
tracer: tracer
|
|
24210
24210
|
)
|
|
24211
24211
|
context[:gem_name] = 'aws-sdk-s3'
|
|
24212
|
-
context[:gem_version] = '1.
|
|
24212
|
+
context[:gem_version] = '1.233.2'
|
|
24213
24213
|
Seahorse::Client::Request.new(handlers, context)
|
|
24214
24214
|
end
|
|
24215
24215
|
|
|
@@ -380,7 +380,9 @@ module Aws
|
|
|
380
380
|
# and {Client#upload_part} can be provided.
|
|
381
381
|
#
|
|
382
382
|
# @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
|
|
383
|
-
# An additional thread is used internally for task coordination.
|
|
383
|
+
# An additional thread is used internally for task coordination. This also bounds
|
|
384
|
+
# how many parts are buffered ahead of the upload, limiting memory usage to roughly
|
|
385
|
+
# `2 * :thread_count * :part_size`.
|
|
384
386
|
#
|
|
385
387
|
# @option options [Boolean] :tempfile (false) Normally read data is stored
|
|
386
388
|
# in memory when building the parts in order to complete the underlying
|
|
@@ -405,7 +407,8 @@ module Aws
|
|
|
405
407
|
# @see Client#upload_part
|
|
406
408
|
def upload_stream(options = {}, &block)
|
|
407
409
|
upload_opts = options.merge(bucket: bucket_name, key: key)
|
|
408
|
-
|
|
410
|
+
thread_count = upload_opts.delete(:thread_count) || DefaultExecutor::DEFAULT_MAX_THREADS
|
|
411
|
+
executor = DefaultExecutor.new(max_threads: thread_count, max_queue: thread_count)
|
|
409
412
|
begin
|
|
410
413
|
uploader = MultipartStreamUploader.new(
|
|
411
414
|
client: client,
|
|
@@ -4,6 +4,13 @@ module Aws
|
|
|
4
4
|
module S3
|
|
5
5
|
# @api private
|
|
6
6
|
class DefaultExecutor
|
|
7
|
+
# Raised when a task is posted to an executor that is shutting down or has been shut down.
|
|
8
|
+
class RejectedExecutionError < RuntimeError
|
|
9
|
+
def initialize(msg = 'Executor has been shutdown and is no longer accepting tasks')
|
|
10
|
+
super
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
7
14
|
DEFAULT_MAX_THREADS = 10
|
|
8
15
|
RUNNING = :running
|
|
9
16
|
SHUTTING_DOWN = :shutting_down
|
|
@@ -11,10 +18,12 @@ module Aws
|
|
|
11
18
|
|
|
12
19
|
def initialize(options = {})
|
|
13
20
|
@max_threads = options[:max_threads] || DEFAULT_MAX_THREADS
|
|
21
|
+
@max_queue = options[:max_queue] || 0
|
|
14
22
|
@state = RUNNING
|
|
15
|
-
@queue = Queue.new
|
|
23
|
+
@queue = @max_queue.zero? ? Queue.new : SizedQueue.new(@max_queue)
|
|
16
24
|
@pool = []
|
|
17
25
|
@mutex = Mutex.new
|
|
26
|
+
@fatal_error = nil
|
|
18
27
|
end
|
|
19
28
|
|
|
20
29
|
# Submits a task for execution.
|
|
@@ -23,12 +32,15 @@ module Aws
|
|
|
23
32
|
# @return [Boolean] Returns true if the task was submitted successfully
|
|
24
33
|
def post(*args, &block)
|
|
25
34
|
@mutex.synchronize do
|
|
26
|
-
raise
|
|
35
|
+
raise RejectedExecutionError unless @state == RUNNING
|
|
27
36
|
|
|
28
|
-
@queue << [args, block]
|
|
29
37
|
ensure_worker_available
|
|
30
38
|
end
|
|
39
|
+
# Outside the mutex so a caller blocked on a full queue can't hold up #shutdown or #kill.
|
|
40
|
+
@queue.push([args, block])
|
|
31
41
|
true
|
|
42
|
+
rescue ClosedQueueError
|
|
43
|
+
raise RejectedExecutionError
|
|
32
44
|
end
|
|
33
45
|
|
|
34
46
|
# Immediately terminates all worker threads and clears pending tasks.
|
|
@@ -38,6 +50,7 @@ module Aws
|
|
|
38
50
|
def kill
|
|
39
51
|
@mutex.synchronize do
|
|
40
52
|
@state = SHUTDOWN
|
|
53
|
+
@queue.close
|
|
41
54
|
@pool.each(&:kill)
|
|
42
55
|
@pool.clear
|
|
43
56
|
@queue.clear
|
|
@@ -52,35 +65,58 @@ module Aws
|
|
|
52
65
|
# If nil, waits indefinitely. If timeout expires, remaining threads are killed.
|
|
53
66
|
# @return [Boolean] true when shutdown is complete
|
|
54
67
|
def shutdown(timeout = nil)
|
|
68
|
+
return true unless begin_shutdown
|
|
69
|
+
|
|
70
|
+
deadline = timeout && (Time.now + timeout)
|
|
71
|
+
join_workers(deadline)
|
|
72
|
+
kill_remaining_workers if timeout
|
|
73
|
+
|
|
74
|
+
finalize_shutdown
|
|
75
|
+
raise @fatal_error if @fatal_error
|
|
76
|
+
|
|
77
|
+
true
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def begin_shutdown
|
|
55
83
|
@mutex.synchronize do
|
|
56
|
-
return
|
|
84
|
+
return false if @state == SHUTDOWN
|
|
57
85
|
|
|
58
86
|
@state = SHUTTING_DOWN
|
|
59
|
-
|
|
87
|
+
# Close rather than push sentinels, which could block on a full queue.
|
|
88
|
+
@queue.close
|
|
60
89
|
end
|
|
90
|
+
true
|
|
91
|
+
end
|
|
61
92
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
remaining = deadline - Time.now
|
|
66
|
-
break if remaining <= 0
|
|
93
|
+
def join_workers(deadline)
|
|
94
|
+
until (threads = @mutex.synchronize { @pool.select(&:alive?) }).empty?
|
|
95
|
+
threads.each do |thread|
|
|
96
|
+
remaining = deadline - Time.now if deadline
|
|
97
|
+
break if remaining && remaining <= 0
|
|
67
98
|
|
|
68
|
-
|
|
99
|
+
begin
|
|
100
|
+
thread.join(remaining && [remaining, 0].max)
|
|
101
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
102
|
+
nil # recorded by #replace_worker and raised by #shutdown
|
|
103
|
+
end
|
|
69
104
|
end
|
|
70
|
-
|
|
71
|
-
else
|
|
72
|
-
@pool.each(&:join)
|
|
105
|
+
break if deadline && Time.now >= deadline
|
|
73
106
|
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def kill_remaining_workers
|
|
110
|
+
@mutex.synchronize { @pool.select(&:alive?).each(&:kill) }
|
|
111
|
+
end
|
|
74
112
|
|
|
113
|
+
def finalize_shutdown
|
|
75
114
|
@mutex.synchronize do
|
|
76
115
|
@pool.clear
|
|
77
116
|
@state = SHUTDOWN
|
|
78
117
|
end
|
|
79
|
-
true
|
|
80
118
|
end
|
|
81
119
|
|
|
82
|
-
private
|
|
83
|
-
|
|
84
120
|
def ensure_worker_available
|
|
85
121
|
return unless @state == RUNNING
|
|
86
122
|
|
|
@@ -91,11 +127,21 @@ module Aws
|
|
|
91
127
|
def spawn_worker
|
|
92
128
|
Thread.new do
|
|
93
129
|
while (job = @queue.shift)
|
|
94
|
-
break if job == :shutdown
|
|
95
|
-
|
|
96
130
|
args, block = job
|
|
97
131
|
block.call(*args)
|
|
98
132
|
end
|
|
133
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
134
|
+
# Replace this worker so queued tasks still run.
|
|
135
|
+
replace_worker(e)
|
|
136
|
+
raise
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def replace_worker(error)
|
|
141
|
+
@mutex.synchronize do
|
|
142
|
+
@fatal_error ||= error
|
|
143
|
+
@pool.delete(Thread.current)
|
|
144
|
+
@pool << spawn_worker unless @state == SHUTDOWN
|
|
99
145
|
end
|
|
100
146
|
end
|
|
101
147
|
end
|
|
@@ -148,8 +148,9 @@ module Aws
|
|
|
148
148
|
stream_objects
|
|
149
149
|
@object_queue << DONE_MARKER
|
|
150
150
|
rescue ClosedQueueError
|
|
151
|
-
# abort requested
|
|
152
|
-
|
|
151
|
+
nil # abort requested
|
|
152
|
+
# Any error, or the consumer waits on the queue forever
|
|
153
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
153
154
|
close
|
|
154
155
|
raise e
|
|
155
156
|
end
|
|
@@ -156,6 +156,10 @@ module Aws
|
|
|
156
156
|
# encountered a traversal error, we must abort immediately
|
|
157
157
|
close
|
|
158
158
|
raise DirectoryUploadError, "Directory traversal failed for '#{@source_dir}': #{e.message}"
|
|
159
|
+
# Any other error, or the consumer waits on the queue forever
|
|
160
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
161
|
+
close
|
|
162
|
+
raise e
|
|
159
163
|
end
|
|
160
164
|
|
|
161
165
|
while (file = @file_queue.shift) && file != DONE_MARKER
|
|
@@ -60,7 +60,7 @@ module Aws
|
|
|
60
60
|
end
|
|
61
61
|
|
|
62
62
|
def download_with_executor(part_list, total_size, opts)
|
|
63
|
-
|
|
63
|
+
queued_parts = 0
|
|
64
64
|
completion_queue = Queue.new
|
|
65
65
|
abort_download = false
|
|
66
66
|
error = nil
|
|
@@ -69,24 +69,31 @@ module Aws
|
|
|
69
69
|
while (part = part_list.shift)
|
|
70
70
|
break if abort_download
|
|
71
71
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
72
|
+
begin
|
|
73
|
+
@executor.post(part) do |p|
|
|
74
|
+
update_progress(progress, p)
|
|
75
|
+
resp = @client.get_object(p.params)
|
|
76
|
+
range = extract_range(resp.content_range)
|
|
77
|
+
validate_range(range, p.params[:range]) if p.params[:range]
|
|
78
|
+
write(resp.body, range, opts)
|
|
79
|
+
|
|
80
|
+
execute_checksum_callback(resp, opts)
|
|
81
|
+
# Any error, or a corrupt file replaces the destination
|
|
82
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
83
|
+
abort_download = true
|
|
84
|
+
error = e
|
|
85
|
+
ensure
|
|
86
|
+
completion_queue << :done
|
|
87
|
+
end
|
|
88
|
+
queued_parts += 1
|
|
81
89
|
rescue StandardError => e
|
|
82
|
-
|
|
90
|
+
# Rejected by the executor, wait for queued parts so none write after cleanup
|
|
83
91
|
error = e
|
|
84
|
-
|
|
85
|
-
completion_queue << :done
|
|
92
|
+
break
|
|
86
93
|
end
|
|
87
94
|
end
|
|
88
95
|
|
|
89
|
-
|
|
96
|
+
queued_parts.times { completion_queue.pop }
|
|
90
97
|
raise error unless error.nil?
|
|
91
98
|
end
|
|
92
99
|
|
|
@@ -140,7 +140,7 @@ module Aws
|
|
|
140
140
|
end
|
|
141
141
|
|
|
142
142
|
def upload_with_executor(pending, completed, options)
|
|
143
|
-
|
|
143
|
+
queued_parts = 0
|
|
144
144
|
completion_queue = Queue.new
|
|
145
145
|
abort_upload = false
|
|
146
146
|
errors = []
|
|
@@ -149,25 +149,32 @@ module Aws
|
|
|
149
149
|
while (part = pending.shift)
|
|
150
150
|
break if abort_upload
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
152
|
+
begin
|
|
153
|
+
@executor.post(part) do |p|
|
|
154
|
+
Thread.current[:net_http_override_body_stream_chunk] = @http_chunk_size if @http_chunk_size
|
|
155
|
+
update_progress(progress, p)
|
|
156
|
+
resp = @client.upload_part(p)
|
|
157
|
+
completed_part = { etag: resp.etag, part_number: p[:part_number] }
|
|
158
|
+
apply_part_checksum(resp, completed_part)
|
|
159
|
+
completed.push(completed_part)
|
|
160
|
+
# Any error, or the upload completes without this part.
|
|
161
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
162
|
+
abort_upload = true
|
|
163
|
+
errors << e
|
|
164
|
+
ensure
|
|
165
|
+
p[:body].close
|
|
166
|
+
Thread.current[:net_http_override_body_stream_chunk] = nil if @http_chunk_size
|
|
167
|
+
completion_queue << :done
|
|
168
|
+
end
|
|
169
|
+
queued_parts += 1
|
|
160
170
|
rescue StandardError => e
|
|
161
|
-
|
|
171
|
+
# Rejected by the executor; abort rather than orphan the upload.
|
|
162
172
|
errors << e
|
|
163
|
-
|
|
164
|
-
p[:body].close
|
|
165
|
-
Thread.current[:net_http_override_body_stream_chunk] = nil if @http_chunk_size
|
|
166
|
-
completion_queue << :done
|
|
173
|
+
break
|
|
167
174
|
end
|
|
168
175
|
end
|
|
169
176
|
|
|
170
|
-
|
|
177
|
+
queued_parts.times { completion_queue.pop }
|
|
171
178
|
errors
|
|
172
179
|
end
|
|
173
180
|
|
|
@@ -113,18 +113,22 @@ module Aws
|
|
|
113
113
|
def read_to_part_body(read_pipe)
|
|
114
114
|
return if read_pipe.closed?
|
|
115
115
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
if
|
|
116
|
+
if @tempfile
|
|
117
|
+
temp_io = Tempfile.new('aws-sdk-s3-upload_stream')
|
|
118
|
+
temp_io.binmode
|
|
119
|
+
bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
|
|
120
|
+
temp_io.rewind
|
|
121
|
+
if bytes_copied.zero?
|
|
122
122
|
temp_io.close
|
|
123
123
|
temp_io.unlink
|
|
124
|
+
nil
|
|
125
|
+
else
|
|
126
|
+
temp_io
|
|
124
127
|
end
|
|
125
|
-
nil
|
|
126
128
|
else
|
|
127
|
-
|
|
129
|
+
# A single sized read; copy_stream into a StringIO grows by doubling and fragments the heap.
|
|
130
|
+
data = read_pipe.read(@part_size)
|
|
131
|
+
data.nil? ? nil : StringIO.new(data)
|
|
128
132
|
end
|
|
129
133
|
end
|
|
130
134
|
|
|
@@ -139,20 +143,31 @@ module Aws
|
|
|
139
143
|
end
|
|
140
144
|
break unless part_body || current_part_num == 1
|
|
141
145
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
begin
|
|
147
|
+
@executor.post(part_body, current_part_num, options) do |body, num, opts|
|
|
148
|
+
part = opts.merge(body: body, part_number: num)
|
|
149
|
+
resp = @client.upload_part(part)
|
|
150
|
+
completed_part = create_completed_part(resp, part)
|
|
151
|
+
completed.push(completed_part)
|
|
152
|
+
# Any error, or the upload completes without this part.
|
|
153
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
154
|
+
mutex.synchronize do
|
|
155
|
+
errors.push(e)
|
|
156
|
+
read_pipe.close_read unless read_pipe.closed?
|
|
157
|
+
end
|
|
158
|
+
ensure
|
|
159
|
+
clear_body(body)
|
|
160
|
+
completion_queue << :done
|
|
161
|
+
end
|
|
162
|
+
queued_parts += 1
|
|
148
163
|
rescue StandardError => e
|
|
164
|
+
# Rejected by the executor. Closing the pipe stops the producer so the upload can abort.
|
|
149
165
|
mutex.synchronize do
|
|
150
166
|
errors.push(e)
|
|
151
167
|
read_pipe.close_read unless read_pipe.closed?
|
|
152
168
|
end
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
completion_queue << :done
|
|
169
|
+
clear_body(part_body)
|
|
170
|
+
break
|
|
156
171
|
end
|
|
157
172
|
end
|
|
158
173
|
queued_parts.times { completion_queue.pop }
|
|
@@ -504,6 +504,9 @@ module Aws
|
|
|
504
504
|
# @option options [Integer] :thread_count (10)
|
|
505
505
|
# The number of parallel multipart uploads. Only used when no custom executor is provided (creates
|
|
506
506
|
# {DefaultExecutor} with the given thread count). An additional thread is used internally for task coordination.
|
|
507
|
+
# This also bounds how many parts are buffered ahead of the upload, limiting memory usage to roughly
|
|
508
|
+
# `2 * :thread_count * :part_size`. When a custom `:executor` is provided, it is responsible for applying
|
|
509
|
+
# its own backpressure.
|
|
507
510
|
#
|
|
508
511
|
# @option options [Boolean] :tempfile (false)
|
|
509
512
|
# Normally read data is stored in memory when building the parts in order to complete the underlying
|
|
@@ -524,8 +527,8 @@ module Aws
|
|
|
524
527
|
# @see Client#upload_part
|
|
525
528
|
def upload_stream(bucket:, key:, **options, &block)
|
|
526
529
|
upload_opts = options.merge(bucket: bucket, key: key)
|
|
527
|
-
thread_count = upload_opts.delete(:thread_count)
|
|
528
|
-
executor = @executor || DefaultExecutor.new(max_threads: thread_count)
|
|
530
|
+
thread_count = upload_opts.delete(:thread_count) || DefaultExecutor::DEFAULT_MAX_THREADS
|
|
531
|
+
executor = @executor || DefaultExecutor.new(max_threads: thread_count, max_queue: thread_count)
|
|
529
532
|
begin
|
|
530
533
|
uploader = MultipartStreamUploader.new(
|
|
531
534
|
client: @client,
|
data/lib/aws-sdk-s3.rb
CHANGED
data/sig/client.rbs
CHANGED
|
@@ -1671,7 +1671,7 @@ module Aws
|
|
|
1671
1671
|
}?,
|
|
1672
1672
|
id: ::String,
|
|
1673
1673
|
included_object_versions: ("All" | "Current"),
|
|
1674
|
-
optional_fields: Array[("Size" | "LastModifiedDate" | "StorageClass" | "ETag" | "IsMultipartUploaded" | "ReplicationStatus" | "EncryptionStatus" | "ObjectLockRetainUntilDate" | "ObjectLockMode" | "ObjectLockLegalHoldStatus" | "ObjectLockEventHoldStatus" | "ObjectLockEventHoldDuration" | "IntelligentTieringAccessTier" | "BucketKeyStatus" | "ChecksumAlgorithm" | "ObjectAccessControlList" | "ObjectOwner" | "LifecycleExpirationDate")]?,
|
|
1674
|
+
optional_fields: Array[("Size" | "LastModifiedDate" | "StorageClass" | "ETag" | "IsMultipartUploaded" | "ReplicationStatus" | "EncryptionStatus" | "ObjectLockRetainUntilDate" | "ObjectLockMode" | "ObjectLockLegalHoldStatus" | "ObjectLockEventHoldStatus" | "ObjectLockEventHoldDuration" | "IntelligentTieringAccessTier" | "BucketKeyStatus" | "ChecksumAlgorithm" | "ObjectAccessControlList" | "ObjectOwner" | "LifecycleExpirationDate" | "IntelligentTieringReferenceDate")]?,
|
|
1675
1675
|
schedule: {
|
|
1676
1676
|
frequency: ("Daily" | "Weekly")
|
|
1677
1677
|
}
|
data/sig/types.rbs
CHANGED
|
@@ -1515,7 +1515,7 @@ module Aws::S3
|
|
|
1515
1515
|
attr_accessor filter: Types::InventoryFilter
|
|
1516
1516
|
attr_accessor id: ::String
|
|
1517
1517
|
attr_accessor included_object_versions: ("All" | "Current")
|
|
1518
|
-
attr_accessor optional_fields: ::Array[("Size" | "LastModifiedDate" | "StorageClass" | "ETag" | "IsMultipartUploaded" | "ReplicationStatus" | "EncryptionStatus" | "ObjectLockRetainUntilDate" | "ObjectLockMode" | "ObjectLockLegalHoldStatus" | "ObjectLockEventHoldStatus" | "ObjectLockEventHoldDuration" | "IntelligentTieringAccessTier" | "BucketKeyStatus" | "ChecksumAlgorithm" | "ObjectAccessControlList" | "ObjectOwner" | "LifecycleExpirationDate")]
|
|
1518
|
+
attr_accessor optional_fields: ::Array[("Size" | "LastModifiedDate" | "StorageClass" | "ETag" | "IsMultipartUploaded" | "ReplicationStatus" | "EncryptionStatus" | "ObjectLockRetainUntilDate" | "ObjectLockMode" | "ObjectLockLegalHoldStatus" | "ObjectLockEventHoldStatus" | "ObjectLockEventHoldDuration" | "IntelligentTieringAccessTier" | "BucketKeyStatus" | "ChecksumAlgorithm" | "ObjectAccessControlList" | "ObjectOwner" | "LifecycleExpirationDate" | "IntelligentTieringReferenceDate")]
|
|
1519
1519
|
attr_accessor schedule: Types::InventorySchedule
|
|
1520
1520
|
SENSITIVE: []
|
|
1521
1521
|
end
|