aws-sdk-s3 1.10.0 → 1.208.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +1517 -0
  3. data/LICENSE.txt +202 -0
  4. data/VERSION +1 -0
  5. data/lib/aws-sdk-s3/access_grants_credentials.rb +57 -0
  6. data/lib/aws-sdk-s3/access_grants_credentials_provider.rb +250 -0
  7. data/lib/aws-sdk-s3/bucket.rb +1062 -99
  8. data/lib/aws-sdk-s3/bucket_acl.rb +67 -17
  9. data/lib/aws-sdk-s3/bucket_cors.rb +80 -17
  10. data/lib/aws-sdk-s3/bucket_lifecycle.rb +71 -19
  11. data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +126 -20
  12. data/lib/aws-sdk-s3/bucket_logging.rb +68 -18
  13. data/lib/aws-sdk-s3/bucket_notification.rb +56 -20
  14. data/lib/aws-sdk-s3/bucket_policy.rb +108 -17
  15. data/lib/aws-sdk-s3/bucket_region_cache.rb +11 -5
  16. data/lib/aws-sdk-s3/bucket_request_payment.rb +60 -15
  17. data/lib/aws-sdk-s3/bucket_tagging.rb +71 -17
  18. data/lib/aws-sdk-s3/bucket_versioning.rb +166 -17
  19. data/lib/aws-sdk-s3/bucket_website.rb +78 -17
  20. data/lib/aws-sdk-s3/client.rb +20068 -3879
  21. data/lib/aws-sdk-s3/client_api.rb +1957 -209
  22. data/lib/aws-sdk-s3/customizations/bucket.rb +57 -38
  23. data/lib/aws-sdk-s3/customizations/errors.rb +40 -0
  24. data/lib/aws-sdk-s3/customizations/multipart_upload.rb +2 -0
  25. data/lib/aws-sdk-s3/customizations/object.rb +338 -68
  26. data/lib/aws-sdk-s3/customizations/object_summary.rb +17 -0
  27. data/lib/aws-sdk-s3/customizations/object_version.rb +13 -0
  28. data/lib/aws-sdk-s3/customizations/types/list_object_versions_output.rb +2 -0
  29. data/lib/aws-sdk-s3/customizations/types/permanent_redirect.rb +26 -0
  30. data/lib/aws-sdk-s3/customizations.rb +30 -27
  31. data/lib/aws-sdk-s3/default_executor.rb +103 -0
  32. data/lib/aws-sdk-s3/encryption/client.rb +29 -8
  33. data/lib/aws-sdk-s3/encryption/decrypt_handler.rb +71 -29
  34. data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb +45 -5
  35. data/lib/aws-sdk-s3/encryption/default_key_provider.rb +2 -0
  36. data/lib/aws-sdk-s3/encryption/encrypt_handler.rb +15 -2
  37. data/lib/aws-sdk-s3/encryption/errors.rb +2 -0
  38. data/lib/aws-sdk-s3/encryption/io_auth_decrypter.rb +11 -3
  39. data/lib/aws-sdk-s3/encryption/io_decrypter.rb +11 -3
  40. data/lib/aws-sdk-s3/encryption/io_encrypter.rb +2 -0
  41. data/lib/aws-sdk-s3/encryption/key_provider.rb +2 -0
  42. data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +48 -11
  43. data/lib/aws-sdk-s3/encryption/materials.rb +8 -6
  44. data/lib/aws-sdk-s3/encryption/utils.rb +25 -0
  45. data/lib/aws-sdk-s3/encryption.rb +4 -0
  46. data/lib/aws-sdk-s3/encryptionV2/client.rb +645 -0
  47. data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +68 -0
  48. data/lib/aws-sdk-s3/encryptionV2/decryption.rb +205 -0
  49. data/lib/aws-sdk-s3/encryptionV2/default_cipher_provider.rb +187 -0
  50. data/lib/aws-sdk-s3/encryptionV2/default_key_provider.rb +40 -0
  51. data/lib/aws-sdk-s3/encryptionV2/encrypt_handler.rb +67 -0
  52. data/lib/aws-sdk-s3/encryptionV2/errors.rb +37 -0
  53. data/lib/aws-sdk-s3/encryptionV2/io_auth_decrypter.rb +58 -0
  54. data/lib/aws-sdk-s3/encryptionV2/io_decrypter.rb +37 -0
  55. data/lib/aws-sdk-s3/encryptionV2/io_encrypter.rb +75 -0
  56. data/lib/aws-sdk-s3/encryptionV2/key_provider.rb +31 -0
  57. data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +181 -0
  58. data/lib/aws-sdk-s3/encryptionV2/materials.rb +60 -0
  59. data/lib/aws-sdk-s3/encryptionV2/utils.rb +108 -0
  60. data/lib/aws-sdk-s3/encryptionV3/client.rb +885 -0
  61. data/lib/aws-sdk-s3/encryptionV3/decrypt_handler.rb +98 -0
  62. data/lib/aws-sdk-s3/encryptionV3/decryption.rb +244 -0
  63. data/lib/aws-sdk-s3/encryptionV3/default_cipher_provider.rb +159 -0
  64. data/lib/aws-sdk-s3/encryptionV3/default_key_provider.rb +35 -0
  65. data/lib/aws-sdk-s3/encryptionV3/encrypt_handler.rb +98 -0
  66. data/lib/aws-sdk-s3/encryptionV3/errors.rb +47 -0
  67. data/lib/aws-sdk-s3/encryptionV3/io_auth_decrypter.rb +60 -0
  68. data/lib/aws-sdk-s3/encryptionV3/io_decrypter.rb +35 -0
  69. data/lib/aws-sdk-s3/encryptionV3/io_encrypter.rb +84 -0
  70. data/lib/aws-sdk-s3/encryptionV3/key_provider.rb +28 -0
  71. data/lib/aws-sdk-s3/encryptionV3/kms_cipher_provider.rb +159 -0
  72. data/lib/aws-sdk-s3/encryptionV3/materials.rb +58 -0
  73. data/lib/aws-sdk-s3/encryptionV3/utils.rb +321 -0
  74. data/lib/aws-sdk-s3/encryption_v2.rb +24 -0
  75. data/lib/aws-sdk-s3/encryption_v3.rb +24 -0
  76. data/lib/aws-sdk-s3/endpoint_parameters.rb +181 -0
  77. data/lib/aws-sdk-s3/endpoint_provider.rb +886 -0
  78. data/lib/aws-sdk-s3/endpoints.rb +1544 -0
  79. data/lib/aws-sdk-s3/errors.rb +181 -1
  80. data/lib/aws-sdk-s3/event_streams.rb +69 -0
  81. data/lib/aws-sdk-s3/express_credentials.rb +55 -0
  82. data/lib/aws-sdk-s3/express_credentials_provider.rb +59 -0
  83. data/lib/aws-sdk-s3/file_downloader.rb +261 -82
  84. data/lib/aws-sdk-s3/file_part.rb +16 -13
  85. data/lib/aws-sdk-s3/file_uploader.rb +37 -22
  86. data/lib/aws-sdk-s3/legacy_signer.rb +19 -26
  87. data/lib/aws-sdk-s3/multipart_download_error.rb +8 -0
  88. data/lib/aws-sdk-s3/multipart_file_uploader.rb +142 -80
  89. data/lib/aws-sdk-s3/multipart_stream_uploader.rb +191 -0
  90. data/lib/aws-sdk-s3/multipart_upload.rb +342 -31
  91. data/lib/aws-sdk-s3/multipart_upload_error.rb +5 -4
  92. data/lib/aws-sdk-s3/multipart_upload_part.rb +387 -47
  93. data/lib/aws-sdk-s3/object.rb +2733 -204
  94. data/lib/aws-sdk-s3/object_acl.rb +112 -25
  95. data/lib/aws-sdk-s3/object_copier.rb +9 -5
  96. data/lib/aws-sdk-s3/object_multipart_copier.rb +50 -23
  97. data/lib/aws-sdk-s3/object_summary.rb +2265 -181
  98. data/lib/aws-sdk-s3/object_version.rb +542 -74
  99. data/lib/aws-sdk-s3/plugins/accelerate.rb +17 -64
  100. data/lib/aws-sdk-s3/plugins/access_grants.rb +178 -0
  101. data/lib/aws-sdk-s3/plugins/arn.rb +70 -0
  102. data/lib/aws-sdk-s3/plugins/bucket_dns.rb +7 -43
  103. data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb +20 -3
  104. data/lib/aws-sdk-s3/plugins/checksum_algorithm.rb +31 -0
  105. data/lib/aws-sdk-s3/plugins/dualstack.rb +7 -50
  106. data/lib/aws-sdk-s3/plugins/endpoints.rb +86 -0
  107. data/lib/aws-sdk-s3/plugins/expect_100_continue.rb +5 -4
  108. data/lib/aws-sdk-s3/plugins/express_session_auth.rb +88 -0
  109. data/lib/aws-sdk-s3/plugins/get_bucket_location_fix.rb +3 -1
  110. data/lib/aws-sdk-s3/plugins/http_200_errors.rb +62 -17
  111. data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb +44 -0
  112. data/lib/aws-sdk-s3/plugins/location_constraint.rb +5 -1
  113. data/lib/aws-sdk-s3/plugins/md5s.rb +14 -67
  114. data/lib/aws-sdk-s3/plugins/redirects.rb +5 -1
  115. data/lib/aws-sdk-s3/plugins/s3_host_id.rb +2 -0
  116. data/lib/aws-sdk-s3/plugins/s3_signer.rb +67 -93
  117. data/lib/aws-sdk-s3/plugins/sse_cpk.rb +3 -1
  118. data/lib/aws-sdk-s3/plugins/streaming_retry.rb +137 -0
  119. data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb +4 -1
  120. data/lib/aws-sdk-s3/presigned_post.rb +160 -99
  121. data/lib/aws-sdk-s3/presigner.rb +178 -81
  122. data/lib/aws-sdk-s3/resource.rb +164 -15
  123. data/lib/aws-sdk-s3/transfer_manager.rb +303 -0
  124. data/lib/aws-sdk-s3/types.rb +15981 -4168
  125. data/lib/aws-sdk-s3/waiters.rb +67 -1
  126. data/lib/aws-sdk-s3.rb +46 -31
  127. data/sig/bucket.rbs +231 -0
  128. data/sig/bucket_acl.rbs +78 -0
  129. data/sig/bucket_cors.rbs +69 -0
  130. data/sig/bucket_lifecycle.rbs +88 -0
  131. data/sig/bucket_lifecycle_configuration.rbs +115 -0
  132. data/sig/bucket_logging.rbs +76 -0
  133. data/sig/bucket_notification.rbs +114 -0
  134. data/sig/bucket_policy.rbs +59 -0
  135. data/sig/bucket_request_payment.rbs +54 -0
  136. data/sig/bucket_tagging.rbs +65 -0
  137. data/sig/bucket_versioning.rbs +77 -0
  138. data/sig/bucket_website.rbs +93 -0
  139. data/sig/client.rbs +2612 -0
  140. data/sig/customizations/bucket.rbs +19 -0
  141. data/sig/customizations/object.rbs +38 -0
  142. data/sig/customizations/object_summary.rbs +35 -0
  143. data/sig/errors.rbs +44 -0
  144. data/sig/multipart_upload.rbs +120 -0
  145. data/sig/multipart_upload_part.rbs +109 -0
  146. data/sig/object.rbs +464 -0
  147. data/sig/object_acl.rbs +86 -0
  148. data/sig/object_summary.rbs +347 -0
  149. data/sig/object_version.rbs +143 -0
  150. data/sig/resource.rbs +141 -0
  151. data/sig/types.rbs +2899 -0
  152. data/sig/waiters.rbs +95 -0
  153. metadata +97 -14
@@ -1,30 +1,33 @@
1
- # utility classes
2
- require 'aws-sdk-s3/bucket_region_cache'
3
- require 'aws-sdk-s3/encryption'
4
- require 'aws-sdk-s3/file_part'
5
- require 'aws-sdk-s3/file_uploader'
6
- require 'aws-sdk-s3/file_downloader'
7
- require 'aws-sdk-s3/legacy_signer'
8
- require 'aws-sdk-s3/multipart_file_uploader'
9
- require 'aws-sdk-s3/multipart_upload_error'
10
- require 'aws-sdk-s3/object_copier'
11
- require 'aws-sdk-s3/object_multipart_copier'
12
- require 'aws-sdk-s3/presigned_post'
13
- require 'aws-sdk-s3/presigner'
1
+ # frozen_string_literal: true
14
2
 
15
- # customizations to generated classes
16
- require 'aws-sdk-s3/customizations/bucket'
17
- require 'aws-sdk-s3/customizations/object'
18
- require 'aws-sdk-s3/customizations/object_summary'
19
- require 'aws-sdk-s3/customizations/multipart_upload'
20
- require 'aws-sdk-s3/customizations/types/list_object_versions_output'
3
+ module Aws
4
+ module S3
5
+ # utility classes
6
+ autoload :BucketRegionCache, 'aws-sdk-s3/bucket_region_cache'
7
+ autoload :Encryption, 'aws-sdk-s3/encryption'
8
+ autoload :EncryptionV2, 'aws-sdk-s3/encryption_v2'
9
+ autoload :EncryptionV3, 'aws-sdk-s3/encryption_v3'
10
+ autoload :FilePart, 'aws-sdk-s3/file_part'
11
+ autoload :DefaultExecutor, 'aws-sdk-s3/default_executor'
12
+ autoload :FileUploader, 'aws-sdk-s3/file_uploader'
13
+ autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
14
+ autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
15
+ autoload :MultipartDownloadError, 'aws-sdk-s3/multipart_download_error'
16
+ autoload :MultipartFileUploader, 'aws-sdk-s3/multipart_file_uploader'
17
+ autoload :MultipartStreamUploader, 'aws-sdk-s3/multipart_stream_uploader'
18
+ autoload :MultipartUploadError, 'aws-sdk-s3/multipart_upload_error'
19
+ autoload :ObjectCopier, 'aws-sdk-s3/object_copier'
20
+ autoload :ObjectMultipartCopier, 'aws-sdk-s3/object_multipart_copier'
21
+ autoload :PresignedPost, 'aws-sdk-s3/presigned_post'
22
+ autoload :Presigner, 'aws-sdk-s3/presigner'
23
+ autoload :TransferManager, 'aws-sdk-s3/transfer_manager'
21
24
 
22
- [
23
- Aws::S3::Object::Collection,
24
- Aws::S3::ObjectSummary::Collection,
25
- Aws::S3::ObjectVersion::Collection,
26
- ].each do |klass|
27
- klass.send(:alias_method, :delete, :batch_delete!)
28
- klass.send(:extend, Aws::Deprecations)
29
- klass.send(:deprecated, :delete, use: :batch_delete!)
25
+ # s3 express session auth
26
+ autoload :ExpressCredentials, 'aws-sdk-s3/express_credentials'
27
+ autoload :ExpressCredentialsProvider, 'aws-sdk-s3/express_credentials_provider'
28
+
29
+ # s3 access grants auth
30
+ autoload :AccessGrantsCredentials, 'aws-sdk-s3/access_grants_credentials'
31
+ autoload :AccessGrantsCredentialsProvider, 'aws-sdk-s3/access_grants_credentials_provider'
32
+ end
30
33
  end
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ # @api private
6
+ class DefaultExecutor
7
+ DEFAULT_MAX_THREADS = 10
8
+ RUNNING = :running
9
+ SHUTTING_DOWN = :shutting_down
10
+ SHUTDOWN = :shutdown
11
+
12
+ def initialize(options = {})
13
+ @max_threads = options[:max_threads] || DEFAULT_MAX_THREADS
14
+ @state = RUNNING
15
+ @queue = Queue.new
16
+ @pool = []
17
+ @mutex = Mutex.new
18
+ end
19
+
20
+ # Submits a task for execution.
21
+ # @param [Object] args Variable number of arguments to pass to the block
22
+ # @param [Proc] block The block to be executed
23
+ # @return [Boolean] Returns true if the task was submitted successfully
24
+ def post(*args, &block)
25
+ @mutex.synchronize do
26
+ raise 'Executor has been shutdown and is no longer accepting tasks' unless @state == RUNNING
27
+
28
+ @queue << [args, block]
29
+ ensure_worker_available
30
+ end
31
+ true
32
+ end
33
+
34
+ # Immediately terminates all worker threads and clears pending tasks.
35
+ # This is a forceful shutdown that doesn't wait for running tasks to complete.
36
+ #
37
+ # @return [Boolean] true when termination is complete
38
+ def kill
39
+ @mutex.synchronize do
40
+ @state = SHUTDOWN
41
+ @pool.each(&:kill)
42
+ @pool.clear
43
+ @queue.clear
44
+ end
45
+ true
46
+ end
47
+
48
+ # Gracefully shuts down the executor, optionally with a timeout.
49
+ # Stops accepting new tasks and waits for running tasks to complete.
50
+ #
51
+ # @param timeout [Numeric, nil] Maximum time in seconds to wait for shutdown.
52
+ # If nil, waits indefinitely. If timeout expires, remaining threads are killed.
53
+ # @return [Boolean] true when shutdown is complete
54
+ def shutdown(timeout = nil)
55
+ @mutex.synchronize do
56
+ return true if @state == SHUTDOWN
57
+
58
+ @state = SHUTTING_DOWN
59
+ @pool.size.times { @queue << :shutdown }
60
+ end
61
+
62
+ if timeout
63
+ deadline = Time.now + timeout
64
+ @pool.each do |thread|
65
+ remaining = deadline - Time.now
66
+ break if remaining <= 0
67
+
68
+ thread.join([remaining, 0].max)
69
+ end
70
+ @pool.select(&:alive?).each(&:kill)
71
+ else
72
+ @pool.each(&:join)
73
+ end
74
+
75
+ @mutex.synchronize do
76
+ @pool.clear
77
+ @state = SHUTDOWN
78
+ end
79
+ true
80
+ end
81
+
82
+ private
83
+
84
+ def ensure_worker_available
85
+ return unless @state == RUNNING
86
+
87
+ @pool.select!(&:alive?)
88
+ @pool << spawn_worker if @pool.size < @max_threads
89
+ end
90
+
91
+ def spawn_worker
92
+ Thread.new do
93
+ while (job = @queue.shift)
94
+ break if job == :shutdown
95
+
96
+ args, block = job
97
+ block.call(*args)
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
@@ -1,6 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'forwardable'
4
+
1
5
  module Aws
2
6
  module S3
3
7
 
8
+ # [MAINTENANCE MODE] There is a new version of the Encryption Client.
9
+ # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
10
+ # which provides updated data security best practices.
11
+ # See documentation for {Aws::S3::EncryptionV3::Client}.
4
12
  # Provides an encryption client that encrypts and decrypts data client-side,
5
13
  # storing the encrypted data in Amazon S3.
6
14
  #
@@ -112,7 +120,7 @@ module Aws
112
120
  # attr_reader :encryption_materials
113
121
  #
114
122
  # def key_for(matdesc)
115
- # key_name = JSON.load(matdesc)['key']
123
+ # key_name = JSON.parse(matdesc)['key']
116
124
  # if key = @keys[key_name]
117
125
  # key
118
126
  # else
@@ -178,8 +186,10 @@ module Aws
178
186
  class Client
179
187
 
180
188
  extend Deprecations
189
+ extend Forwardable
190
+ def_delegators :@client, :config, :delete_object, :head_object, :build_request
181
191
 
182
- # Creates a new encryption client. You must provide on of the following
192
+ # Creates a new encryption client. You must provide one of the following
183
193
  # options:
184
194
  #
185
195
  # * `:encryption_key`
@@ -223,6 +233,13 @@ module Aws
223
233
  @envelope_location = extract_location(options)
224
234
  @instruction_file_suffix = extract_suffix(options)
225
235
  end
236
+ deprecated :initialize,
237
+ message:
238
+ '[MAINTENANCE MODE] This version of the S3 Encryption client is currently in maintenance mode. ' \
239
+ 'AWS strongly recommends upgrading to the Aws::S3::EncryptionV2::Client, ' \
240
+ 'which provides updated data security best practices. ' \
241
+ 'See documentation for Aws::S3::EncryptionV2::Client.'
242
+
226
243
 
227
244
  # @return [S3::Client]
228
245
  attr_reader :client
@@ -253,7 +270,9 @@ module Aws
253
270
  envelope_location: @envelope_location,
254
271
  instruction_file_suffix: @instruction_file_suffix,
255
272
  }
256
- req.send_request
273
+ Aws::Plugins::UserAgent.metric('S3_CRYPTO_V1N') do
274
+ req.send_request
275
+ end
257
276
  end
258
277
 
259
278
  # Gets an object from Amazon S3, decrypting data locally.
@@ -281,7 +300,9 @@ module Aws
281
300
  envelope_location: envelope_location,
282
301
  instruction_file_suffix: instruction_file_suffix,
283
302
  }
284
- req.send_request(target: block)
303
+ Aws::Plugins::UserAgent.metric('S3_CRYPTO_V1N') do
304
+ req.send_request(target: block)
305
+ end
285
306
  end
286
307
 
287
308
  private
@@ -327,7 +348,7 @@ module Aws
327
348
  elsif options[:encryption_key]
328
349
  DefaultKeyProvider.new(options)
329
350
  else
330
- msg = "you must pass a :kms_key_id, :key_provider, or :encryption_key"
351
+ msg = 'you must pass a :kms_key_id, :key_provider, or :encryption_key'
331
352
  raise ArgumentError, msg
332
353
  end
333
354
  end
@@ -347,8 +368,8 @@ module Aws
347
368
  if [:metadata, :instruction_file].include?(location)
348
369
  location
349
370
  else
350
- msg = ":envelope_location must be :metadata or :instruction_file "
351
- msg << "got #{location.inspect}"
371
+ msg = ':envelope_location must be :metadata or :instruction_file '\
372
+ "got #{location.inspect}"
352
373
  raise ArgumentError, msg
353
374
  end
354
375
  end
@@ -358,7 +379,7 @@ module Aws
358
379
  if String === suffix
359
380
  suffix
360
381
  else
361
- msg = ":instruction_file_suffix must be a String"
382
+ msg = ':instruction_file_suffix must be a String'
362
383
  raise ArgumentError, msg
363
384
  end
364
385
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'base64'
2
4
 
3
5
  module Aws
@@ -5,6 +7,7 @@ module Aws
5
7
  module Encryption
6
8
  # @api private
7
9
  class DecryptHandler < Seahorse::Client::Handler
10
+ @@warned_response_target_proc = false
8
11
 
9
12
  V1_ENVELOPE_KEYS = %w(
10
13
  x-amz-key
@@ -20,7 +23,17 @@ module Aws
20
23
  x-amz-matdesc
21
24
  )
22
25
 
23
- POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS + V2_ENVELOPE_KEYS).uniq
26
+ V2_OPTIONAL_KEYS = %w(x-amz-tag-len)
27
+
28
+ POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS +
29
+ V2_ENVELOPE_KEYS + V2_OPTIONAL_KEYS).uniq
30
+
31
+ POSSIBLE_WRAPPING_FORMATS = %w(
32
+ AES/GCM
33
+ kms
34
+ kms+context
35
+ RSA-OAEP-SHA1
36
+ )
24
37
 
25
38
  POSSIBLE_ENCRYPTION_FORMATS = %w(
26
39
  AES/GCM/NoPadding
@@ -28,8 +41,21 @@ module Aws
28
41
  AES/CBC/PKCS7Padding
29
42
  )
30
43
 
44
+ AUTH_REQUIRED_CEK_ALGS = %w(AES/GCM/NoPadding)
45
+
31
46
  def call(context)
32
47
  attach_http_event_listeners(context)
48
+ apply_cse_user_agent(context)
49
+
50
+ if context[:response_target].is_a?(Proc) && !@@warned_response_target_proc
51
+ @@warned_response_target_proc = true
52
+ warn(':response_target is a Proc, or a block was provided. ' \
53
+ 'Read the entire object to the ' \
54
+ 'end before you start using the decrypted data. This is to ' \
55
+ 'verify that the object has not been modified since it ' \
56
+ 'was encrypted.')
57
+ end
58
+
33
59
  @handler.call(context)
34
60
  end
35
61
 
@@ -38,9 +64,9 @@ module Aws
38
64
  def attach_http_event_listeners(context)
39
65
 
40
66
  context.http_response.on_headers(200) do
41
- cipher = decryption_cipher(context)
42
- decrypter = body_contains_auth_tag?(context) ?
43
- authenticated_decrypter(context, cipher) :
67
+ cipher, envelope = decryption_cipher(context)
68
+ decrypter = body_contains_auth_tag?(envelope) ?
69
+ authenticated_decrypter(context, cipher, envelope) :
44
70
  IODecrypter.new(cipher, context.http_response.body)
45
71
  context.http_response.body = decrypter
46
72
  end
@@ -60,8 +86,13 @@ module Aws
60
86
  end
61
87
 
62
88
  def decryption_cipher(context)
63
- if envelope = get_encryption_envelope(context)
64
- context[:encryption][:cipher_provider].decryption_cipher(envelope)
89
+ if (envelope = get_encryption_envelope(context))
90
+ cipher = context[:encryption][:cipher_provider]
91
+ .decryption_cipher(
92
+ envelope,
93
+ context[:encryption]
94
+ )
95
+ [cipher, envelope]
65
96
  else
66
97
  raise Errors::DecryptionError, "unable to locate encryption envelope"
67
98
  end
@@ -97,13 +128,12 @@ module Aws
97
128
  end
98
129
 
99
130
  def extract_envelope(hash)
131
+ return nil unless hash
100
132
  return v1_envelope(hash) if hash.key?('x-amz-key')
101
133
  return v2_envelope(hash) if hash.key?('x-amz-key-v2')
102
134
  if hash.keys.any? { |key| key.match(/^x-amz-key-(.+)$/) }
103
135
  msg = "unsupported envelope encryption version #{$1}"
104
136
  raise Errors::DecryptionError, msg
105
- else
106
- nil # no envelope found
107
137
  end
108
138
  end
109
139
 
@@ -117,39 +147,27 @@ module Aws
117
147
  msg = "unsupported content encrypting key (cek) format: #{alg}"
118
148
  raise Errors::DecryptionError, msg
119
149
  end
120
- unless envelope['x-amz-wrap-alg'] == 'kms'
121
- # possible to support
122
- # RSA/ECB/OAEPWithSHA-256AndMGF1Padding
150
+ unless POSSIBLE_WRAPPING_FORMATS.include? envelope['x-amz-wrap-alg']
123
151
  alg = envelope['x-amz-wrap-alg'].inspect
124
152
  msg = "unsupported key wrapping algorithm: #{alg}"
125
153
  raise Errors::DecryptionError, msg
126
154
  end
127
- unless V2_ENVELOPE_KEYS.sort == envelope.keys.sort
155
+ unless (missing_keys = V2_ENVELOPE_KEYS - envelope.keys).empty?
128
156
  msg = "incomplete v2 encryption envelope:\n"
129
- msg += " expected: #{V2_ENVELOPE_KEYS.join(',')}\n"
130
- msg += " got: #{envelope_keys.join(', ')}"
157
+ msg += " missing: #{missing_keys.join(',')}\n"
131
158
  raise Errors::DecryptionError, msg
132
159
  end
133
160
  envelope
134
161
  end
135
162
 
136
- # When the x-amz-meta-x-amz-tag-len header is present, it indicates
137
- # that the body of this object has a trailing auth tag. The header
138
- # indicates the length of that tag.
139
- #
140
163
  # This method fetches the tag from the end of the object by
141
164
  # making a GET Object w/range request. This auth tag is used
142
165
  # to initialize the cipher, and the decrypter truncates the
143
166
  # auth tag from the body when writing the final bytes.
144
- def authenticated_decrypter(context, cipher)
145
- if RUBY_VERSION.match(/1.9/)
146
- raise "authenticated decryption not supported by OpeenSSL in Ruby version ~> 1.9"
147
- raise Aws::Errors::NonSupportedRubyVersionError, msg
148
- end
167
+ def authenticated_decrypter(context, cipher, envelope)
149
168
  http_resp = context.http_response
150
169
  content_length = http_resp.headers['content-length'].to_i
151
- auth_tag_length = http_resp.headers['x-amz-meta-x-amz-tag-len']
152
- auth_tag_length = auth_tag_length.to_i / 8
170
+ auth_tag_length = auth_tag_length(envelope)
153
171
 
154
172
  auth_tag = context.client.get_object(
155
173
  bucket: context.params[:bucket],
@@ -161,16 +179,40 @@ module Aws
161
179
  cipher.auth_data = ''
162
180
 
163
181
  # The encrypted object contains both the cipher text
164
- # plus a trailing auth tag. This decrypter will the body
165
- # expect for the trailing auth tag.
182
+ # plus a trailing auth tag.
166
183
  IOAuthDecrypter.new(
167
184
  io: http_resp.body,
168
185
  encrypted_content_length: content_length - auth_tag_length,
169
186
  cipher: cipher)
170
187
  end
171
188
 
172
- def body_contains_auth_tag?(context)
173
- context.http_response.headers['x-amz-meta-x-amz-tag-len']
189
+ def body_contains_auth_tag?(envelope)
190
+ AUTH_REQUIRED_CEK_ALGS.include?(envelope['x-amz-cek-alg'])
191
+ end
192
+
193
+ # Determine the auth tag length from the algorithm
194
+ # Validate it against the value provided in the x-amz-tag-len
195
+ # Return the tag length in bytes
196
+ def auth_tag_length(envelope)
197
+ tag_length =
198
+ case envelope['x-amz-cek-alg']
199
+ when 'AES/GCM/NoPadding' then AES_GCM_TAG_LEN_BYTES
200
+ else
201
+ raise ArgumentError, 'Unsupported cek-alg: ' \
202
+ "#{envelope['x-amz-cek-alg']}"
203
+ end
204
+ if (tag_length * 8) != envelope['x-amz-tag-len'].to_i
205
+ raise Errors::DecryptionError, 'x-amz-tag-len does not match expected'
206
+ end
207
+ tag_length
208
+ end
209
+
210
+ def apply_cse_user_agent(context)
211
+ if context.config.user_agent_suffix.nil?
212
+ context.config.user_agent_suffix = EC_USER_AGENT
213
+ elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
214
+ context.config.user_agent_suffix += " #{EC_USER_AGENT}"
215
+ end
174
216
  end
175
217
 
176
218
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'base64'
2
4
 
3
5
  module Aws
@@ -14,6 +16,8 @@ module Aws
14
16
  # envelope and encryption cipher.
15
17
  def encryption_cipher
16
18
  cipher = Utils.aes_encryption_cipher(:CBC)
19
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
20
+ ##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
17
21
  envelope = {
18
22
  'x-amz-key' => encode64(encrypt(envelope_key(cipher))),
19
23
  'x-amz-iv' => encode64(envelope_iv(cipher)),
@@ -24,11 +28,48 @@ module Aws
24
28
 
25
29
  # @return [Cipher] Given an encryption envelope, returns a
26
30
  # decryption cipher.
27
- def decryption_cipher(envelope)
31
+ def decryption_cipher(envelope, options = {})
28
32
  master_key = @key_provider.key_for(envelope['x-amz-matdesc'])
29
- key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
30
- iv = decode64(envelope['x-amz-iv'])
31
- Utils.aes_decryption_cipher(:CBC, key, iv)
33
+ if envelope.key? 'x-amz-key'
34
+ # Support for decryption of legacy objects
35
+ key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
36
+ iv = decode64(envelope['x-amz-iv'])
37
+ Utils.aes_decryption_cipher(:CBC, key, iv)
38
+ else
39
+ if envelope['x-amz-cek-alg'] != 'AES/GCM/NoPadding'
40
+ raise ArgumentError, 'Unsupported cek-alg: ' \
41
+ "#{envelope['x-amz-cek-alg']}"
42
+ end
43
+ key =
44
+ case envelope['x-amz-wrap-alg']
45
+ when 'AES/GCM'
46
+ if master_key.is_a? OpenSSL::PKey::RSA
47
+ raise ArgumentError, 'Key mismatch - Client is configured' \
48
+ ' with an RSA key and the x-amz-wrap-alg is AES/GCM.'
49
+ end
50
+ Utils.decrypt_aes_gcm(master_key,
51
+ decode64(envelope['x-amz-key-v2']),
52
+ envelope['x-amz-cek-alg'])
53
+ when 'RSA-OAEP-SHA1'
54
+ unless master_key.is_a? OpenSSL::PKey::RSA
55
+ raise ArgumentError, 'Key mismatch - Client is configured' \
56
+ ' with an AES key and the x-amz-wrap-alg is RSA-OAEP-SHA1.'
57
+ end
58
+ key, cek_alg = Utils.decrypt_rsa(master_key, decode64(envelope['x-amz-key-v2']))
59
+ raise Errors::DecryptionError unless cek_alg == envelope['x-amz-cek-alg']
60
+ key
61
+ when 'kms+context'
62
+ raise ArgumentError, 'Key mismatch - Client is configured' \
63
+ ' with a user provided key and the x-amz-wrap-alg is' \
64
+ ' kms+context. Please configure the client with the' \
65
+ ' required kms_key_id'
66
+ else
67
+ raise ArgumentError, 'Unsupported wrap-alg: ' \
68
+ "#{envelope['x-amz-wrap-alg']}"
69
+ end
70
+ iv = decode64(envelope['x-amz-iv'])
71
+ Utils.aes_decryption_cipher(:GCM, key, iv)
72
+ end
32
73
  end
33
74
 
34
75
  private
@@ -56,7 +97,6 @@ module Aws
56
97
  def decode64(str)
57
98
  Base64.decode64(str)
58
99
  end
59
-
60
100
  end
61
101
  end
62
102
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Aws
2
4
  module S3
3
5
  module Encryption
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'base64'
2
4
 
3
5
  module Aws
@@ -10,6 +12,7 @@ module Aws
10
12
  envelope, cipher = context[:encryption][:cipher_provider].encryption_cipher
11
13
  apply_encryption_envelope(context, envelope, cipher)
12
14
  apply_encryption_cipher(context, cipher)
15
+ apply_cse_user_agent(context)
13
16
  @handler.call(context)
14
17
  end
15
18
 
@@ -35,15 +38,25 @@ module Aws
35
38
  io = StringIO.new(io) if String === io
36
39
  context.params[:body] = IOEncrypter.new(cipher, io)
37
40
  context.params[:metadata] ||= {}
41
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys
42
+ ##% - The mapkey "x-amz-unencrypted-content-length" SHOULD be present for V1 format objects.
38
43
  context.params[:metadata]['x-amz-unencrypted-content-length'] = io.size
39
- if md5 = context.params.delete(:content_md5)
40
- context.params[:metadata]['x-amz-unencrypted-content-md5'] = md5
44
+ if context.params.delete(:content_md5)
45
+ warn('Setting content_md5 on client side encrypted objects is deprecated')
41
46
  end
42
47
  context.http_response.on_headers do
43
48
  context.params[:body].close
44
49
  end
45
50
  end
46
51
 
52
+ def apply_cse_user_agent(context)
53
+ if context.config.user_agent_suffix.nil?
54
+ context.config.user_agent_suffix = EC_USER_AGENT
55
+ elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
56
+ context.config.user_agent_suffix += " #{EC_USER_AGENT}"
57
+ end
58
+ end
59
+
47
60
  end
48
61
  end
49
62
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Aws
2
4
  module S3
3
5
  module Encryption
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Aws
2
4
  module S3
3
5
  module Encryption
@@ -22,8 +24,10 @@ module Aws
22
24
 
23
25
  def write(chunk)
24
26
  chunk = truncate_chunk(chunk)
25
- @bytes_written += chunk.bytesize
26
- @decrypter.write(chunk)
27
+ if chunk.bytesize > 0
28
+ @bytes_written += chunk.bytesize
29
+ @decrypter.write(chunk)
30
+ end
27
31
  end
28
32
 
29
33
  def finalize
@@ -39,8 +43,12 @@ module Aws
39
43
  def truncate_chunk(chunk)
40
44
  if chunk.bytesize + @bytes_written <= @max_bytes
41
45
  chunk
42
- else
46
+ elsif @bytes_written < @max_bytes
43
47
  chunk[0..(@max_bytes - @bytes_written - 1)]
48
+ else
49
+ # If the tag was sent over after the full body has been read,
50
+ # we don't want to accidentally append it.
51
+ ""
44
52
  end
45
53
  end
46
54
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Aws
2
4
  module S3
3
5
  module Encryption
@@ -7,8 +9,10 @@ module Aws
7
9
  # @param [OpenSSL::Cipher] cipher
8
10
  # @param [IO#write] io An IO-like object that responds to `#write`.
9
11
  def initialize(cipher, io)
10
- @cipher = cipher.clone
11
- @io = io
12
+ @cipher = cipher
13
+ # Ensure that IO is reset between retries
14
+ @io = io.tap { |io| io.truncate(0) if io.respond_to?(:truncate) }
15
+ @cipher_buffer = String.new
12
16
  end
13
17
 
14
18
  # @return [#write]
@@ -16,7 +20,11 @@ module Aws
16
20
 
17
21
  def write(chunk)
18
22
  # decrypt and write
19
- @io.write(@cipher.update(chunk))
23
+ if @cipher.method(:update).arity == 1
24
+ @io.write(@cipher.update(chunk))
25
+ else
26
+ @io.write(@cipher.update(chunk, @cipher_buffer))
27
+ end
20
28
  end
21
29
 
22
30
  def finalize
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'stringio'
2
4
  require 'tempfile'
3
5
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Aws
2
4
  module S3
3
5
  module Encryption