aws-sdk 1.6.2 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/aws/core.rb +13 -2
- data/lib/aws/core/autoloader.rb +1 -1
- data/lib/aws/core/client.rb +69 -30
- data/lib/aws/core/configuration.rb +12 -1
- data/lib/aws/core/http/handler.rb +28 -16
- data/lib/aws/core/http/net_http_handler.rb +31 -11
- data/lib/aws/core/http/request.rb +52 -16
- data/lib/aws/core/http/response.rb +20 -16
- data/lib/aws/core/indifferent_hash.rb +14 -14
- data/lib/aws/core/query_client.rb +1 -0
- data/lib/aws/core/response.rb +32 -14
- data/lib/aws/core/signature/version_2.rb +1 -0
- data/lib/aws/core/signature/version_4.rb +16 -16
- data/lib/aws/dynamo_db/client.rb +2 -2
- data/lib/aws/dynamo_db/request.rb +0 -6
- data/lib/aws/ec2/security_group/ip_permission.rb +4 -1
- data/lib/aws/rails.rb +10 -10
- data/lib/aws/s3.rb +44 -29
- data/lib/aws/s3/bucket.rb +171 -6
- data/lib/aws/s3/cipher_io.rb +119 -0
- data/lib/aws/s3/client.rb +75 -45
- data/lib/aws/s3/config.rb +6 -0
- data/lib/aws/s3/data_options.rb +136 -49
- data/lib/aws/s3/encryption_utils.rb +144 -0
- data/lib/aws/s3/errors.rb +14 -0
- data/lib/aws/s3/multipart_upload.rb +7 -4
- data/lib/aws/s3/object_collection.rb +2 -2
- data/lib/aws/s3/policy.rb +1 -1
- data/lib/aws/s3/request.rb +21 -33
- data/lib/aws/s3/s3_object.rb +797 -237
- data/lib/aws/simple_email_service/request.rb +0 -2
- data/lib/aws/simple_workflow/request.rb +0 -3
- data/lib/net/http/connection_pool.rb +63 -75
- data/lib/net/http/connection_pool/connection.rb +69 -15
- data/lib/net/http/connection_pool/session.rb +39 -6
- metadata +4 -2
data/lib/aws/s3/s3_object.rb
CHANGED
@@ -16,30 +16,228 @@ require 'uri'
|
|
16
16
|
module AWS
|
17
17
|
class S3
|
18
18
|
|
19
|
-
# Represents an object in S3
|
19
|
+
# Represents an object in S3. Objects live in a bucket and have
|
20
|
+
# unique keys.
|
20
21
|
#
|
21
|
-
#
|
22
|
-
#
|
22
|
+
# = Getting Objects
|
23
|
+
#
|
24
|
+
# You can get an object by its key.
|
25
|
+
#
|
26
|
+
# s3 = AWS::S3.new
|
27
|
+
# obj = s3.buckets['my-bucket'].objects['key'] # no request made
|
28
|
+
#
|
29
|
+
# You can also get objects by enumerating a objects in a bucket.
|
30
|
+
#
|
31
|
+
# bucket.objects.each do |obj|
|
32
|
+
# puts obj.key
|
33
|
+
# end
|
23
34
|
#
|
24
35
|
# See {ObjectCollection} for more information on finding objects.
|
25
|
-
#
|
26
|
-
#
|
36
|
+
#
|
37
|
+
# = Creating Objects
|
38
|
+
#
|
39
|
+
# You create an object by writing to it. The following two
|
40
|
+
# expressions are equivalent.
|
41
|
+
#
|
42
|
+
# obj = bucket.objects.create('key', 'data')
|
43
|
+
# obj = bucket.objects['key'].write('data')
|
44
|
+
#
|
45
|
+
# = Writing Objects
|
46
|
+
#
|
47
|
+
# To upload data to S3, you simply need to call {#write} on an object.
|
48
|
+
#
|
49
|
+
# obj.write('Hello World!')
|
50
|
+
# obj.read
|
51
|
+
# #=> 'Hello World!'
|
52
|
+
#
|
53
|
+
# == Uploading Files
|
54
|
+
#
|
55
|
+
# You can upload a file to S3 in a variety of ways. Given a path
|
56
|
+
# to a file (as a string) you can do any of the following:
|
57
|
+
#
|
58
|
+
# # specify the data as a path to a file
|
59
|
+
# obj.write(Pathname.new(path_to_file))
|
60
|
+
#
|
61
|
+
# # also works this way
|
62
|
+
# obj.write(:file => path_to_file)
|
63
|
+
#
|
64
|
+
# # Also accepts an open file object
|
65
|
+
# file = File.open(path_to_file, 'r')
|
66
|
+
# obj.write(file)
|
67
|
+
#
|
68
|
+
# All three examples above produce the same result. The file
|
69
|
+
# will be streamed to S3 in chunks. It will not be loaded
|
70
|
+
# entirely into memory.
|
71
|
+
#
|
72
|
+
# == Streaming Uploads
|
73
|
+
#
|
74
|
+
# When you call {#write} with any IO-like object (must respond to
|
75
|
+
# #read and #eof?), it will be streamed to S3 in chunks.
|
76
|
+
#
|
77
|
+
# While it is possible to determine the size of many IO objects, you may
|
78
|
+
# have to specify the :content_length of your IO object.
|
79
|
+
# If the exact size can not be known, you may provide an
|
80
|
+
# +:estimated_content_length+. Depending on the size (actual or
|
81
|
+
# estimated) of your data, it will be uploaded in a single request or
|
82
|
+
# in multiple requests via {#multipart_upload}.
|
83
|
+
#
|
84
|
+
# You may also stream uploads to S3 using a block:
|
85
|
+
#
|
86
|
+
# obj.write do |buffer, bytes|
|
87
|
+
# # writing fewer than the requested number of bytes to the buffer
|
88
|
+
# # will cause write to stop yielding to the block
|
89
|
+
# end
|
90
|
+
#
|
91
|
+
# = Reading Objects
|
92
|
+
#
|
93
|
+
# You can read an object directly using {#read}. Be warned, this will
|
94
|
+
# load the entire object into memory and is not recommended for large
|
95
|
+
# objects.
|
96
|
+
#
|
97
|
+
# obj.write('abc')
|
98
|
+
# puts obj.read
|
99
|
+
# #=> abc
|
100
|
+
#
|
101
|
+
# == Streaming Downloads
|
102
|
+
#
|
103
|
+
# If you want to stream an object from S3, you can pass a block
|
104
|
+
# to {#read}.
|
105
|
+
#
|
106
|
+
# File.open('output', 'w') do |file|
|
107
|
+
# large_object.read do |chunk|
|
108
|
+
# file.write(chunk)
|
109
|
+
# end
|
110
|
+
# end
|
111
|
+
#
|
112
|
+
# = Encryption
|
113
|
+
#
|
114
|
+
# Amazon S3 can encrypt objects for you service-side. You can also
|
115
|
+
# use client-side encryption.
|
116
|
+
#
|
117
|
+
# == Server Side Encryption
|
118
|
+
#
|
119
|
+
# Amazon S3 provides server side encryption for an additional cost.
|
120
|
+
# You can specify to use server side encryption when writing an object.
|
121
|
+
#
|
122
|
+
# obj.write('data', :server_size_encryption => :aes256)
|
123
|
+
#
|
124
|
+
# You can also make this the default behavior.
|
125
|
+
#
|
126
|
+
# AWS.config(:s3_server_side_encryption => :aes256)
|
127
|
+
#
|
128
|
+
# s3 = AWS::S3.new
|
129
|
+
# s3.buckets['name'].objects['key'].write('abc') # will be encrypted
|
130
|
+
#
|
131
|
+
# == Client Side Encryption
|
132
|
+
#
|
133
|
+
# Client side encryption utilizes envelope encryption, so that your keys are
|
134
|
+
# never sent to S3. You can use a symetric key or an asymmetric
|
135
|
+
# key pair.
|
136
|
+
#
|
137
|
+
# === Symmetric Key Encryption
|
138
|
+
#
|
139
|
+
# An AES key is used for symmetric encryption. The key can be 128, 192,
|
140
|
+
# and 256 bit sizes. Start by generating key or read a previously
|
141
|
+
# generated key.
|
142
|
+
#
|
143
|
+
# # generate a new random key
|
144
|
+
# my_key = OpenSSL::Cipher.new("AES-256-ECB").random_key
|
145
|
+
#
|
146
|
+
# # read an existing key from disk
|
147
|
+
# my_key = File.read("my_key.der")
|
148
|
+
#
|
149
|
+
# Now you can encrypt locally and upload the encrypted data to S3.
|
150
|
+
# To do this, you need to provide your key.
|
27
151
|
#
|
28
152
|
# obj = bucket.objects["my-text-object"]
|
29
153
|
#
|
30
|
-
#
|
154
|
+
# # encrypt then upload data
|
155
|
+
# obj.write("MY TEXT", :encryption_key => my_key)
|
156
|
+
#
|
157
|
+
# # try read the object without decrypting, oops
|
31
158
|
# obj.read
|
159
|
+
# #=> '.....'
|
160
|
+
#
|
161
|
+
# Lastly, you can download and decrypt by providing the same key.
|
162
|
+
#
|
163
|
+
# obj.read(:encryption_key => my_key)
|
32
164
|
# #=> "MY TEXT"
|
33
165
|
#
|
34
|
-
#
|
35
|
-
#
|
36
|
-
#
|
166
|
+
# === Asymmetric Key Pair
|
167
|
+
#
|
168
|
+
# A RSA key pair is used for asymmetric encryption. The public key is used
|
169
|
+
# for encryption and the private key is used for decryption. Start
|
170
|
+
# by generating a key.
|
171
|
+
#
|
172
|
+
# my_key = OpenSSL::PKey::RSA.new(1024)
|
173
|
+
#
|
174
|
+
# Provide your key to #write and the data will be encrypted before it
|
175
|
+
# is uploaded. Pass the same key to #read to decrypt the data
|
176
|
+
# when you download it.
|
177
|
+
#
|
178
|
+
# obj = bucket.objects["my-text-object"]
|
179
|
+
#
|
180
|
+
# # encrypt and upload the data
|
181
|
+
# obj.write("MY TEXT", :encryption_key => my_key)
|
182
|
+
#
|
183
|
+
# # download and decrypt the data
|
184
|
+
# obj.read(:encryption_key => my_key)
|
185
|
+
# #=> "MY TEXT"
|
186
|
+
#
|
187
|
+
# === Configuring storage locations
|
188
|
+
#
|
189
|
+
# By default, encryption materials are stored in the object metadata.
|
190
|
+
# If you prefer, you can store the encryption materials in a separate
|
191
|
+
# object in S3. This object will have the same key + '.instruction'.
|
192
|
+
#
|
193
|
+
# # new object, does not exist yet
|
194
|
+
# obj = bucket.objects["my-text-object"]
|
195
|
+
#
|
196
|
+
# # no instruction file present
|
197
|
+
# bucket.objects['my-text-object.instruction'].exists?
|
198
|
+
# #=> false
|
199
|
+
#
|
200
|
+
# # store the encryption materials in the instruction file
|
201
|
+
# # instead of obj#metadata
|
202
|
+
# obj.write("MY TEXT",
|
203
|
+
# :encryption_key => MY_KEY,
|
204
|
+
# :encryption_materials_location => :instruction_file)
|
205
|
+
#
|
206
|
+
# bucket.objects['my-text-object.instruction'].exists?
|
207
|
+
# #=> true
|
208
|
+
#
|
209
|
+
# If you store the encryption materials in an instruction file, you
|
210
|
+
# must tell #read this or it will fail to find your encryption materials.
|
211
|
+
#
|
212
|
+
# # reading an encrypted file whos materials are stored in an
|
213
|
+
# # instruction file, and not metadata
|
214
|
+
# obj.read(:encryption_key => MY_KEY,
|
215
|
+
# :encryption_materials_location => :instruction_file)
|
216
|
+
#
|
217
|
+
# === Configuring default behaviors
|
218
|
+
#
|
219
|
+
# You can configure the default key such that it will automatically
|
220
|
+
# encrypt and decrypt for you. You can do this globally or for a
|
221
|
+
# single S3 interface
|
222
|
+
#
|
223
|
+
# # all objects uploaded/downloaded with this s3 object will be
|
224
|
+
# # encrypted/decrypted
|
225
|
+
# s3 = AWS::S3.new(:s3_encryption_key => "MY_KEY")
|
226
|
+
#
|
227
|
+
# # set the key to always encrypt/decrypt
|
228
|
+
# AWS.config(:s3_encryption_key => "MY_KEY")
|
229
|
+
#
|
230
|
+
# You can also configure the default storage location for the encryption
|
231
|
+
# materials.
|
232
|
+
#
|
233
|
+
# AWS.config(:s3_encryption_materials_location => :instruction_file)
|
37
234
|
#
|
38
235
|
class S3Object
|
39
236
|
|
40
237
|
include Core::Model
|
41
238
|
include DataOptions
|
42
239
|
include ACLOptions
|
240
|
+
include AWS::S3::EncryptionUtils
|
43
241
|
|
44
242
|
# @param [Bucket] bucket The bucket this object belongs to.
|
45
243
|
# @param [String] key The object's key.
|
@@ -62,12 +260,12 @@ module AWS
|
|
62
260
|
|
63
261
|
# @return [Boolean] Returns true if the other object belongs to the
|
64
262
|
# same bucket and has the same key.
|
65
|
-
def ==
|
263
|
+
def == other
|
66
264
|
other.kind_of?(S3Object) and other.bucket == bucket and other.key == key
|
67
265
|
end
|
68
|
-
|
69
266
|
alias_method :eql?, :==
|
70
267
|
|
268
|
+
# @return [Boolean] Returns +true+ if the object exists in S3.
|
71
269
|
def exists?
|
72
270
|
head
|
73
271
|
rescue Errors::NoSuchKey => e
|
@@ -85,20 +283,20 @@ module AWS
|
|
85
283
|
# * etag (typically the object's MD5)
|
86
284
|
# * server_side_encryption (the algorithm used to encrypt the
|
87
285
|
# object on the server side, e.g. +:aes256+)
|
88
|
-
#
|
286
|
+
#
|
89
287
|
# @param [Hash] options
|
90
288
|
# @option options [String] :version_id Which version of this object
|
91
289
|
# to make a HEAD request against.
|
92
|
-
# @return A head object response with
|
290
|
+
# @return A head object response with metadata,
|
93
291
|
# content_length, content_type, etag and server_side_encryption.
|
94
292
|
def head options = {}
|
95
293
|
client.head_object(options.merge(
|
96
294
|
:bucket_name => bucket.name, :key => key))
|
97
295
|
end
|
98
296
|
|
99
|
-
# Returns the object's ETag.
|
297
|
+
# Returns the object's ETag.
|
100
298
|
#
|
101
|
-
# Generally the ETAG is the MD5 of the object. If the object was
|
299
|
+
# Generally the ETAG is the MD5 of the object. If the object was
|
102
300
|
# uploaded using multipart upload then this is the MD5 all of the
|
103
301
|
# upload-part-md5s.
|
104
302
|
#
|
@@ -121,7 +319,7 @@ module AWS
|
|
121
319
|
|
122
320
|
# @note S3 does not compute content-type. It reports the content-type
|
123
321
|
# as was reported during the file upload.
|
124
|
-
# @return [String] Returns the content type as reported by S3,
|
322
|
+
# @return [String] Returns the content type as reported by S3,
|
125
323
|
# defaults to an empty string when not provided during upload.
|
126
324
|
def content_type
|
127
325
|
head.content_type
|
@@ -153,181 +351,205 @@ module AWS
|
|
153
351
|
# Deletes the object from its S3 bucket.
|
154
352
|
#
|
155
353
|
# @param [Hash] options
|
354
|
+
#
|
156
355
|
# @option [String] :version_id (nil) If present the specified version
|
157
356
|
# of this object will be deleted. Only works for buckets that have
|
158
357
|
# had versioning enabled.
|
358
|
+
#
|
359
|
+
# @option [Boolean] :delete_instruction_file (false) Set this to +true+
|
360
|
+
# if you use client-side encryption and the encryption materials
|
361
|
+
# were stored in a separate object in S3 (key.instruction).
|
362
|
+
#
|
159
363
|
# @return [nil]
|
160
364
|
def delete options = {}
|
161
|
-
|
162
|
-
options
|
163
|
-
|
365
|
+
|
366
|
+
client.delete_object(options.merge(
|
367
|
+
:bucket_name => bucket.name,
|
368
|
+
:key => key))
|
369
|
+
|
370
|
+
if options[:delete_instruction_file]
|
371
|
+
client.delete_object(
|
372
|
+
:bucket_name => bucket.name,
|
373
|
+
:key => key + '.instruction')
|
374
|
+
end
|
375
|
+
|
164
376
|
nil
|
377
|
+
|
165
378
|
end
|
166
379
|
|
167
380
|
# @option [String] :version_id (nil) If present the metadata object
|
168
381
|
# will be for the specified version.
|
169
|
-
# @return [ObjectMetadata] Returns an instance of ObjectMetadata
|
382
|
+
# @return [ObjectMetadata] Returns an instance of ObjectMetadata
|
170
383
|
# representing the metadata for this object.
|
171
384
|
def metadata options = {}
|
172
385
|
options[:config] = config
|
173
386
|
ObjectMetadata.new(self, options)
|
174
387
|
end
|
175
388
|
|
176
|
-
# Returns a
|
389
|
+
# Returns a collection representing all the object versions
|
177
390
|
# for this object.
|
178
391
|
#
|
179
392
|
# bucket.versioning_enabled? # => true
|
180
393
|
# version = bucket.objects["mykey"].versions.latest
|
181
394
|
#
|
182
|
-
# @return [ObjectVersionCollection]
|
395
|
+
# @return [ObjectVersionCollection]
|
183
396
|
def versions
|
184
397
|
ObjectVersionCollection.new(self)
|
185
398
|
end
|
186
399
|
|
187
|
-
#
|
188
|
-
# to intelligently choose between uploading in one request and
|
189
|
-
# using {#multipart_upload}.
|
190
|
-
#
|
191
|
-
# Unless versioning is enabled, any data currently in S3 at {#key}
|
192
|
-
# will be replaced.
|
400
|
+
# Uploads data to the object in S3.
|
193
401
|
#
|
194
|
-
#
|
195
|
-
# options. Example usage:
|
402
|
+
# obj = s3.buckets['bucket-name'].objects['key']
|
196
403
|
#
|
197
|
-
#
|
404
|
+
# # strings
|
198
405
|
# obj.write("HELLO")
|
199
|
-
# obj.write(:data => "HELLO")
|
200
|
-
# obj.write(Pathname.new("myfile"))
|
201
|
-
# obj.write(:file => "myfile")
|
202
|
-
#
|
203
|
-
# # writes zero-length data
|
204
|
-
# obj.write(:metadata => { "avg-rating" => "5 stars" })
|
205
|
-
#
|
206
|
-
# @overload write(options = {})
|
207
|
-
# @overload write(data, options = {})
|
208
|
-
# @param data The data to upload (see the +:data+ option).
|
209
|
-
#
|
210
|
-
# @param options [Hash] Additional upload options.
|
211
|
-
#
|
212
|
-
# @option options :data The data to upload. Valid values include:
|
213
|
-
#
|
214
|
-
# * A string
|
215
|
-
# * A Pathname object
|
216
|
-
# * Any object responding to +read+ and +eof?+; the object
|
217
|
-
# must support the following access methods:
|
218
|
-
# read # all at once
|
219
|
-
# read(length) until eof? # in chunks
|
220
|
-
#
|
221
|
-
# If you specify data this way, you must also include the
|
222
|
-
# +:content_length+ option.
|
223
|
-
#
|
224
|
-
# @option options [String] :file Can be specified instead of +:data+;
|
225
|
-
# its value specifies the path of a file to upload.
|
226
|
-
#
|
227
|
-
# @option options [Boolean] :single_request If this option is
|
228
|
-
# true, the method will always generate exactly one request
|
229
|
-
# to S3 regardless of how much data is being uploaded.
|
230
406
|
#
|
231
|
-
#
|
232
|
-
#
|
233
|
-
# during the operation. This option is required if +:data+
|
234
|
-
# is an IO-like object without a +size+ method.
|
407
|
+
# # files (by path)
|
408
|
+
# obj.write(Pathname.new('path/to/file.txt'))
|
235
409
|
#
|
236
|
-
#
|
237
|
-
#
|
238
|
-
# data being uploaded is larger than this threshold, it will
|
239
|
-
# be uploaded using {#multipart_upload}.
|
410
|
+
# # file objects
|
411
|
+
# obj.write(File.open('path/to/file.txt', 'r'))
|
240
412
|
#
|
241
|
-
#
|
242
|
-
#
|
243
|
-
# will reject non-final parts smaller than 5MB, and the
|
244
|
-
# default for this option is 5MB.
|
413
|
+
# # IO objects (must respond to #read and #eof?)
|
414
|
+
# obj.write(io)
|
245
415
|
#
|
246
|
-
#
|
247
|
-
# included with the object. These will be sent to S3 as
|
248
|
-
# headers prefixed with +x-amz-meta+. Each name, value pair
|
249
|
-
# must conform to US-ASCII.
|
250
|
-
#
|
251
|
-
# @option options [Symbol,String] :acl (:private) A canned access
|
252
|
-
# control policy. Valid values are:
|
253
|
-
#
|
254
|
-
# * +:private+
|
255
|
-
# * +:public_read+
|
256
|
-
# * +:public_read_write+
|
257
|
-
# * +:authenticated_read+
|
258
|
-
# * +:bucket_owner_read+
|
259
|
-
# * +:bucket_owner_full_control+
|
260
|
-
#
|
261
|
-
# @option options [String] :grant_read
|
262
|
-
# @option options [String] :grant_write
|
263
|
-
# @option options [String] :grant_read_acp
|
264
|
-
# @option options [String] :grant_write_acp
|
265
|
-
# @option options [String] :grant_full_control
|
416
|
+
# === Multipart Uploads vs Single Uploads
|
266
417
|
#
|
267
|
-
#
|
268
|
-
#
|
269
|
-
#
|
270
|
-
#
|
271
|
-
#
|
272
|
-
# @option options :cache_control [String] Can be used to specify
|
273
|
-
# caching behavior. See
|
274
|
-
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
|
418
|
+
# This method will intelligently choose between uploading the
|
419
|
+
# file in a signal request and using {#multipart_upload}.
|
420
|
+
# You can control this behavior by configuring the thresholds
|
421
|
+
# and you can disable the multipart feature as well.
|
275
422
|
#
|
276
|
-
#
|
277
|
-
#
|
278
|
-
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec19.5.1
|
423
|
+
# # always send the file in a single request
|
424
|
+
# obj.write(file, :single_request => true)
|
279
425
|
#
|
280
|
-
#
|
281
|
-
#
|
282
|
-
# what decoding mechanisms must be applied to obtain the
|
283
|
-
# media-type referenced by the +Content-Type+ header field.
|
284
|
-
# See
|
285
|
-
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
|
426
|
+
# # upload the file in parts if the total file size exceeds 100MB
|
427
|
+
# obj.write(file, :multipart_threshold => 100 * 1024 * 1024)
|
286
428
|
#
|
287
|
-
# @
|
288
|
-
# describing the format of the object data.
|
429
|
+
# @overload write(data, options = {})
|
289
430
|
#
|
290
|
-
#
|
291
|
-
#
|
292
|
-
#
|
293
|
-
#
|
294
|
-
#
|
295
|
-
#
|
296
|
-
#
|
297
|
-
#
|
431
|
+
# @param [String,Pathname,File,IO] data The data to upload.
|
432
|
+
# This may be a:
|
433
|
+
# * String
|
434
|
+
# * Pathname
|
435
|
+
# * File
|
436
|
+
# * IO
|
437
|
+
# * Any object that responds to +#read+ and +#eof?+.
|
438
|
+
#
|
439
|
+
# @param options [Hash] Additional upload options.
|
440
|
+
#
|
441
|
+
# @option options [Integer] :content_length If provided, this
|
442
|
+
# option must match the total number of bytes written to S3.
|
443
|
+
# This options is *required* when it is not possible to
|
444
|
+
# automatically determine the size of +data+.
|
445
|
+
#
|
446
|
+
# @option options [Integer] :estimated_content_length When uploading
|
447
|
+
# data of unknown content length, you may specify this option to
|
448
|
+
# hint what mode of upload should take place. When
|
449
|
+
# +:estimated_content_length+ exceeds the +:multipart_threshold+,
|
450
|
+
# then the data will be uploaded in parts, otherwise it will
|
451
|
+
# be read into memory and uploaded via {Client#put_object}.
|
452
|
+
#
|
453
|
+
# @option options [Boolean] :single_request (false) When +true+,
|
454
|
+
# this method will always upload the data in a single request
|
455
|
+
# (via {Client#put_object}). When +false+, this method will
|
456
|
+
# choose between {Client#put_object} and {#multipart_upload}.
|
457
|
+
#
|
458
|
+
# @option options [Integer] :multipart_threshold (16777216) Specifies
|
459
|
+
# the maximum size (in bytes) of a single-request upload. If the
|
460
|
+
# data exceeds this threshold, it will be uploaded via
|
461
|
+
# {#multipart_upload}. The default threshold is 16MB and can
|
462
|
+
# be configured via AWS.config(:s3_multipart_threshold => ...).
|
463
|
+
#
|
464
|
+
# @option options [Integer] :multipart_min_part_size (5242880) The
|
465
|
+
# minimum size of a part to upload to S3 when using
|
466
|
+
# {#multipart_upload}. S3 will reject parts smaller than 5MB
|
467
|
+
# (except the final part). The default is 5MB and can be
|
468
|
+
# configured via AWS.config(:s3_multipart_min_part_size => ...).
|
469
|
+
#
|
470
|
+
# @option options [Hash] :metadata A hash of metadata to be
|
471
|
+
# included with the object. These will be sent to S3 as
|
472
|
+
# headers prefixed with +x-amz-meta+. Each name, value pair
|
473
|
+
# must conform to US-ASCII.
|
474
|
+
#
|
475
|
+
# @option options [Symbol,String] :acl (:private) A canned access
|
476
|
+
# control policy. Valid values are:
|
477
|
+
#
|
478
|
+
# * +:private+
|
479
|
+
# * +:public_read+
|
480
|
+
# * +:public_read_write+
|
481
|
+
# * +:authenticated_read+
|
482
|
+
# * +:bucket_owner_read+
|
483
|
+
# * +:bucket_owner_full_control+
|
484
|
+
#
|
485
|
+
# @option options [String] :grant_read
|
486
|
+
#
|
487
|
+
# @option options [String] :grant_write
|
488
|
+
#
|
489
|
+
# @option options [String] :grant_read_acp
|
490
|
+
#
|
491
|
+
# @option options [String] :grant_write_acp
|
492
|
+
#
|
493
|
+
# @option options [String] :grant_full_control
|
494
|
+
#
|
495
|
+
# @option options [Boolean] :reduced_redundancy (false) When +true+,
|
496
|
+
# this object will be stored with Reduced Redundancy Storage.
|
497
|
+
#
|
498
|
+
# @option options :cache_control [String] Can be used to specify
|
499
|
+
# caching behavior. See
|
500
|
+
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
|
501
|
+
#
|
502
|
+
# @option options :content_disposition [String] Specifies
|
503
|
+
# presentational information for the object. See
|
504
|
+
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec19.5.1
|
505
|
+
#
|
506
|
+
# @option options :content_encoding [String] Specifies what
|
507
|
+
# content encodings have been applied to the object and thus
|
508
|
+
# what decoding mechanisms must be applied to obtain the
|
509
|
+
# media-type referenced by the +Content-Type+ header field.
|
510
|
+
# See
|
511
|
+
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
|
512
|
+
#
|
513
|
+
# @option options :content_type A standard MIME type
|
514
|
+
# describing the format of the object data.
|
515
|
+
#
|
516
|
+
# @option options [Symbol] :server_side_encryption (nil) If this
|
517
|
+
# option is set, the object will be stored using server side
|
518
|
+
# encryption. The only valid value is +:aes256+, which
|
519
|
+
# specifies that the object should be stored using the AES
|
520
|
+
# encryption algorithm with 256 bit keys. By default, this
|
521
|
+
# option uses the value of the +:s3_server_side_encryption+
|
522
|
+
# option in the current configuration; for more information,
|
523
|
+
# see {AWS.config}.
|
524
|
+
#
|
525
|
+
# @option options [OpenSSL::PKey::RSA, String] :encryption_key
|
526
|
+
# Set this to encrypt the data client-side using envelope
|
527
|
+
# encryption. The key must be an OpenSSL asymmetric key
|
528
|
+
# or a symmetric key string (16, 24 or 32 bytes in length).
|
529
|
+
#
|
530
|
+
# @option options [Symbol] :encryption_materials_location (:metadata)
|
531
|
+
# Set this to +:instruction_file+ if you prefer to store the
|
532
|
+
# client-side encryption materials in a separate object in S3
|
533
|
+
# instead of in the object metadata.
|
298
534
|
#
|
299
535
|
# @return [S3Object, ObjectVersion] If the bucket has versioning
|
300
|
-
# enabled, returns
|
301
|
-
#
|
302
|
-
# returns self.
|
536
|
+
# enabled, this methods returns an {ObjectVersion}, otherwise
|
537
|
+
# this method returns +self+.
|
303
538
|
#
|
304
|
-
def write
|
539
|
+
def write *args, &block
|
305
540
|
|
306
|
-
(
|
307
|
-
compute_put_options(options_or_data, options)
|
541
|
+
options = compute_write_options(*args, &block)
|
308
542
|
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
add_configured_write_options(put_options)
|
543
|
+
add_storage_class_option(options)
|
544
|
+
add_sse_options(options)
|
545
|
+
add_cse_options(options)
|
314
546
|
|
315
|
-
if use_multipart?(
|
316
|
-
|
317
|
-
multipart_upload(put_options) do |upload|
|
318
|
-
each_part(data_options, put_options) do |part|
|
319
|
-
upload.add_part(part)
|
320
|
-
end
|
321
|
-
end
|
547
|
+
if use_multipart?(options)
|
548
|
+
write_with_multipart(options)
|
322
549
|
else
|
323
|
-
|
324
|
-
resp = client.put_object(opts.merge(put_options).merge(data_options))
|
325
|
-
if resp.data[:version_id]
|
326
|
-
ObjectVersion.new(self, resp.data[:version_id])
|
327
|
-
else
|
328
|
-
self
|
329
|
-
end
|
550
|
+
write_with_put_object(options)
|
330
551
|
end
|
552
|
+
|
331
553
|
end
|
332
554
|
|
333
555
|
# Performs a multipart upload. Use this if you have specific
|
@@ -421,10 +643,11 @@ module AWS
|
|
421
643
|
# enabled, returns the {ObjectVersion} representing the
|
422
644
|
# version that was uploaded. If versioning is disabled,
|
423
645
|
# returns self.
|
646
|
+
#
|
424
647
|
def multipart_upload(options = {})
|
425
648
|
|
426
649
|
options = options.dup
|
427
|
-
|
650
|
+
add_sse_options(options)
|
428
651
|
|
429
652
|
upload = multipart_uploads.create(options)
|
430
653
|
|
@@ -432,8 +655,9 @@ module AWS
|
|
432
655
|
begin
|
433
656
|
yield(upload)
|
434
657
|
upload.close
|
435
|
-
rescue
|
658
|
+
rescue => e
|
436
659
|
upload.abort
|
660
|
+
raise e
|
437
661
|
end
|
438
662
|
else
|
439
663
|
upload
|
@@ -450,10 +674,10 @@ module AWS
|
|
450
674
|
ObjectUploadCollection.new(self)
|
451
675
|
end
|
452
676
|
|
453
|
-
# Moves an object to a new key.
|
677
|
+
# Moves an object to a new key.
|
454
678
|
#
|
455
679
|
# This works by copying the object to a new key and then
|
456
|
-
# deleting the old object. This function returns the
|
680
|
+
# deleting the old object. This function returns the
|
457
681
|
# new object once this is done.
|
458
682
|
#
|
459
683
|
# bucket = s3.buckets['old-bucket']
|
@@ -464,7 +688,7 @@ module AWS
|
|
464
688
|
#
|
465
689
|
# old_obj.key #=> 'old-key'
|
466
690
|
# old_obj.exists? #=> false
|
467
|
-
#
|
691
|
+
#
|
468
692
|
# new_obj.key #=> 'new-key'
|
469
693
|
# new_obj.exists? #=> true
|
470
694
|
#
|
@@ -473,9 +697,9 @@ module AWS
|
|
473
697
|
#
|
474
698
|
# obj = s3.buckets['old-bucket'].objects['old-key']
|
475
699
|
# obj.move_to('new-key', :bucket_name => 'new_bucket')
|
476
|
-
#
|
700
|
+
#
|
477
701
|
# If the copy succeeds, but the then the delete fails, an error
|
478
|
-
# will be raised.
|
702
|
+
# will be raised.
|
479
703
|
#
|
480
704
|
# @param [String] target The key to move this object to.
|
481
705
|
#
|
@@ -492,10 +716,10 @@ module AWS
|
|
492
716
|
end
|
493
717
|
alias_method :rename_to, :move_to
|
494
718
|
|
495
|
-
# Copies data from one S3 object to another.
|
719
|
+
# Copies data from one S3 object to another.
|
496
720
|
#
|
497
721
|
# S3 handles the copy so the clients does not need to fetch the data
|
498
|
-
# and upload it again. You can also change the storage class and
|
722
|
+
# and upload it again. You can also change the storage class and
|
499
723
|
# metadata of the object when copying.
|
500
724
|
#
|
501
725
|
# @note This operation does not copy the ACL, storage class
|
@@ -548,6 +772,10 @@ module AWS
|
|
548
772
|
# option in the current configuration; for more information,
|
549
773
|
# see {AWS.config}.
|
550
774
|
#
|
775
|
+
# @option options [Boolean] :client_side_encrypted (false) Set to true
|
776
|
+
# when the object being copied was client-side encrypted. This
|
777
|
+
# is important so the encryption metadata will be copied.
|
778
|
+
#
|
551
779
|
# @option options :cache_control [String] Can be used to specify
|
552
780
|
# caching behavior. See
|
553
781
|
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
|
@@ -558,13 +786,13 @@ module AWS
|
|
558
786
|
copy_opts = { :bucket_name => bucket.name, :key => key }
|
559
787
|
|
560
788
|
copy_opts[:copy_source] = case source
|
561
|
-
when S3Object
|
789
|
+
when S3Object
|
562
790
|
"#{source.bucket.name}/#{source.key}"
|
563
|
-
when ObjectVersion
|
791
|
+
when ObjectVersion
|
564
792
|
copy_opts[:version_id] = source.version_id
|
565
793
|
"#{source.object.bucket.name}/#{source.object.key}"
|
566
794
|
else
|
567
|
-
case
|
795
|
+
case
|
568
796
|
when options[:bucket] then "#{options[:bucket].name}/#{source}"
|
569
797
|
when options[:bucket_name] then "#{options[:bucket_name]}/#{source}"
|
570
798
|
else "#{self.bucket.name}/#{source}"
|
@@ -573,9 +801,12 @@ module AWS
|
|
573
801
|
|
574
802
|
copy_opts[:metadata_directive] = 'COPY'
|
575
803
|
|
576
|
-
|
577
|
-
|
578
|
-
|
804
|
+
# Saves client-side encryption headers and copies the instruction file
|
805
|
+
copy_cse_materials(source, options) do |cse_materials|
|
806
|
+
if options[:metadata]
|
807
|
+
copy_opts[:metadata] = options[:metadata].merge(cse_materials)
|
808
|
+
copy_opts[:metadata_directive] = 'REPLACE'
|
809
|
+
end
|
579
810
|
end
|
580
811
|
|
581
812
|
if options[:content_type]
|
@@ -588,9 +819,9 @@ module AWS
|
|
588
819
|
copy_opts[:server_side_encryption] =
|
589
820
|
options[:server_side_encryption] if
|
590
821
|
options.key?(:server_side_encryption)
|
591
|
-
copy_opts[:cache_control] = options[:cache_control] if
|
822
|
+
copy_opts[:cache_control] = options[:cache_control] if
|
592
823
|
options[:cache_control]
|
593
|
-
|
824
|
+
add_sse_options(copy_opts)
|
594
825
|
|
595
826
|
if options[:reduced_redundancy]
|
596
827
|
copy_opts[:storage_class] = 'REDUCED_REDUNDANCY'
|
@@ -607,7 +838,7 @@ module AWS
|
|
607
838
|
# Copies data from the current object to another object in S3.
|
608
839
|
#
|
609
840
|
# S3 handles the copy so the client does not need to fetch the data
|
610
|
-
# and upload it again. You can also change the storage class and
|
841
|
+
# and upload it again. You can also change the storage class and
|
611
842
|
# metadata of the object when copying.
|
612
843
|
#
|
613
844
|
# @note This operation does not copy the ACL, storage class
|
@@ -655,6 +886,11 @@ module AWS
|
|
655
886
|
# option in the current configuration; for more information,
|
656
887
|
# see {AWS.config}.
|
657
888
|
#
|
889
|
+
# @option options [Boolean] :client_side_encrypted (false) When +true+,
|
890
|
+
# the client-side encryption materials will be copied. Without this
|
891
|
+
# option, the key and iv are not guaranteed to be transferred to
|
892
|
+
# the new object.
|
893
|
+
#
|
658
894
|
# @return [S3Object] Returns the copy (target) object.
|
659
895
|
#
|
660
896
|
def copy_to target, options = {}
|
@@ -663,7 +899,7 @@ module AWS
|
|
663
899
|
|
664
900
|
bucket = case
|
665
901
|
when options[:bucket] then options[:bucket]
|
666
|
-
when options[:bucket_name]
|
902
|
+
when options[:bucket_name]
|
667
903
|
Bucket.new(options[:bucket_name], :config => config)
|
668
904
|
else self.bucket
|
669
905
|
end
|
@@ -677,17 +913,37 @@ module AWS
|
|
677
913
|
|
678
914
|
target.copy_from(self, copy_opts)
|
679
915
|
target
|
680
|
-
|
916
|
+
|
681
917
|
end
|
682
918
|
|
683
|
-
# Fetches the object data from S3.
|
919
|
+
# Fetches the object data from S3. If you pass a block to this
|
920
|
+
# method, the data will be yielded to the block in chunks as it
|
921
|
+
# is read off the HTTP response.
|
922
|
+
#
|
923
|
+
# === Read an object from S3 in chunks
|
924
|
+
#
|
925
|
+
# When downloading large objects it is recommended to pass a block
|
926
|
+
# to #read. Data will be yielded to the block as it is read off
|
927
|
+
# the HTTP response.
|
928
|
+
#
|
929
|
+
# # read an object from S3 to a file
|
930
|
+
# File.open('output.txt', 'w') do |file|
|
931
|
+
# bucket.objects['key'].read do |chunk|
|
932
|
+
# file.write(chunk)
|
933
|
+
# end
|
934
|
+
# end
|
684
935
|
#
|
685
|
-
#
|
686
|
-
#
|
687
|
-
#
|
688
|
-
#
|
936
|
+
# === Reading an object without a block
|
937
|
+
#
|
938
|
+
# When you omit the block argument to #read, then the entire
|
939
|
+
# HTTP response and read and the object data is loaded into
|
940
|
+
# memory.
|
941
|
+
#
|
942
|
+
# bucket.objects['key'].read
|
943
|
+
# #=> 'object-contents-here'
|
689
944
|
#
|
690
945
|
# @param [Hash] options
|
946
|
+
#
|
691
947
|
# @option options [String] :version_id Reads data from a
|
692
948
|
# specific version of this object.
|
693
949
|
#
|
@@ -710,10 +966,41 @@ module AWS
|
|
710
966
|
#
|
711
967
|
# @option options [Range] :range A byte range to read data from
|
712
968
|
#
|
713
|
-
|
969
|
+
# @option options [OpenSSL::PKey::RSA, String] :encryption_key
|
970
|
+
# (nil) If this option is set, the object will be decrypted using
|
971
|
+
# envelope encryption. The valid values are OpenSSL asymmetric keys
|
972
|
+
# +OpenSSL::Pkey::RSA+ or strings representing symmetric keys
|
973
|
+
# of an AES-128/192/256-ECB cipher as a +String+.
|
974
|
+
# This value defaults to the value in +s3_encryption_key+;
|
975
|
+
# for more information, see {AWS.config}.
|
976
|
+
#
|
977
|
+
# Symmetric Keys:
|
978
|
+
#
|
979
|
+
# cipher = OpenSSL::Cipher.new('AES-256-ECB')
|
980
|
+
# key = cipher.random_key
|
981
|
+
#
|
982
|
+
# Asymmetric keys can also be generated as so:
|
983
|
+
# key = OpenSSL::PKey::RSA.new(KEY_SIZE)
|
984
|
+
#
|
985
|
+
# @option options [Symbol] :encryption_materials_location (:metadata)
|
986
|
+
# Set this to +:instruction_file+ if the encryption materials
|
987
|
+
# are not stored in the object metadata
|
988
|
+
#
|
989
|
+
# @note +:range+ option cannot be used with client-side encryption
|
990
|
+
#
|
991
|
+
# @note All decryption reads incur at least an extra HEAD operation.
|
992
|
+
#
|
993
|
+
def read options = {}, &read_block
|
994
|
+
|
714
995
|
options[:bucket_name] = bucket.name
|
715
996
|
options[:key] = key
|
716
|
-
|
997
|
+
|
998
|
+
if should_decrypt?(options)
|
999
|
+
get_encrypted_object(options, &read_block)
|
1000
|
+
else
|
1001
|
+
get_object(options, &read_block)
|
1002
|
+
end
|
1003
|
+
|
717
1004
|
end
|
718
1005
|
|
719
1006
|
# @private
|
@@ -792,7 +1079,8 @@ module AWS
|
|
792
1079
|
#
|
793
1080
|
# @example Override response headers for reading an object
|
794
1081
|
# object = bucket.objects.myobject
|
795
|
-
# url = object.url_for(:read,
|
1082
|
+
# url = object.url_for(:read,
|
1083
|
+
# :response_content_type => "application/json")
|
796
1084
|
#
|
797
1085
|
# @example Generate a url that expires in 10 minutes
|
798
1086
|
# bucket.objects.myobject.url_for(:read, :expires => 10*60)
|
@@ -846,11 +1134,13 @@ module AWS
|
|
846
1134
|
|
847
1135
|
method = http_method(method)
|
848
1136
|
expires = expiration_timestamp(options[:expires])
|
849
|
-
req.add_param("AWSAccessKeyId",
|
1137
|
+
req.add_param("AWSAccessKeyId",
|
1138
|
+
config.credential_provider.access_key_id)
|
850
1139
|
req.add_param("versionId", options[:version_id]) if options[:version_id]
|
851
1140
|
req.add_param("Signature", signature(method, expires, req))
|
852
1141
|
req.add_param("Expires", expires)
|
853
|
-
req.add_param("x-amz-security-token",
|
1142
|
+
req.add_param("x-amz-security-token",
|
1143
|
+
config.credential_provider.session_token) if
|
854
1144
|
config.credential_provider.session_token
|
855
1145
|
|
856
1146
|
build_uri(options[:secure] != false, req)
|
@@ -898,8 +1188,74 @@ module AWS
|
|
898
1188
|
value
|
899
1189
|
end
|
900
1190
|
|
901
|
-
|
902
|
-
|
1191
|
+
protected
|
1192
|
+
|
1193
|
+
# @return [Boolean]
|
1194
|
+
def should_decrypt? options
|
1195
|
+
options[:encryption_key]
|
1196
|
+
end
|
1197
|
+
|
1198
|
+
# A small wrapper around client#get_object
|
1199
|
+
def get_object options, &read_block
|
1200
|
+
client.get_object(options, &read_block).data[:data]
|
1201
|
+
end
|
1202
|
+
|
1203
|
+
# A wrapper around get_object that decrypts
|
1204
|
+
def get_encrypted_object options, &read_block
|
1205
|
+
decryption_cipher(options) do |cipher|
|
1206
|
+
if block_given?
|
1207
|
+
get_object(options) do |chunk|
|
1208
|
+
yield(cipher.update(chunk))
|
1209
|
+
end
|
1210
|
+
yield(cipher.final)
|
1211
|
+
else
|
1212
|
+
cipher.update(get_object(options)) + cipher.final
|
1213
|
+
end
|
1214
|
+
end
|
1215
|
+
end
|
1216
|
+
|
1217
|
+
# @return [Boolean] Returns +true+ if the :data option is large or
|
1218
|
+
# guessed to be larger than a configured threshold.
|
1219
|
+
def use_multipart? options
|
1220
|
+
estimated_content_length(options) > multipart_threshold(options) and
|
1221
|
+
!options[:single_request]
|
1222
|
+
end
|
1223
|
+
|
1224
|
+
# @return [Integer] Returns the number of bytes where a multipart
|
1225
|
+
# upload is used instead of #put_object.
|
1226
|
+
def multipart_threshold options
|
1227
|
+
threshold = options[:multipart_threshold] ||
|
1228
|
+
config.s3_multipart_threshold
|
1229
|
+
end
|
1230
|
+
|
1231
|
+
# @return [Integer] Returns the size of each multipart chunk.
|
1232
|
+
def compute_part_size options
|
1233
|
+
|
1234
|
+
max_parts = options[:multipart_max_parts] ||
|
1235
|
+
config.s3_multipart_max_parts
|
1236
|
+
|
1237
|
+
min_size = options[:multipart_min_part_size] ||
|
1238
|
+
config.s3_multipart_min_part_size
|
1239
|
+
|
1240
|
+
estimated_size = estimated_content_length(options)
|
1241
|
+
|
1242
|
+
[(estimated_size.to_f / max_parts).ceil, min_size].max.to_i
|
1243
|
+
|
1244
|
+
end
|
1245
|
+
|
1246
|
+
# @return [Integer] Returns the size of the data or an estimated
|
1247
|
+
# size as provided by the user (useful for IO streams).
|
1248
|
+
def estimated_content_length options
|
1249
|
+
estimate = options[:content_length] ||
|
1250
|
+
options[:estimated_content_length]
|
1251
|
+
unless estimate
|
1252
|
+
msg = "unknown content length, must set :content_length or " +
|
1253
|
+
":estimated_content_length"
|
1254
|
+
raise ArgumentError, msg
|
1255
|
+
end
|
1256
|
+
estimate
|
1257
|
+
end
|
1258
|
+
|
903
1259
|
def build_uri(secure, request)
|
904
1260
|
uri_class = secure ? URI::HTTPS : URI::HTTP
|
905
1261
|
uri_class.build(:host => request.host,
|
@@ -907,8 +1263,6 @@ module AWS
|
|
907
1263
|
:query => request.querystring)
|
908
1264
|
end
|
909
1265
|
|
910
|
-
# @private
|
911
|
-
private
|
912
1266
|
def signature(method, expires, request)
|
913
1267
|
|
914
1268
|
parts = []
|
@@ -916,8 +1270,10 @@ module AWS
|
|
916
1270
|
parts << ""
|
917
1271
|
parts << ""
|
918
1272
|
parts << expires
|
919
|
-
|
920
|
-
|
1273
|
+
if config.credential_provider.session_token
|
1274
|
+
parts << "x-amz-security-token:"
|
1275
|
+
parts << "#{config.credential_provider.session_token}"
|
1276
|
+
end
|
921
1277
|
parts << request.canonicalized_resource
|
922
1278
|
|
923
1279
|
string_to_sign = parts.join("\n")
|
@@ -927,8 +1283,6 @@ module AWS
|
|
927
1283
|
|
928
1284
|
end
|
929
1285
|
|
930
|
-
# @private
|
931
|
-
private
|
932
1286
|
def expiration_timestamp(input)
|
933
1287
|
case input
|
934
1288
|
when Time
|
@@ -944,8 +1298,6 @@ module AWS
|
|
944
1298
|
end
|
945
1299
|
end
|
946
1300
|
|
947
|
-
# @private
|
948
|
-
private
|
949
1301
|
def http_method(input)
|
950
1302
|
symbol = case input
|
951
1303
|
when :read then :get
|
@@ -956,8 +1308,6 @@ module AWS
|
|
956
1308
|
symbol.to_s.upcase
|
957
1309
|
end
|
958
1310
|
|
959
|
-
# @private
|
960
|
-
private
|
961
1311
|
def request_for_signing(options)
|
962
1312
|
req = Request.new
|
963
1313
|
|
@@ -973,76 +1323,286 @@ module AWS
|
|
973
1323
|
req
|
974
1324
|
end
|
975
1325
|
|
976
|
-
|
977
|
-
|
978
|
-
|
979
|
-
|
1326
|
+
def add_sse_options(options)
|
1327
|
+
options[:server_side_encryption] =
|
1328
|
+
config.s3_server_side_encryption unless
|
1329
|
+
options.key?(:server_side_encryption)
|
1330
|
+
options.delete(:server_side_encryption) if
|
1331
|
+
options[:server_side_encryption] == nil
|
1332
|
+
end
|
980
1333
|
|
981
|
-
|
1334
|
+
# Adds client-side encryption metadata headers and encrypts key
|
1335
|
+
def add_cse_options(options)
|
1336
|
+
encryption_key_for(options) do |encryption_key|
|
982
1337
|
|
983
|
-
|
984
|
-
|
1338
|
+
check_encryption_materials(:encrypt, encryption_key)
|
1339
|
+
cipher = get_aes_cipher(:encrypt, :CBC)
|
985
1340
|
|
986
|
-
|
987
|
-
|
1341
|
+
generate_aes_key(cipher) do |envelope_key, envelope_iv|
|
1342
|
+
envelope_key, envelope_iv =
|
1343
|
+
encode_envelope_key(encryption_key, envelope_key, envelope_iv)
|
988
1344
|
|
989
|
-
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
data_options = { :data => options_or_data[:data] || "" }
|
1345
|
+
build_cse_metadata(options,
|
1346
|
+
envelope_key,
|
1347
|
+
envelope_iv) do |headers, encryption_materials|
|
1348
|
+
store_encryption_materials(options, headers, encryption_materials)
|
1349
|
+
end
|
995
1350
|
end
|
996
1351
|
|
997
|
-
|
1352
|
+
# Wrap current stream in encryption
|
1353
|
+
options[:data] = CipherIO.new(cipher,
|
1354
|
+
options[:data],
|
1355
|
+
options[:content_length])
|
1356
|
+
|
1357
|
+
# Update content_length
|
1358
|
+
options[:content_length] =
|
1359
|
+
get_encrypted_size(options[:content_length]) if
|
1360
|
+
options[:content_length]
|
1361
|
+
|
1362
|
+
end
|
1363
|
+
remove_cse_options(options)
|
1364
|
+
end
|
1365
|
+
|
1366
|
+
# @yield [String, String] Yields an encrypted encoded key and iv pair
|
1367
|
+
def encode_envelope_key encryption_key, envelope_key, envelope_iv, &block
|
1368
|
+
envelope_key = encrypt(envelope_key, encryption_key)
|
1369
|
+
[encode64(envelope_key), encode64(envelope_iv)]
|
1370
|
+
end
|
1371
|
+
|
1372
|
+
# @yield [Hash, Hash] Yields headers and encryption materials that are
|
1373
|
+
# to be stored in the metadata and/or instruction file
|
1374
|
+
def build_cse_metadata options, enc_envelope_key, enc_envelope_iv, &block
|
1375
|
+
# Ensure metadata exists
|
1376
|
+
options[:metadata] = {} unless options[:metadata]
|
1377
|
+
|
1378
|
+
encryption_materials = {'x-amz-key' => enc_envelope_key,
|
1379
|
+
'x-amz-iv' => enc_envelope_iv,
|
1380
|
+
'x-amz-matdesc' => "{}"}
|
1381
|
+
orig_headers = {}
|
1382
|
+
|
1383
|
+
# Save the unencrypted content length
|
1384
|
+
if options[:content_length]
|
1385
|
+
orig_headers['x-amz-unencrypted-content-length'] =
|
1386
|
+
options[:content_length]
|
1387
|
+
end
|
1388
|
+
|
1389
|
+
# Save the unencrypted content MD5
|
1390
|
+
if options[:content_md5]
|
1391
|
+
orig_headers['x-amz-unencrypted-content-md5'] =
|
1392
|
+
options[:content_md5]
|
1393
|
+
options.delete(:content_md5)
|
1394
|
+
end
|
1395
|
+
|
1396
|
+
options[:metadata].merge!(orig_headers)
|
1397
|
+
|
1398
|
+
yield([orig_headers, encryption_materials])
|
1399
|
+
end
|
1400
|
+
|
1401
|
+
# Stores the headers and encryption materials needed to decrypt the data
|
1402
|
+
# and to know unencrypted information about the object
|
1403
|
+
def store_encryption_materials options, orig_headers, encryption_materials
|
1404
|
+
# Get the storage location
|
1405
|
+
cse_location = options[:encryption_materials_location] ||
|
1406
|
+
config.s3_encryption_materials_location
|
1407
|
+
|
1408
|
+
# Encryption type specific metadata
|
1409
|
+
case cse_location
|
1410
|
+
when :metadata
|
1411
|
+
options[:metadata].merge!(encryption_materials)
|
1412
|
+
when :instruction_file
|
1413
|
+
json_string = JSON.generate(encryption_materials)
|
1414
|
+
inst_headers = {'x-amz-crypto-instr-file' => ""}.merge(orig_headers)
|
1415
|
+
bucket.objects["#{key}.instruction"].write(json_string,
|
1416
|
+
:metadata => inst_headers)
|
998
1417
|
else
|
999
|
-
|
1418
|
+
msg = "invalid :encryption_materials_location, expected "
|
1419
|
+
msg << ":metadata or :instruction_file, got: #{cse_location.inspect}"
|
1420
|
+
raise ArgumentError, msg
|
1000
1421
|
end
|
1422
|
+
nil
|
1001
1423
|
end
|
1002
1424
|
|
1003
|
-
|
1004
|
-
def
|
1005
|
-
options
|
1006
|
-
|
1007
|
-
|
1008
|
-
options.delete(:server_side_encryption) if
|
1009
|
-
options[:server_side_encryption] == nil
|
1425
|
+
# Removes any extra headers client-side encryption uses.
|
1426
|
+
def remove_cse_options options
|
1427
|
+
options.delete(:encryption_key)
|
1428
|
+
options.delete(:encryption_materials_location)
|
1429
|
+
options.delete(:encryption_matdesc)
|
1010
1430
|
end
|
1011
1431
|
|
1012
|
-
#
|
1013
|
-
|
1014
|
-
def
|
1015
|
-
|
1016
|
-
|
1432
|
+
# Yields a decryption cipher for the given client-side encryption key
|
1433
|
+
# or raises an error.
|
1434
|
+
def decryption_cipher options, &block
|
1435
|
+
encryption_key_for(options) do |encryption_key|
|
1436
|
+
|
1437
|
+
check_encryption_materials(:decrypt, encryption_key)
|
1438
|
+
|
1439
|
+
location = options[:encryption_materials_location] ||
|
1440
|
+
config.s3_encryption_materials_location
|
1017
1441
|
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1442
|
+
cipher =
|
1443
|
+
decryption_materials(location) do |envelope_key, envelope_iv|
|
1444
|
+
envelope_key, envelope_iv =
|
1445
|
+
decode_envelope_key(envelope_key, envelope_iv, options)
|
1446
|
+
get_aes_cipher(:decrypt, :CBC, envelope_key, envelope_iv)
|
1447
|
+
end
|
1448
|
+
|
1449
|
+
remove_cse_options(options)
|
1450
|
+
|
1451
|
+
yield(cipher)
|
1452
|
+
|
1453
|
+
end
|
1021
1454
|
end
|
1022
1455
|
|
1023
|
-
#
|
1024
|
-
|
1025
|
-
|
1026
|
-
|
1027
|
-
|
1028
|
-
|
1029
|
-
|
1030
|
-
|
1456
|
+
# Decodes the envelope key for decryption
|
1457
|
+
def decode_envelope_key envelope_key, envelope_iv, options
|
1458
|
+
decrypted_key =
|
1459
|
+
begin
|
1460
|
+
decrypt(decode64(envelope_key), options[:encryption_key])
|
1461
|
+
rescue RuntimeError
|
1462
|
+
msg = "Master key used to decrypt data key is not correct."
|
1463
|
+
raise AWS::S3::Errors::IncorrectClientSideEncryptionKey, msg
|
1031
1464
|
end
|
1465
|
+
|
1466
|
+
[decrypted_key, decode64(envelope_iv)]
|
1032
1467
|
end
|
1033
1468
|
|
1034
|
-
# @private
|
1035
|
-
private
|
1036
|
-
def optimal_part_size(total_size, options)
|
1037
|
-
maximum_parts = config.s3_multipart_max_parts
|
1038
|
-
min_size = options[:multipart_min_part_size] ||
|
1039
|
-
config.s3_multipart_min_part_size
|
1040
1469
|
|
1041
|
-
|
1042
|
-
|
1470
|
+
# @yield [String, String, String] Yields encryption materials for
|
1471
|
+
# decryption
|
1472
|
+
def decryption_materials location, &block
|
1473
|
+
|
1474
|
+
materials = case location
|
1475
|
+
when :metadata then get_metadata_materials
|
1476
|
+
when :instruction_file then get_inst_file_materials
|
1477
|
+
else
|
1478
|
+
msg = "invalid :encryption_materials_location option, expected "
|
1479
|
+
msg << ":metadata or :instruction_file, got: #{location.inspect}"
|
1480
|
+
raise ArgumentError, msg
|
1481
|
+
end
|
1482
|
+
|
1483
|
+
envelope_key, envelope_iv = materials
|
1484
|
+
|
1485
|
+
unless envelope_key and envelope_iv
|
1486
|
+
raise 'no encryption materials found, unable to decrypt'
|
1487
|
+
end
|
1488
|
+
|
1489
|
+
yield(envelope_key, envelope_iv)
|
1490
|
+
|
1043
1491
|
end
|
1044
1492
|
|
1045
|
-
|
1493
|
+
# @return [String, String, String] Returns the data key, envelope_iv, and the
|
1494
|
+
# material description for decryption from the metadata.
|
1495
|
+
def get_metadata_materials
|
1496
|
+
metadata.to_h.values_at(*%w(x-amz-key x-amz-iv))
|
1497
|
+
end
|
1498
|
+
|
1499
|
+
# @return [String, String, String] Returns the data key, envelope_iv, and the
|
1500
|
+
# material description for decryption from the instruction file.
|
1501
|
+
def get_inst_file_materials
|
1502
|
+
obj = bucket.objects["#{key}.instruction"]
|
1503
|
+
JSON.parse(obj.read).values_at(*%w(x-amz-key x-amz-iv))
|
1504
|
+
end
|
1505
|
+
|
1506
|
+
# @yield [Hash] Yields the metadata to be saved for client-side encryption
|
1507
|
+
def copy_cse_materials source, options
|
1508
|
+
cse_materials = {}
|
1509
|
+
if options[:client_side_encrypted]
|
1510
|
+
meta = source.metadata.to_h
|
1511
|
+
cse_materials['x-amz-key'] = meta['x-amz-key'] if meta['x-amz-key']
|
1512
|
+
cse_materials['x-amz-iv'] = meta['x-amz-iv'] if meta['x-amz-iv']
|
1513
|
+
cse_materials['x-amz-matdesc'] = meta['x-amz-matdesc'] if
|
1514
|
+
meta['x-amz-matdesc']
|
1515
|
+
cse_materials['x-amz-unencrypted-content-length'] =
|
1516
|
+
meta['x-amz-unencrypted-content-length'] if
|
1517
|
+
meta['x-amz-unencrypted-content-length']
|
1518
|
+
cse_materials['x-amz-unencrypted-content-md5'] =
|
1519
|
+
meta['x-amz-unencrypted-content-md5'] if
|
1520
|
+
meta['x-amz-unencrypted-content-md5']
|
1521
|
+
|
1522
|
+
# Handling instruction file
|
1523
|
+
unless cse_materials['x-amz-key'] and
|
1524
|
+
cse_materials['x-amz-iv'] and
|
1525
|
+
cse_materials['x-amz-matdesc']
|
1526
|
+
source_inst = "#{source.key}.instruction"
|
1527
|
+
dest_inst = "#{key}.instruction"
|
1528
|
+
self.bucket.objects[dest_inst].copy_from(
|
1529
|
+
source.bucket.objects[source_inst])
|
1530
|
+
end
|
1531
|
+
end
|
1532
|
+
yield(cse_materials)
|
1533
|
+
end
|
1534
|
+
|
1535
|
+
# Removes unwanted options that should not be passed to the client.
|
1536
|
+
def clean_up_options(options)
|
1537
|
+
options.delete(:estimated_content_length)
|
1538
|
+
options.delete(:single_request)
|
1539
|
+
options.delete(:multipart_threshold)
|
1540
|
+
end
|
1541
|
+
|
1542
|
+
# Performs a write using a multipart upload
|
1543
|
+
def write_with_multipart options
|
1544
|
+
part_size = compute_part_size(options)
|
1545
|
+
clean_up_options(options)
|
1546
|
+
options.delete(:content_length)
|
1547
|
+
|
1548
|
+
multipart_upload(options) do |upload|
|
1549
|
+
upload.add_part(options[:data].read(part_size)) until
|
1550
|
+
options[:data].eof?
|
1551
|
+
end
|
1552
|
+
end
|
1553
|
+
|
1554
|
+
# Performs a write using a single request
|
1555
|
+
def write_with_put_object options
|
1556
|
+
|
1557
|
+
# its possible we don't know the content length of the data
|
1558
|
+
# option, but the :estimated_content_length was sufficiently
|
1559
|
+
# small that we will read the entire stream into memory
|
1560
|
+
# so we can tell s3 the content length (this is required).
|
1561
|
+
unless options[:content_length]
|
1562
|
+
data = StringIO.new
|
1563
|
+
|
1564
|
+
while (chunk = options[:data].read(4 * 1024))
|
1565
|
+
data << chunk
|
1566
|
+
end
|
1046
1567
|
|
1568
|
+
options[:content_length] = data.size
|
1569
|
+
data.rewind
|
1570
|
+
options[:data] = data
|
1571
|
+
end
|
1572
|
+
|
1573
|
+
clean_up_options(options)
|
1574
|
+
|
1575
|
+
options[:bucket_name] = bucket.name
|
1576
|
+
options[:key] = key
|
1577
|
+
|
1578
|
+
resp = client.put_object(options)
|
1579
|
+
|
1580
|
+
resp.data[:version_id] ?
|
1581
|
+
ObjectVersion.new(self, resp.data[:version_id]) : self
|
1582
|
+
end
|
1583
|
+
|
1584
|
+
def encryption_key_for options, &block
|
1585
|
+
if key = options[:encryption_key] || config.s3_encryption_key
|
1586
|
+
yield(key)
|
1587
|
+
end
|
1588
|
+
end
|
1589
|
+
|
1590
|
+
def add_storage_class_option options
|
1591
|
+
if options[:reduced_redundancy] == true
|
1592
|
+
options[:storage_class] = 'REDUCED_REDUNDANCY'
|
1593
|
+
end
|
1594
|
+
end
|
1595
|
+
|
1596
|
+
# @return [String] Encodes a +String+ in base 64 regardless of version of
|
1597
|
+
# Ruby for http headers (removes newlines).
|
1598
|
+
def encode64 input
|
1599
|
+
Base64.encode64(input).split("\n") * ""
|
1600
|
+
end
|
1601
|
+
|
1602
|
+
# @return [String] Decodes a +String+ in base 64.
|
1603
|
+
def decode64 input
|
1604
|
+
Base64.decode64(input)
|
1605
|
+
end
|
1606
|
+
end
|
1047
1607
|
end
|
1048
1608
|
end
|