aws-sdk-resources 2.3.23 → 3.69.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/bin/aws-v3.rb +201 -0
- data/lib/aws-sdk-resources.rb +223 -87
- metadata +3073 -71
- data/lib/aws-sdk-resources/batch.rb +0 -143
- data/lib/aws-sdk-resources/builder.rb +0 -85
- data/lib/aws-sdk-resources/builder_sources.rb +0 -105
- data/lib/aws-sdk-resources/collection.rb +0 -107
- data/lib/aws-sdk-resources/definition.rb +0 -331
- data/lib/aws-sdk-resources/documenter.rb +0 -70
- data/lib/aws-sdk-resources/documenter/base_operation_documenter.rb +0 -279
- data/lib/aws-sdk-resources/documenter/data_operation_documenter.rb +0 -25
- data/lib/aws-sdk-resources/documenter/has_many_operation_documenter.rb +0 -67
- data/lib/aws-sdk-resources/documenter/has_operation_documenter.rb +0 -66
- data/lib/aws-sdk-resources/documenter/operation_documenter.rb +0 -20
- data/lib/aws-sdk-resources/documenter/resource_operation_documenter.rb +0 -53
- data/lib/aws-sdk-resources/documenter/waiter_operation_documenter.rb +0 -77
- data/lib/aws-sdk-resources/errors.rb +0 -15
- data/lib/aws-sdk-resources/operation_methods.rb +0 -83
- data/lib/aws-sdk-resources/operations.rb +0 -280
- data/lib/aws-sdk-resources/options.rb +0 -17
- data/lib/aws-sdk-resources/request.rb +0 -39
- data/lib/aws-sdk-resources/request_params.rb +0 -140
- data/lib/aws-sdk-resources/resource.rb +0 -243
- data/lib/aws-sdk-resources/services/ec2.rb +0 -21
- data/lib/aws-sdk-resources/services/ec2/instance.rb +0 -29
- data/lib/aws-sdk-resources/services/iam.rb +0 -19
- data/lib/aws-sdk-resources/services/s3.rb +0 -19
- data/lib/aws-sdk-resources/services/s3/bucket.rb +0 -127
- data/lib/aws-sdk-resources/services/s3/encryption.rb +0 -21
- data/lib/aws-sdk-resources/services/s3/encryption/client.rb +0 -369
- data/lib/aws-sdk-resources/services/s3/encryption/decrypt_handler.rb +0 -174
- data/lib/aws-sdk-resources/services/s3/encryption/default_cipher_provider.rb +0 -63
- data/lib/aws-sdk-resources/services/s3/encryption/default_key_provider.rb +0 -38
- data/lib/aws-sdk-resources/services/s3/encryption/encrypt_handler.rb +0 -50
- data/lib/aws-sdk-resources/services/s3/encryption/errors.rb +0 -13
- data/lib/aws-sdk-resources/services/s3/encryption/io_auth_decrypter.rb +0 -50
- data/lib/aws-sdk-resources/services/s3/encryption/io_decrypter.rb +0 -29
- data/lib/aws-sdk-resources/services/s3/encryption/io_encrypter.rb +0 -69
- data/lib/aws-sdk-resources/services/s3/encryption/key_provider.rb +0 -29
- data/lib/aws-sdk-resources/services/s3/encryption/kms_cipher_provider.rb +0 -71
- data/lib/aws-sdk-resources/services/s3/encryption/materials.rb +0 -58
- data/lib/aws-sdk-resources/services/s3/encryption/utils.rb +0 -79
- data/lib/aws-sdk-resources/services/s3/file_part.rb +0 -75
- data/lib/aws-sdk-resources/services/s3/file_uploader.rb +0 -58
- data/lib/aws-sdk-resources/services/s3/multipart_file_uploader.rb +0 -187
- data/lib/aws-sdk-resources/services/s3/multipart_upload.rb +0 -42
- data/lib/aws-sdk-resources/services/s3/multipart_upload_error.rb +0 -16
- data/lib/aws-sdk-resources/services/s3/object.rb +0 -257
- data/lib/aws-sdk-resources/services/s3/object_copier.rb +0 -99
- data/lib/aws-sdk-resources/services/s3/object_multipart_copier.rb +0 -179
- data/lib/aws-sdk-resources/services/s3/object_summary.rb +0 -65
- data/lib/aws-sdk-resources/services/s3/presigned_post.rb +0 -647
- data/lib/aws-sdk-resources/services/sns.rb +0 -7
- data/lib/aws-sdk-resources/services/sns/message_verifier.rb +0 -157
- data/lib/aws-sdk-resources/services/sqs.rb +0 -7
- data/lib/aws-sdk-resources/services/sqs/queue_poller.rb +0 -521
- data/lib/aws-sdk-resources/source.rb +0 -39
@@ -1,29 +0,0 @@
|
|
1
|
-
module Aws
|
2
|
-
module S3
|
3
|
-
module Encryption
|
4
|
-
|
5
|
-
# This module defines the interface required for a {Client#key_provider}.
|
6
|
-
# A key provider is any object that:
|
7
|
-
#
|
8
|
-
# * Responds to {#encryption_materials} with an {Materials} object.
|
9
|
-
#
|
10
|
-
# * Responds to {#key_for}, receiving a JSON document String,
|
11
|
-
# returning an encryption key. The returned encryption key
|
12
|
-
# must be one of:
|
13
|
-
#
|
14
|
-
# * `OpenSSL::PKey::RSA` - for asymmetric encryption
|
15
|
-
# * `String` - 32, 24, or 16 bytes long, for symmetric encryption
|
16
|
-
#
|
17
|
-
module KeyProvider
|
18
|
-
|
19
|
-
# @return [Materials]
|
20
|
-
def encryption_materials; end
|
21
|
-
|
22
|
-
# @param [String<JSON>] materials_description
|
23
|
-
# @return [OpenSSL::PKey::RSA, String] encryption_key
|
24
|
-
def key_for(materials_description); end
|
25
|
-
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
@@ -1,71 +0,0 @@
|
|
1
|
-
require 'base64'
|
2
|
-
|
3
|
-
module Aws
|
4
|
-
module S3
|
5
|
-
module Encryption
|
6
|
-
# @api private
|
7
|
-
class KmsCipherProvider
|
8
|
-
|
9
|
-
def initialize(options = {})
|
10
|
-
@kms_key_id = options[:kms_key_id]
|
11
|
-
@kms_client = options[:kms_client]
|
12
|
-
end
|
13
|
-
|
14
|
-
# @return [Array<Hash,Cipher>] Creates an returns a new encryption
|
15
|
-
# envelope and encryption cipher.
|
16
|
-
def encryption_cipher
|
17
|
-
encryption_context = { "kms_cmk_id" => @kms_key_id }
|
18
|
-
key_data = @kms_client.generate_data_key(
|
19
|
-
key_id: @kms_key_id,
|
20
|
-
encryption_context: encryption_context,
|
21
|
-
key_spec: 'AES_256',
|
22
|
-
)
|
23
|
-
cipher = Utils.aes_encryption_cipher(:CBC)
|
24
|
-
cipher.key = key_data.plaintext
|
25
|
-
envelope = {
|
26
|
-
'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
|
27
|
-
'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
|
28
|
-
'x-amz-cek-alg' => 'AES/CBC/PKCS5Padding',
|
29
|
-
'x-amz-wrap-alg' => 'kms',
|
30
|
-
'x-amz-matdesc' => Json.dump(encryption_context)
|
31
|
-
}
|
32
|
-
[envelope, cipher]
|
33
|
-
end
|
34
|
-
|
35
|
-
# @return [Cipher] Given an encryption envelope, returns a
|
36
|
-
# decryption cipher.
|
37
|
-
def decryption_cipher(envelope)
|
38
|
-
encryption_context = Json.load(envelope['x-amz-matdesc'])
|
39
|
-
key = @kms_client.decrypt(
|
40
|
-
ciphertext_blob: decode64(envelope['x-amz-key-v2']),
|
41
|
-
encryption_context: encryption_context,
|
42
|
-
).plaintext
|
43
|
-
iv = decode64(envelope['x-amz-iv'])
|
44
|
-
block_mode =
|
45
|
-
case envelope['x-amz-cek-alg']
|
46
|
-
when 'AES/CBC/PKCS5Padding'
|
47
|
-
:CBC
|
48
|
-
when 'AES/GCM/NoPadding'
|
49
|
-
:GCM
|
50
|
-
else
|
51
|
-
type = envelope['x-amz-cek-alg'].inspect
|
52
|
-
msg = "unsupported content encrypting key (cek) format: #{type}"
|
53
|
-
raise Errors::DecryptionError, msg
|
54
|
-
end
|
55
|
-
Utils.aes_decryption_cipher(block_mode, key, iv)
|
56
|
-
end
|
57
|
-
|
58
|
-
private
|
59
|
-
|
60
|
-
def encode64(str)
|
61
|
-
Base64.encode64(str).split("\n") * ""
|
62
|
-
end
|
63
|
-
|
64
|
-
def decode64(str)
|
65
|
-
Base64.decode64(str)
|
66
|
-
end
|
67
|
-
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
@@ -1,58 +0,0 @@
|
|
1
|
-
require 'base64'
|
2
|
-
|
3
|
-
module Aws
|
4
|
-
module S3
|
5
|
-
module Encryption
|
6
|
-
class Materials
|
7
|
-
|
8
|
-
# @option options [required, OpenSSL::PKey::RSA, String] :key
|
9
|
-
# The master key to use for encrypting/decrypting all objects.
|
10
|
-
#
|
11
|
-
# @option options [String<JSON>] :description ('{}')
|
12
|
-
# The encryption materials description. This is must be
|
13
|
-
# a JSON document string.
|
14
|
-
#
|
15
|
-
def initialize(options = {})
|
16
|
-
@key = validate_key(options[:key])
|
17
|
-
@description = validate_desc(options[:description])
|
18
|
-
end
|
19
|
-
|
20
|
-
# @return [OpenSSL::PKey::RSA, String]
|
21
|
-
attr_reader :key
|
22
|
-
|
23
|
-
# @return [String<JSON>]
|
24
|
-
attr_reader :description
|
25
|
-
|
26
|
-
private
|
27
|
-
|
28
|
-
def validate_key(key)
|
29
|
-
case key
|
30
|
-
when OpenSSL::PKey::RSA then key
|
31
|
-
when String
|
32
|
-
if [32, 24, 16].include?(key.bytesize)
|
33
|
-
key
|
34
|
-
else
|
35
|
-
msg = "invalid key, symmetric key required to be 16, 24, or "
|
36
|
-
msg << "32 bytes in length, saw length 31"
|
37
|
-
raise ArgumentError, msg
|
38
|
-
end
|
39
|
-
else
|
40
|
-
msg = "invalid encryption key, expected an OpenSSL::PKey::RSA key "
|
41
|
-
msg << "(for asymmetric encryption) or a String (for symmetric "
|
42
|
-
msg << "encryption)."
|
43
|
-
raise ArgumentError, msg
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def validate_desc(description)
|
48
|
-
Json.load(description)
|
49
|
-
description
|
50
|
-
rescue Json::ParseError
|
51
|
-
msg = "expected description to be a valid JSON document string"
|
52
|
-
raise ArgumentError, msg
|
53
|
-
end
|
54
|
-
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
@@ -1,79 +0,0 @@
|
|
1
|
-
require 'openssl'
|
2
|
-
|
3
|
-
module Aws
|
4
|
-
module S3
|
5
|
-
module Encryption
|
6
|
-
# @api private
|
7
|
-
module Utils
|
8
|
-
|
9
|
-
UNSAFE_MSG = "unsafe encryption, data is longer than key length"
|
10
|
-
|
11
|
-
class << self
|
12
|
-
|
13
|
-
def encrypt(key, data)
|
14
|
-
case key
|
15
|
-
when OpenSSL::PKey::RSA # asymmetric encryption
|
16
|
-
warn(UNSAFE_MSG) if key.public_key.n.num_bits < cipher_size(data)
|
17
|
-
key.public_encrypt(data)
|
18
|
-
when String # symmetric encryption
|
19
|
-
warn(UNSAFE_MSG) if cipher_size(key) < cipher_size(data)
|
20
|
-
cipher = aes_encryption_cipher(:ECB, key)
|
21
|
-
cipher.update(data) + cipher.final
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
def decrypt(key, data)
|
26
|
-
begin
|
27
|
-
case key
|
28
|
-
when OpenSSL::PKey::RSA # asymmetric decryption
|
29
|
-
key.private_decrypt(data)
|
30
|
-
when String # symmetric Decryption
|
31
|
-
cipher = aes_cipher(:decrypt, :ECB, key, nil)
|
32
|
-
cipher.update(data) + cipher.final
|
33
|
-
end
|
34
|
-
rescue OpenSSL::Cipher::CipherError
|
35
|
-
msg = 'decryption failed, possible incorrect key'
|
36
|
-
raise Errors::DecryptionError, msg
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
# @param [String] block_mode "CBC" or "ECB"
|
41
|
-
# @param [OpenSSL::PKey::RSA, String, nil] key
|
42
|
-
# @param [String, nil] iv The initialization vector
|
43
|
-
def aes_encryption_cipher(block_mode, key = nil, iv = nil)
|
44
|
-
aes_cipher(:encrypt, block_mode, key, iv)
|
45
|
-
end
|
46
|
-
|
47
|
-
# @param [String] block_mode "CBC" or "ECB"
|
48
|
-
# @param [OpenSSL::PKey::RSA, String, nil] key
|
49
|
-
# @param [String, nil] iv The initialization vector
|
50
|
-
def aes_decryption_cipher(block_mode, key = nil, iv = nil)
|
51
|
-
aes_cipher(:decrypt, block_mode, key, iv)
|
52
|
-
end
|
53
|
-
|
54
|
-
# @param [String] mode "encrypt" or "decrypt"
|
55
|
-
# @param [String] block_mode "CBC" or "ECB"
|
56
|
-
# @param [OpenSSL::PKey::RSA, String, nil] key
|
57
|
-
# @param [String, nil] iv The initialization vector
|
58
|
-
def aes_cipher(mode, block_mode, key, iv)
|
59
|
-
cipher = key ?
|
60
|
-
OpenSSL::Cipher.new("aes-#{cipher_size(key)}-#{block_mode.downcase}") :
|
61
|
-
OpenSSL::Cipher.new("aes-256-#{block_mode.downcase}")
|
62
|
-
cipher.send(mode) # encrypt or decrypt
|
63
|
-
cipher.key = key if key
|
64
|
-
cipher.iv = iv if iv
|
65
|
-
cipher
|
66
|
-
end
|
67
|
-
|
68
|
-
# @param [String] key
|
69
|
-
# @return [Integer]
|
70
|
-
# @raise ArgumentError
|
71
|
-
def cipher_size(key)
|
72
|
-
key.bytesize * 8
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
79
|
-
end
|
@@ -1,75 +0,0 @@
|
|
1
|
-
module Aws
|
2
|
-
module S3
|
3
|
-
|
4
|
-
# A utility class that provides an IO-like interface to a portion of
|
5
|
-
# a file on disk.
|
6
|
-
# @api private
|
7
|
-
class FilePart
|
8
|
-
|
9
|
-
# @option options [required,String,Pathname,File,Tempfile] :source
|
10
|
-
# @option options [required,Integer] :offset The file part will read
|
11
|
-
# starting at this byte offset.
|
12
|
-
# @option options [required,Integer] :size The maximum number of bytes to
|
13
|
-
# read from the `:offset`.
|
14
|
-
def initialize(options = {})
|
15
|
-
@source = options[:source]
|
16
|
-
@first_byte = options[:offset]
|
17
|
-
@last_byte = @first_byte + options[:size]
|
18
|
-
@size = options[:size]
|
19
|
-
@file = nil
|
20
|
-
end
|
21
|
-
|
22
|
-
# @return [String,Pathname,File,Tempfile]
|
23
|
-
attr_reader :source
|
24
|
-
|
25
|
-
# @return [Integer]
|
26
|
-
attr_reader :first_byte
|
27
|
-
|
28
|
-
# @return [Integer]
|
29
|
-
attr_reader :last_byte
|
30
|
-
|
31
|
-
# @return [Integer]
|
32
|
-
attr_reader :size
|
33
|
-
|
34
|
-
def read(bytes = nil, output_buffer = nil)
|
35
|
-
open_file unless @file
|
36
|
-
read_from_file(bytes, output_buffer)
|
37
|
-
end
|
38
|
-
|
39
|
-
def rewind
|
40
|
-
if @file
|
41
|
-
@file.seek(@first_byte)
|
42
|
-
@position = @first_byte
|
43
|
-
end
|
44
|
-
0
|
45
|
-
end
|
46
|
-
|
47
|
-
def close
|
48
|
-
@file.close if @file
|
49
|
-
end
|
50
|
-
|
51
|
-
private
|
52
|
-
|
53
|
-
def open_file
|
54
|
-
@file = File.open(@source, 'rb')
|
55
|
-
rewind
|
56
|
-
end
|
57
|
-
|
58
|
-
def read_from_file(bytes, output_buffer)
|
59
|
-
if bytes
|
60
|
-
data = @file.read([remaining_bytes, bytes].min)
|
61
|
-
data = nil if data == ''
|
62
|
-
else
|
63
|
-
data = @file.read(remaining_bytes)
|
64
|
-
end
|
65
|
-
@position += data ? data.bytesize : 0
|
66
|
-
output_buffer ? output_buffer.replace(data || '') : data
|
67
|
-
end
|
68
|
-
|
69
|
-
def remaining_bytes
|
70
|
-
@last_byte - @position
|
71
|
-
end
|
72
|
-
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
@@ -1,58 +0,0 @@
|
|
1
|
-
require 'pathname'
|
2
|
-
|
3
|
-
module Aws
|
4
|
-
module S3
|
5
|
-
# @api private
|
6
|
-
class FileUploader
|
7
|
-
|
8
|
-
FIFTEEN_MEGABYTES = 15 * 1024 * 1024
|
9
|
-
|
10
|
-
# @option options [Client] :client
|
11
|
-
# @option options [Integer] :multipart_threshold Files greater than
|
12
|
-
# `:multipart_threshold` bytes are uploaded using S3 multipart APIs.
|
13
|
-
def initialize(options = {})
|
14
|
-
@options = options
|
15
|
-
@client = options[:client] || Client.new
|
16
|
-
@multipart_threshold = options[:multipart_threshold] || FIFTEEN_MEGABYTES
|
17
|
-
end
|
18
|
-
|
19
|
-
# @return [Client]
|
20
|
-
attr_reader :client
|
21
|
-
|
22
|
-
# @return [Integer] Files larger than this in bytes are uploaded
|
23
|
-
# using a {MultipartFileUploader}.
|
24
|
-
attr_reader :multipart_threshold
|
25
|
-
|
26
|
-
# @param [String,Pathname,File,Tempfile] source
|
27
|
-
# @option options [required,String] :bucket
|
28
|
-
# @option options [required,String] :key
|
29
|
-
# @return [void]
|
30
|
-
def upload(source, options = {})
|
31
|
-
if File.size(source) >= multipart_threshold
|
32
|
-
MultipartFileUploader.new(@options).upload(source, options)
|
33
|
-
else
|
34
|
-
put_object(source, options)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
private
|
39
|
-
|
40
|
-
def put_object(source, options)
|
41
|
-
open_file(source) do |file|
|
42
|
-
@client.put_object(options.merge(body: file))
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
def open_file(source)
|
47
|
-
if String === source || Pathname === source
|
48
|
-
file = File.open(source, 'rb')
|
49
|
-
yield(file)
|
50
|
-
file.close
|
51
|
-
else
|
52
|
-
yield(source)
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
@@ -1,187 +0,0 @@
|
|
1
|
-
require 'pathname'
|
2
|
-
require 'thread'
|
3
|
-
require 'set'
|
4
|
-
|
5
|
-
module Aws
|
6
|
-
module S3
|
7
|
-
# @api private
|
8
|
-
class MultipartFileUploader
|
9
|
-
|
10
|
-
MIN_PART_SIZE = 5 * 1024 * 1024 # 5MB
|
11
|
-
|
12
|
-
FILE_TOO_SMALL = "unable to multipart upload files smaller than 5MB"
|
13
|
-
|
14
|
-
MAX_PARTS = 10_000
|
15
|
-
|
16
|
-
THREAD_COUNT = 10
|
17
|
-
|
18
|
-
# @api private
|
19
|
-
CREATE_OPTIONS =
|
20
|
-
Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
|
21
|
-
|
22
|
-
# @api private
|
23
|
-
UPLOAD_PART_OPTIONS =
|
24
|
-
Set.new(Client.api.operation(:upload_part).input.shape.member_names)
|
25
|
-
|
26
|
-
# @option options [Client] :client
|
27
|
-
def initialize(options = {})
|
28
|
-
@client = options[:client] || Client.new
|
29
|
-
@thread_count = options[:thread_count] || THREAD_COUNT
|
30
|
-
end
|
31
|
-
|
32
|
-
# @return [Client]
|
33
|
-
attr_reader :client
|
34
|
-
|
35
|
-
# @param [String,Pathname,File,Tempfile] source
|
36
|
-
# @option options [required,String] :bucket
|
37
|
-
# @option options [required,String] :key
|
38
|
-
# @return [void]
|
39
|
-
def upload(source, options = {})
|
40
|
-
if File.size(source) < MIN_PART_SIZE
|
41
|
-
raise ArgumentError, FILE_TOO_SMALL
|
42
|
-
else
|
43
|
-
upload_id = initiate_upload(options)
|
44
|
-
parts = upload_parts(upload_id, source, options)
|
45
|
-
complete_upload(upload_id, parts, options)
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
private
|
50
|
-
|
51
|
-
def initiate_upload(options)
|
52
|
-
@client.create_multipart_upload(create_opts(options)).upload_id
|
53
|
-
end
|
54
|
-
|
55
|
-
def complete_upload(upload_id, parts, options)
|
56
|
-
@client.complete_multipart_upload(
|
57
|
-
bucket: options[:bucket],
|
58
|
-
key: options[:key],
|
59
|
-
upload_id: upload_id,
|
60
|
-
multipart_upload: { parts: parts })
|
61
|
-
end
|
62
|
-
|
63
|
-
def upload_parts(upload_id, source, options)
|
64
|
-
pending = PartList.new(compute_parts(upload_id, source, options))
|
65
|
-
completed = PartList.new
|
66
|
-
errors = upload_in_threads(pending, completed)
|
67
|
-
if errors.empty?
|
68
|
-
completed.to_a.sort_by { |part| part[:part_number] }
|
69
|
-
else
|
70
|
-
abort_upload(upload_id, options, errors)
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
def abort_upload(upload_id, options, errors)
|
75
|
-
@client.abort_multipart_upload(
|
76
|
-
bucket: options[:bucket],
|
77
|
-
key: options[:key],
|
78
|
-
upload_id: upload_id
|
79
|
-
)
|
80
|
-
msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
|
81
|
-
raise MultipartUploadError.new(msg, errors)
|
82
|
-
rescue MultipartUploadError => error
|
83
|
-
raise error
|
84
|
-
rescue => error
|
85
|
-
msg = "failed to abort multipart upload: #{error.message}"
|
86
|
-
raise MultipartUploadError.new(msg, errors + [error])
|
87
|
-
end
|
88
|
-
|
89
|
-
def compute_parts(upload_id, source, options)
|
90
|
-
size = File.size(source)
|
91
|
-
default_part_size = compute_default_part_size(size)
|
92
|
-
offset = 0
|
93
|
-
part_number = 1
|
94
|
-
parts = []
|
95
|
-
while offset < size
|
96
|
-
parts << upload_part_opts(options).merge({
|
97
|
-
upload_id: upload_id,
|
98
|
-
part_number: part_number,
|
99
|
-
body: FilePart.new(
|
100
|
-
source: source,
|
101
|
-
offset: offset,
|
102
|
-
size: part_size(size, default_part_size, offset)
|
103
|
-
)
|
104
|
-
})
|
105
|
-
part_number += 1
|
106
|
-
offset += default_part_size
|
107
|
-
end
|
108
|
-
parts
|
109
|
-
end
|
110
|
-
|
111
|
-
def create_opts(options)
|
112
|
-
CREATE_OPTIONS.inject({}) do |hash, key|
|
113
|
-
hash[key] = options[key] if options.key?(key)
|
114
|
-
hash
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
|
-
def upload_part_opts(options)
|
119
|
-
UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
|
120
|
-
hash[key] = options[key] if options.key?(key)
|
121
|
-
hash
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
def upload_in_threads(pending, completed)
|
126
|
-
threads = []
|
127
|
-
@thread_count.times do
|
128
|
-
thread = Thread.new do
|
129
|
-
begin
|
130
|
-
while part = pending.shift
|
131
|
-
resp = @client.upload_part(part)
|
132
|
-
part[:body].close
|
133
|
-
completed.push(etag: resp.etag, part_number: part[:part_number])
|
134
|
-
end
|
135
|
-
nil
|
136
|
-
rescue => error
|
137
|
-
# keep other threads from uploading other parts
|
138
|
-
pending.clear!
|
139
|
-
error
|
140
|
-
end
|
141
|
-
end
|
142
|
-
thread.abort_on_exception = true
|
143
|
-
threads << thread
|
144
|
-
end
|
145
|
-
threads.map(&:value).compact
|
146
|
-
end
|
147
|
-
|
148
|
-
def compute_default_part_size(source_size)
|
149
|
-
[(source_size.to_f / MAX_PARTS).ceil, MIN_PART_SIZE].max.to_i
|
150
|
-
end
|
151
|
-
|
152
|
-
def part_size(total_size, part_size, offset)
|
153
|
-
if offset + part_size > total_size
|
154
|
-
total_size - offset
|
155
|
-
else
|
156
|
-
part_size
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
|
-
# @api private
|
161
|
-
class PartList
|
162
|
-
|
163
|
-
def initialize(parts = [])
|
164
|
-
@parts = parts
|
165
|
-
@mutex = Mutex.new
|
166
|
-
end
|
167
|
-
|
168
|
-
def push(part)
|
169
|
-
@mutex.synchronize { @parts.push(part) }
|
170
|
-
end
|
171
|
-
|
172
|
-
def shift
|
173
|
-
@mutex.synchronize { @parts.shift }
|
174
|
-
end
|
175
|
-
|
176
|
-
def clear!
|
177
|
-
@mutex.synchronize { @parts.clear }
|
178
|
-
end
|
179
|
-
|
180
|
-
def to_a
|
181
|
-
@mutex.synchronize { @parts.dup }
|
182
|
-
end
|
183
|
-
|
184
|
-
end
|
185
|
-
end
|
186
|
-
end
|
187
|
-
end
|