aws-sdk-resources 2.1.13 → 2.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/aws-sdk-resources/definition.rb +1 -1
- data/lib/aws-sdk-resources/documenter.rb +41 -0
- data/lib/aws-sdk-resources/request_params.rb +1 -1
- data/lib/aws-sdk-resources/services/s3.rb +2 -0
- data/lib/aws-sdk-resources/services/s3/object.rb +57 -0
- data/lib/aws-sdk-resources/services/s3/object_copier.rb +61 -0
- data/lib/aws-sdk-resources/services/s3/object_multipart_copier.rb +171 -0
- metadata +7 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 34475603466832ba05d1de7ada5968455f9751f1
|
4
|
+
data.tar.gz: a204f4de98bf6c67d24ac161b279ee17356a2902
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bda165ec79702fc8ae6b3063be7358dd4ba94cf2e3bc5c0589981ce1866139ed5d014ecf05f570c6cdd4e0e27a8d9c5e2aeb2b7840e3709f4997ca00a0805f5c
|
7
|
+
data.tar.gz: d3853941e7d5f3f70429a49fd593a9518b5d39f81c05d609d8e4195bb0167568a59400a85ffe6a5881b740bb109ccd7c3b45899df51864e45c969a153b786258
|
@@ -17,6 +17,7 @@ module Aws
|
|
17
17
|
|
18
18
|
def apply_customizations
|
19
19
|
document_s3_object_upload_file_additional_options
|
20
|
+
document_s3_object_copy_from_options
|
20
21
|
end
|
21
22
|
|
22
23
|
private
|
@@ -47,6 +48,46 @@ module Aws
|
|
47
48
|
tags.each { |tag| m.add_tag(tag) }
|
48
49
|
end
|
49
50
|
|
51
|
+
def document_s3_object_copy_from_options
|
52
|
+
copy_from = YARD::Registry['Aws::S3::Object#copy_from']
|
53
|
+
copy_to = YARD::Registry['Aws::S3::Object#copy_to']
|
54
|
+
existing_tags = copy_from.tags
|
55
|
+
copy_from.docstring = 'Copies another object to this object. Use `multipart_copy: true` for large objects. This is required for objects that exceed 5GB.'
|
56
|
+
existing_tags.each do |tag|
|
57
|
+
if tag.tag_name == 'option' && tag.pair.name != ":copy_source"
|
58
|
+
copy_from.add_tag(tag)
|
59
|
+
copy_to.add_tag(tag)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
copy_from.add_tag(tag(<<-EXAMPLE))
|
63
|
+
@example Basic object copy
|
64
|
+
|
65
|
+
bucket = Aws::S3::Bucket.new('target-bucket')
|
66
|
+
object = bucket.object('target-key')
|
67
|
+
|
68
|
+
# source as String
|
69
|
+
object.copy_from('source-bucket/source-key')
|
70
|
+
|
71
|
+
# source as Hash
|
72
|
+
object.copy_from(bucket:'source-bucket', key:'source-key')
|
73
|
+
|
74
|
+
# source as Aws::S3::Object
|
75
|
+
object.copy_from(bucket.object('source-key'))
|
76
|
+
EXAMPLE
|
77
|
+
|
78
|
+
copy_from.add_tag(tag(<<-EXAMPLE))
|
79
|
+
@example Managed copy of large objects
|
80
|
+
|
81
|
+
# uses multipart upload APIs to copy object
|
82
|
+
object.copy_from('src-bucket/src-key', multipart_copy: true)
|
83
|
+
EXAMPLE
|
84
|
+
|
85
|
+
end
|
86
|
+
|
87
|
+
def tag(string)
|
88
|
+
YARD::DocstringParser.new.parse(string).to_docstring.tags.first
|
89
|
+
end
|
90
|
+
|
50
91
|
end
|
51
92
|
end
|
52
93
|
end
|
@@ -11,6 +11,8 @@ module Aws
|
|
11
11
|
autoload :FileUploader, 'aws-sdk-resources/services/s3/file_uploader'
|
12
12
|
autoload :MultipartFileUploader, 'aws-sdk-resources/services/s3/multipart_file_uploader'
|
13
13
|
autoload :MultipartUploadError, 'aws-sdk-resources/services/s3/multipart_upload_error'
|
14
|
+
autoload :ObjectCopier, 'aws-sdk-resources/services/s3/object_copier'
|
15
|
+
autoload :ObjectMultipartCopier, 'aws-sdk-resources/services/s3/object_multipart_copier'
|
14
16
|
autoload :PresignedPost, 'aws-sdk-resources/services/s3/presigned_post'
|
15
17
|
|
16
18
|
end
|
@@ -135,6 +135,63 @@ module Aws
|
|
135
135
|
options.merge(key: key))
|
136
136
|
end
|
137
137
|
|
138
|
+
# @param [S3::Object, String, Hash] source Where to copy object
|
139
|
+
# data from. `source` must be one of the following:
|
140
|
+
#
|
141
|
+
# * {Aws::S3::Object}
|
142
|
+
# * Hash - with `:bucket` and `:key`
|
143
|
+
# * String - formatted like `"source-bucket-name/source-key"`
|
144
|
+
#
|
145
|
+
# @option options [Boolean] :multipart_copy (false) When `true`,
|
146
|
+
# the object will be copied using the multipart APIs. This is
|
147
|
+
# necessary for objects larger than 5GB and can provide
|
148
|
+
# performance improvements on large objects. Amazon S3 does
|
149
|
+
# not accept multipart copies for objects smaller than 5MB.
|
150
|
+
#
|
151
|
+
# @see #copy_to
|
152
|
+
#
|
153
|
+
def copy_from(source, options = {})
|
154
|
+
if Hash === source && source[:copy_source]
|
155
|
+
# for backwards compatibility
|
156
|
+
@client.copy_object(source.merge(bucket: bucket_name, key: key))
|
157
|
+
else
|
158
|
+
ObjectCopier.new(self, options).copy_from(source, options)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
# Copies this object to another object. Use `multipart_copy: true`
|
163
|
+
# for large objects. This is required for objects that exceed 5GB.
|
164
|
+
#
|
165
|
+
# @param [S3::Object, String, Hash] target Where to copy the object
|
166
|
+
# data to. `target` must be one of the following:
|
167
|
+
#
|
168
|
+
# * {Aws::S3::Object}
|
169
|
+
# * Hash - with `:bucket` and `:key`
|
170
|
+
# * String - formatted like `"target-bucket-name/target-key"`
|
171
|
+
#
|
172
|
+
# @example Basic object copy
|
173
|
+
#
|
174
|
+
# bucket = Aws::S3::Bucket.new('source-bucket')
|
175
|
+
# object = bucket.object('source-key')
|
176
|
+
#
|
177
|
+
# # target as String
|
178
|
+
# object.copy_to('target-bucket/target-key')
|
179
|
+
#
|
180
|
+
# # target as Hash
|
181
|
+
# object.copy_to(bucket: 'target-bucket', key: 'target-key')
|
182
|
+
#
|
183
|
+
# # target as Aws::S3::Object
|
184
|
+
# object.copy_to(bucket.object('target-key'))
|
185
|
+
#
|
186
|
+
# @example Managed copy of large objects
|
187
|
+
#
|
188
|
+
# # uses multipart upload APIs to copy object
|
189
|
+
# object.copy_to('src-bucket/src-key', multipart_copy: true)
|
190
|
+
#
|
191
|
+
def copy_to(target, options = {})
|
192
|
+
ObjectCopier.new(self, options).copy_to(target, options)
|
193
|
+
end
|
194
|
+
|
138
195
|
end
|
139
196
|
end
|
140
197
|
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
module Aws
|
4
|
+
module S3
|
5
|
+
# @api private
|
6
|
+
class ObjectCopier
|
7
|
+
|
8
|
+
# @param [S3::Objecst] object
|
9
|
+
def initialize(object, options = {})
|
10
|
+
@object = object
|
11
|
+
@options = options.merge(client: @object.client)
|
12
|
+
end
|
13
|
+
|
14
|
+
def copy_from(source, options = {})
|
15
|
+
copy_object(source, @object, options)
|
16
|
+
end
|
17
|
+
|
18
|
+
|
19
|
+
def copy_to(target, options = {})
|
20
|
+
copy_object(@object, target, options)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def copy_object(source, target, options)
|
26
|
+
target_bucket, target_key = copy_target(target)
|
27
|
+
options[:bucket] = target_bucket
|
28
|
+
options[:key] = target_key
|
29
|
+
options[:copy_source] = copy_source(source)
|
30
|
+
if options.delete(:multipart_copy)
|
31
|
+
ObjectMultipartCopier.new(@options).copy(options)
|
32
|
+
else
|
33
|
+
@object.client.copy_object(options)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def copy_source(source)
|
38
|
+
case source
|
39
|
+
when String then source
|
40
|
+
when Hash then "#{source[:bucket]}/#{source[:key]}"
|
41
|
+
when S3::Object then "#{source.bucket_name}/#{source.key}"
|
42
|
+
else
|
43
|
+
msg = "expected source to be an Aws::S3::Object, Hash, or String"
|
44
|
+
raise ArgumentError, msg
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def copy_target(target)
|
49
|
+
case target
|
50
|
+
when String then target.match(/([^\/]+?)\/(.+)/)[1,2]
|
51
|
+
when Hash then target.values_at(:bucket, :key)
|
52
|
+
when S3::Object then [target.bucket_name, target.key]
|
53
|
+
else
|
54
|
+
msg = "expected target to be an Aws::S3::Object, Hash, or String"
|
55
|
+
raise ArgumentError, msg
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,171 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
module Aws
|
4
|
+
module S3
|
5
|
+
# @api private
|
6
|
+
class ObjectMultipartCopier
|
7
|
+
|
8
|
+
FIVE_MB = 5 * 1024 * 1024 # 5MB
|
9
|
+
|
10
|
+
FILE_TOO_SMALL = "unable to multipart copy files smaller than 5MB"
|
11
|
+
|
12
|
+
MAX_PARTS = 10_000
|
13
|
+
|
14
|
+
# @option options [Client] :client
|
15
|
+
# @option [Integer] :min_part_size (52428800) Size of copied parts.
|
16
|
+
# Defaults to 50MB.
|
17
|
+
# will be constructed from the given `options' hash.
|
18
|
+
# @option [Integer] :thread_count (10) Number of concurrent threads to
|
19
|
+
# use for copying parts.
|
20
|
+
def initialize(options = {})
|
21
|
+
@thread_count = options.delete(:thread_count) || 10
|
22
|
+
@min_part_size = options.delete(:min_part_size) || (FIVE_MB * 10)
|
23
|
+
@client = options[:client] || Client.new
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [Client]
|
27
|
+
attr_reader :client
|
28
|
+
|
29
|
+
# @option (see S3::Client#copy_object)
|
30
|
+
def copy(options = {})
|
31
|
+
size = source_size(options)
|
32
|
+
options[:upload_id] = initiate_upload(options)
|
33
|
+
begin
|
34
|
+
parts = copy_parts(size, default_part_size(size), options)
|
35
|
+
complete_upload(parts, options)
|
36
|
+
rescue => error
|
37
|
+
abort_upload(options)
|
38
|
+
raise error
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def initiate_upload(options)
|
45
|
+
options = options_for(:create_multipart_upload, options)
|
46
|
+
@client.create_multipart_upload(options).upload_id
|
47
|
+
end
|
48
|
+
|
49
|
+
def copy_parts(size, default_part_size, options)
|
50
|
+
queue = PartQueue.new(compute_parts(size, default_part_size, options))
|
51
|
+
threads = []
|
52
|
+
@thread_count.times do
|
53
|
+
threads << copy_part_thread(queue)
|
54
|
+
end
|
55
|
+
threads.map(&:value).flatten.sort_by{ |part| part[:part_number] }
|
56
|
+
end
|
57
|
+
|
58
|
+
def copy_part_thread(queue)
|
59
|
+
Thread.new do
|
60
|
+
begin
|
61
|
+
completed = []
|
62
|
+
while part = queue.shift
|
63
|
+
completed << copy_part(part)
|
64
|
+
end
|
65
|
+
completed
|
66
|
+
rescue => error
|
67
|
+
queue.clear!
|
68
|
+
raise error
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def copy_part(part)
|
74
|
+
{
|
75
|
+
etag: @client.upload_part_copy(part).copy_part_result.etag,
|
76
|
+
part_number: part[:part_number],
|
77
|
+
}
|
78
|
+
end
|
79
|
+
|
80
|
+
def complete_upload(parts, options)
|
81
|
+
options = options_for(:complete_multipart_upload, options)
|
82
|
+
options[:multipart_upload] = { parts: parts }
|
83
|
+
@client.complete_multipart_upload(options)
|
84
|
+
end
|
85
|
+
|
86
|
+
def abort_upload(options)
|
87
|
+
@client.abort_multipart_upload({
|
88
|
+
bucket: options[:bucket],
|
89
|
+
key: options[:key],
|
90
|
+
upload_id: options[:upload_id],
|
91
|
+
})
|
92
|
+
end
|
93
|
+
|
94
|
+
def compute_parts(size, default_part_size, options)
|
95
|
+
part_number = 1
|
96
|
+
offset = 0
|
97
|
+
parts = []
|
98
|
+
options = options_for(:upload_part_copy, options)
|
99
|
+
while offset < size
|
100
|
+
parts << options.merge({
|
101
|
+
part_number: part_number,
|
102
|
+
copy_source_range: byte_range(offset, default_part_size, size),
|
103
|
+
})
|
104
|
+
part_number += 1
|
105
|
+
offset += default_part_size
|
106
|
+
end
|
107
|
+
parts
|
108
|
+
end
|
109
|
+
|
110
|
+
def byte_range(offset, default_part_size, size)
|
111
|
+
if offset + default_part_size < size
|
112
|
+
"bytes=#{offset}-#{offset + default_part_size - 1}"
|
113
|
+
else
|
114
|
+
"bytes=#{offset}-#{size - 1}"
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def source_size(options)
|
119
|
+
if options[:content_length]
|
120
|
+
options.delete(:content_length)
|
121
|
+
else
|
122
|
+
bucket, key = options[:copy_source].match(/([^\/]+?)\/(.+)/)[1,2]
|
123
|
+
@client.head_object(bucket:bucket, key:key).content_length
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def default_part_size(source_size)
|
128
|
+
if source_size < FIVE_MB
|
129
|
+
raise ArgumentError, FILE_TOO_SMALL
|
130
|
+
else
|
131
|
+
[(source_size.to_f / MAX_PARTS).ceil, @min_part_size].max.to_i
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def options_for(operation_name, options)
|
136
|
+
API_OPTIONS[operation_name].inject({}) do |hash, opt_name|
|
137
|
+
hash[opt_name] = options[opt_name] if options.key?(opt_name)
|
138
|
+
hash
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
# @api private
|
143
|
+
def self.options_for(shape_name)
|
144
|
+
Client.api.metadata['shapes'][shape_name].member_names
|
145
|
+
end
|
146
|
+
|
147
|
+
API_OPTIONS = {
|
148
|
+
create_multipart_upload: options_for('CreateMultipartUploadRequest'),
|
149
|
+
upload_part_copy: options_for('UploadPartCopyRequest'),
|
150
|
+
complete_multipart_upload: options_for('CompleteMultipartUploadRequest'),
|
151
|
+
}
|
152
|
+
|
153
|
+
class PartQueue
|
154
|
+
|
155
|
+
def initialize(parts = [])
|
156
|
+
@parts = parts
|
157
|
+
@mutex = Mutex.new
|
158
|
+
end
|
159
|
+
|
160
|
+
def shift
|
161
|
+
@mutex.synchronize { @parts.shift }
|
162
|
+
end
|
163
|
+
|
164
|
+
def clear!
|
165
|
+
@mutex.synchronize { @parts.clear }
|
166
|
+
end
|
167
|
+
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aws-sdk-resources
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amazon Web Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-08-
|
11
|
+
date: 2015-08-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-core
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - '='
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 2.1.
|
19
|
+
version: 2.1.14
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 2.1.
|
26
|
+
version: 2.1.14
|
27
27
|
description: Provides resource oriented interfaces and other higher-level abstractions
|
28
28
|
for many AWS services. This gem is part of the official AWS SDK for Ruby.
|
29
29
|
email:
|
@@ -77,6 +77,8 @@ files:
|
|
77
77
|
- lib/aws-sdk-resources/services/s3/multipart_upload.rb
|
78
78
|
- lib/aws-sdk-resources/services/s3/multipart_upload_error.rb
|
79
79
|
- lib/aws-sdk-resources/services/s3/object.rb
|
80
|
+
- lib/aws-sdk-resources/services/s3/object_copier.rb
|
81
|
+
- lib/aws-sdk-resources/services/s3/object_multipart_copier.rb
|
80
82
|
- lib/aws-sdk-resources/services/s3/object_summary.rb
|
81
83
|
- lib/aws-sdk-resources/services/s3/presigned_post.rb
|
82
84
|
- lib/aws-sdk-resources/services/sns.rb
|
@@ -104,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
104
106
|
version: '0'
|
105
107
|
requirements: []
|
106
108
|
rubyforge_project:
|
107
|
-
rubygems_version: 2.
|
109
|
+
rubygems_version: 2.5.0
|
108
110
|
signing_key:
|
109
111
|
specification_version: 4
|
110
112
|
summary: AWS SDK for Ruby - Resources
|