miasma-aws 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/LICENSE +13 -0
- data/README.md +20 -0
- data/lib/miasma-aws.rb +2 -0
- data/lib/miasma-aws/version.rb +4 -0
- data/lib/miasma/contrib/aws.rb +444 -0
- data/lib/miasma/contrib/aws/auto_scale.rb +86 -0
- data/lib/miasma/contrib/aws/compute.rb +113 -0
- data/lib/miasma/contrib/aws/load_balancer.rb +187 -0
- data/lib/miasma/contrib/aws/orchestration.rb +350 -0
- data/lib/miasma/contrib/aws/storage.rb +405 -0
- data/miasma-aws.gemspec +15 -0
- metadata +70 -0
@@ -0,0 +1,405 @@
|
|
1
|
+
require 'stringio'
|
2
|
+
require 'xmlsimple'
|
3
|
+
require 'miasma'
|
4
|
+
|
5
|
+
module Miasma
|
6
|
+
module Models
|
7
|
+
class Storage
|
8
|
+
class Aws < Storage
|
9
|
+
|
10
|
+
# Service name of the API
|
11
|
+
API_SERVICE = 's3'
|
12
|
+
# Supported version of the AutoScaling API
|
13
|
+
API_VERSION = '2006-03-01'
|
14
|
+
|
15
|
+
include Contrib::AwsApiCore::ApiCommon
|
16
|
+
include Contrib::AwsApiCore::RequestUtils
|
17
|
+
|
18
|
+
# Simple init override to force HOST and adjust region for
|
19
|
+
# signatures if required
|
20
|
+
def initialize(args)
|
21
|
+
args = args.to_smash
|
22
|
+
cache_region = args[:aws_region]
|
23
|
+
args[:aws_region] = args.fetch(:aws_bucket_region, 'us-east-1')
|
24
|
+
super(args)
|
25
|
+
aws_region = cache_region
|
26
|
+
if(aws_bucket_region)
|
27
|
+
self.aws_host = "s3-#{aws_bucket_region}.amazonaws.com"
|
28
|
+
else
|
29
|
+
self.aws_host = 's3.amazonaws.com'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Save bucket
|
34
|
+
#
|
35
|
+
# @param bucket [Models::Storage::Bucket]
|
36
|
+
# @return [Models::Storage::Bucket]
|
37
|
+
def bucket_save(bucket)
|
38
|
+
unless(bucket.persisted?)
|
39
|
+
req_args = Smash.new(
|
40
|
+
:method => :put,
|
41
|
+
:path => '/',
|
42
|
+
:endpoint => bucket_endpoint(bucket)
|
43
|
+
)
|
44
|
+
if(aws_bucket_region)
|
45
|
+
req_args[:body] = XmlSimple.xml_out(
|
46
|
+
Smash.new(
|
47
|
+
'CreateBucketConfiguration' => {
|
48
|
+
'LocationConstraint' => aws_bucket_region
|
49
|
+
}
|
50
|
+
),
|
51
|
+
'AttrPrefix' => true,
|
52
|
+
'KeepRoot' => true
|
53
|
+
)
|
54
|
+
req_args[:headers] = Smash.new(
|
55
|
+
'Content-Length' => req_args[:body].size.to_s
|
56
|
+
)
|
57
|
+
end
|
58
|
+
request(req_args)
|
59
|
+
bucket.id = bucket.name
|
60
|
+
bucket.valid_state
|
61
|
+
end
|
62
|
+
bucket
|
63
|
+
end
|
64
|
+
|
65
|
+
# Destroy bucket
|
66
|
+
#
|
67
|
+
# @param bucket [Models::Storage::Bucket]
|
68
|
+
# @return [TrueClass, FalseClass]
|
69
|
+
def bucket_destroy(bucket)
|
70
|
+
if(bucket.persisted?)
|
71
|
+
request(
|
72
|
+
:path => '/',
|
73
|
+
:method => :delete,
|
74
|
+
:endpoint => bucket_endpoint(bucket),
|
75
|
+
:expects => 204
|
76
|
+
)
|
77
|
+
true
|
78
|
+
else
|
79
|
+
false
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# Reload the bucket
|
84
|
+
#
|
85
|
+
# @param bucket [Models::Storage::Bucket]
|
86
|
+
# @return [Models::Storage::Bucket]
|
87
|
+
def bucket_reload(bucket)
|
88
|
+
if(bucket.persisted?)
|
89
|
+
begin
|
90
|
+
result = request(
|
91
|
+
:path => '/',
|
92
|
+
:method => :head,
|
93
|
+
:endpoint => bucket_endpoint(bucket)
|
94
|
+
)
|
95
|
+
rescue Error::ApiError::RequestError => e
|
96
|
+
if(e.response.status == 404)
|
97
|
+
bucket.data.clear
|
98
|
+
bucket.dirty.clear
|
99
|
+
else
|
100
|
+
raise
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
bucket
|
105
|
+
end
|
106
|
+
|
107
|
+
# Custom bucket endpoint
|
108
|
+
#
|
109
|
+
# @param bucket [Models::Storage::Bucket]
|
110
|
+
# @return [String]
|
111
|
+
# @todo properly escape bucket name
|
112
|
+
def bucket_endpoint(bucket)
|
113
|
+
::File.join(endpoint, bucket.name)
|
114
|
+
end
|
115
|
+
|
116
|
+
# Return all buckets
|
117
|
+
#
|
118
|
+
# @return [Array<Models::Storage::Bucket>]
|
119
|
+
def bucket_all
|
120
|
+
result = all_result_pages(nil, :body, 'ListAllMyBucketsResult', 'Buckets', 'Bucket') do |options|
|
121
|
+
request(
|
122
|
+
:path => '/',
|
123
|
+
:params => options
|
124
|
+
)
|
125
|
+
end
|
126
|
+
result.map do |bkt|
|
127
|
+
Bucket.new(
|
128
|
+
self,
|
129
|
+
:id => bkt['Name'],
|
130
|
+
:name => bkt['Name'],
|
131
|
+
:created => bkt['CreationDate']
|
132
|
+
).valid_state
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
# Return filtered files
|
137
|
+
#
|
138
|
+
# @param args [Hash] filter options
|
139
|
+
# @return [Array<Models::Storage::File>]
|
140
|
+
def file_filter(bucket, args)
|
141
|
+
if(args[:prefix])
|
142
|
+
result = request(
|
143
|
+
:path => '/',
|
144
|
+
:endpoint => bucket_endpoint(bucket),
|
145
|
+
:params => Smash.new(
|
146
|
+
:prefix => args[:prefix]
|
147
|
+
)
|
148
|
+
)
|
149
|
+
[result.get(:body, 'ListBucketResult', 'Contents')].flatten.compact.map do |file|
|
150
|
+
File.new(
|
151
|
+
bucket,
|
152
|
+
:id => ::File.join(bucket.name, file['Key']),
|
153
|
+
:name => file['Key'],
|
154
|
+
:updated => file['LastModified'],
|
155
|
+
:size => file['Size'].to_i
|
156
|
+
).valid_state
|
157
|
+
end
|
158
|
+
else
|
159
|
+
bucket_all
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
# Return all files within bucket
|
164
|
+
#
|
165
|
+
# @param bucket [Bucket]
|
166
|
+
# @return [Array<File>]
|
167
|
+
def file_all(bucket)
|
168
|
+
result = all_result_pages(nil, :body, 'ListBucketResult', 'Contents') do |options|
|
169
|
+
request(
|
170
|
+
:path => '/',
|
171
|
+
:params => options,
|
172
|
+
:endpoint => bucket_endpoint(bucket)
|
173
|
+
)
|
174
|
+
end
|
175
|
+
result.map do |file|
|
176
|
+
File.new(
|
177
|
+
bucket,
|
178
|
+
:id => ::File.join(bucket.name, file['Key']),
|
179
|
+
:name => file['Key'],
|
180
|
+
:updated => file['LastModified'],
|
181
|
+
:size => file['Size'].to_i
|
182
|
+
).valid_state
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
# Save file
|
187
|
+
#
|
188
|
+
# @param file [Models::Storage::File]
|
189
|
+
# @return [Models::Storage::File]
|
190
|
+
def file_save(file)
|
191
|
+
if(file.dirty?)
|
192
|
+
file.load_data(file.attributes)
|
193
|
+
args = Smash.new
|
194
|
+
args[:headers] = Smash[
|
195
|
+
Smash.new(
|
196
|
+
:content_type => 'Content-Type',
|
197
|
+
:content_disposition => 'Content-Disposition',
|
198
|
+
:content_encoding => 'Content-Encoding'
|
199
|
+
).map do |attr, key|
|
200
|
+
if(file.attributes[attr])
|
201
|
+
[key, file.attributes[attr]]
|
202
|
+
end
|
203
|
+
end.compact
|
204
|
+
]
|
205
|
+
if(file.attributes[:body].is_a?(IO) && file.body.size >= Storage::MAX_BODY_SIZE_FOR_STRINGIFY)
|
206
|
+
upload_id = request(
|
207
|
+
args.merge(
|
208
|
+
Smash.new(
|
209
|
+
:path => file_path(file),
|
210
|
+
:endpoint => bucket_endpoint(bucket),
|
211
|
+
:params => {
|
212
|
+
:uploads => true
|
213
|
+
}
|
214
|
+
)
|
215
|
+
)
|
216
|
+
).get(:body, 'InitiateMultipartUploadResult', 'UploadId')
|
217
|
+
count = 1
|
218
|
+
parts = []
|
219
|
+
file.body.rewind
|
220
|
+
while(content = file.body.read(Storage::READ_BODY_CHUNK_SIZE))
|
221
|
+
parts << [
|
222
|
+
count,
|
223
|
+
request(
|
224
|
+
:method => :put,
|
225
|
+
:path => file_path(file),
|
226
|
+
:endpoint => bucket_endpoint(bucket),
|
227
|
+
:headers => Smash.new(
|
228
|
+
'Content-Length' => content.size,
|
229
|
+
'Content-MD5' => Digest::MD5.hexdigest(content)
|
230
|
+
),
|
231
|
+
:params => Smash.new(
|
232
|
+
'partNumber' => count,
|
233
|
+
'uploadId' => upload_id
|
234
|
+
),
|
235
|
+
:body => content
|
236
|
+
).get(:body, :headers, :etag)
|
237
|
+
]
|
238
|
+
count += 1
|
239
|
+
end
|
240
|
+
complete = SimpleXml.xml_out(
|
241
|
+
Smash.new(
|
242
|
+
'CompleteMultipartUpload' => {
|
243
|
+
'Part' => parts.map{|part|
|
244
|
+
{'PartNumber' => part.first, 'ETag' => part.last}
|
245
|
+
}
|
246
|
+
}
|
247
|
+
),
|
248
|
+
'AttrPrefix' => true,
|
249
|
+
'KeepRoot' => true
|
250
|
+
)
|
251
|
+
result = request(
|
252
|
+
:method => :post,
|
253
|
+
:path => file_path(file),
|
254
|
+
:endpoint => bucket_endpoint(file.bucket),
|
255
|
+
:params => Smash.new(
|
256
|
+
'UploadId' => upload_id
|
257
|
+
),
|
258
|
+
:headers => Smash.new(
|
259
|
+
'Content-Length' => complete.size
|
260
|
+
),
|
261
|
+
:body => complete
|
262
|
+
)
|
263
|
+
file.etag = result.get(:body, 'CompleteMultipartUploadResult', 'ETag')
|
264
|
+
else
|
265
|
+
if(file.attributes[:body].is_a?(IO) || file.attributes[:body].is_a?(StringIO))
|
266
|
+
args[:headers]['Content-Length'] = file.body.size.to_s
|
267
|
+
file.body.rewind
|
268
|
+
args[:body] = file.body.read
|
269
|
+
file.body.rewind
|
270
|
+
end
|
271
|
+
result = request(
|
272
|
+
args.merge(
|
273
|
+
Smash.new(
|
274
|
+
:method => :put,
|
275
|
+
:path => file_path(file),
|
276
|
+
:endpoint => bucket_endpoint(file.bucket)
|
277
|
+
)
|
278
|
+
)
|
279
|
+
)
|
280
|
+
file.etag = result.get(:headers, :etag)
|
281
|
+
end
|
282
|
+
file.id = ::File.join(file.bucket.name, file.name)
|
283
|
+
file.valid_state
|
284
|
+
end
|
285
|
+
file
|
286
|
+
end
|
287
|
+
|
288
|
+
# Destroy file
|
289
|
+
#
|
290
|
+
# @param file [Models::Storage::File]
|
291
|
+
# @return [TrueClass, FalseClass]
|
292
|
+
def file_destroy(file)
|
293
|
+
if(file.persisted?)
|
294
|
+
request(
|
295
|
+
:method => :delete,
|
296
|
+
:path => file_path(file),
|
297
|
+
:endpoint => bucket_endpoint(file.bucket),
|
298
|
+
:expects => 204
|
299
|
+
)
|
300
|
+
true
|
301
|
+
else
|
302
|
+
false
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
# Reload the file
|
307
|
+
#
|
308
|
+
# @param file [Models::Storage::File]
|
309
|
+
# @return [Models::Storage::File]
|
310
|
+
def file_reload(file)
|
311
|
+
if(file.persisted?)
|
312
|
+
name = file.name
|
313
|
+
result = request(
|
314
|
+
:path => file_path(file),
|
315
|
+
:endpoint => bucket_endpoint(file.bucket)
|
316
|
+
)
|
317
|
+
file.data.clear && file.dirty.clear
|
318
|
+
info = result[:headers]
|
319
|
+
file.load_data(
|
320
|
+
:id => ::File.join(file.bucket.name, name),
|
321
|
+
:name => name,
|
322
|
+
:updated => info[:last_modified],
|
323
|
+
:etag => info[:etag],
|
324
|
+
:size => info[:content_length].to_i,
|
325
|
+
:content_type => info[:content_type]
|
326
|
+
).valid_state
|
327
|
+
end
|
328
|
+
file
|
329
|
+
end
|
330
|
+
|
331
|
+
# Create publicly accessible URL
|
332
|
+
#
|
333
|
+
# @param timeout_secs [Integer] seconds available
|
334
|
+
# @return [String] URL
|
335
|
+
def file_url(file, timeout_secs)
|
336
|
+
if(file.persisted?)
|
337
|
+
signer.generate_url(
|
338
|
+
:get, ::File.join(uri_escape(file.bucket.name), file_path(file)),
|
339
|
+
:headers => Smash.new(
|
340
|
+
'Host' => aws_host
|
341
|
+
),
|
342
|
+
:params => Smash.new(
|
343
|
+
'X-Amz-Date' => Contrib::AwsApiCore.time_iso8601,
|
344
|
+
'X-Amz-Expires' => timeout_secs
|
345
|
+
)
|
346
|
+
)
|
347
|
+
else
|
348
|
+
raise Error::ModelPersistError.new "#{file} has not been saved!"
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
352
|
+
# Fetch the contents of the file
|
353
|
+
#
|
354
|
+
# @param file [Models::Storage::File]
|
355
|
+
# @return [IO, HTTP::Response::Body]
|
356
|
+
def file_body(file)
|
357
|
+
if(file.persisted?)
|
358
|
+
result = request(
|
359
|
+
:path => file_path(file),
|
360
|
+
:endpoint => bucket_endpoint(file.bucket),
|
361
|
+
:disable_body_extraction => true
|
362
|
+
)
|
363
|
+
content = result[:body]
|
364
|
+
begin
|
365
|
+
if(content.is_a?(String))
|
366
|
+
StringIO.new(content)
|
367
|
+
else
|
368
|
+
if(content.respond_to?(:stream!))
|
369
|
+
content.stream!
|
370
|
+
end
|
371
|
+
content
|
372
|
+
end
|
373
|
+
rescue HTTP::StateError
|
374
|
+
StringIO.new(content.to_s)
|
375
|
+
end
|
376
|
+
else
|
377
|
+
StringIO.new('')
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
381
|
+
# Simple callback to allow request option adjustments prior to
|
382
|
+
# signature calculation
|
383
|
+
#
|
384
|
+
# @param opts [Smash] request options
|
385
|
+
# @return [TrueClass]
|
386
|
+
# @note this only updates when :body is defined. if a :post is
|
387
|
+
# happening (which implicitly forces :form) or :json is used
|
388
|
+
# it will not properly checksum. (but that's probably okay)
|
389
|
+
def update_request(con, opts)
|
390
|
+
con.default_headers['x-amz-content-sha256'] = Digest::SHA256.
|
391
|
+
hexdigest(opts.fetch(:body, ''))
|
392
|
+
true
|
393
|
+
end
|
394
|
+
|
395
|
+
# @return [String] escaped file path
|
396
|
+
def file_path(file)
|
397
|
+
file.name.split('/').map do |part|
|
398
|
+
uri_escape(part)
|
399
|
+
end.join('/')
|
400
|
+
end
|
401
|
+
|
402
|
+
end
|
403
|
+
end
|
404
|
+
end
|
405
|
+
end
|
data/miasma-aws.gemspec
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
|
2
|
+
require 'miasma-aws/version'
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = 'miasma-aws'
|
5
|
+
s.version = MiasmaAws::VERSION.version
|
6
|
+
s.summary = 'Smoggy AWS API'
|
7
|
+
s.author = 'Chris Roberts'
|
8
|
+
s.email = 'code@chrisroberts.org'
|
9
|
+
s.homepage = 'https://github.com/miasma-rb/miasma-aws'
|
10
|
+
s.description = 'Smoggy AWS API'
|
11
|
+
s.license = 'Apache 2.0'
|
12
|
+
s.require_path = 'lib'
|
13
|
+
s.add_dependency 'miasma'
|
14
|
+
s.files = Dir['lib/**/*'] + %w(miasma-aws.gemspec README.md CHANGELOG.md LICENSE)
|
15
|
+
end
|
metadata
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: miasma-aws
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Chris Roberts
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-01-05 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: miasma
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
description: Smoggy AWS API
|
28
|
+
email: code@chrisroberts.org
|
29
|
+
executables: []
|
30
|
+
extensions: []
|
31
|
+
extra_rdoc_files: []
|
32
|
+
files:
|
33
|
+
- CHANGELOG.md
|
34
|
+
- LICENSE
|
35
|
+
- README.md
|
36
|
+
- lib/miasma-aws.rb
|
37
|
+
- lib/miasma-aws/version.rb
|
38
|
+
- lib/miasma/contrib/aws.rb
|
39
|
+
- lib/miasma/contrib/aws/auto_scale.rb
|
40
|
+
- lib/miasma/contrib/aws/compute.rb
|
41
|
+
- lib/miasma/contrib/aws/load_balancer.rb
|
42
|
+
- lib/miasma/contrib/aws/orchestration.rb
|
43
|
+
- lib/miasma/contrib/aws/storage.rb
|
44
|
+
- miasma-aws.gemspec
|
45
|
+
homepage: https://github.com/miasma-rb/miasma-aws
|
46
|
+
licenses:
|
47
|
+
- Apache 2.0
|
48
|
+
metadata: {}
|
49
|
+
post_install_message:
|
50
|
+
rdoc_options: []
|
51
|
+
require_paths:
|
52
|
+
- lib
|
53
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: '0'
|
58
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
requirements: []
|
64
|
+
rubyforge_project:
|
65
|
+
rubygems_version: 2.2.2
|
66
|
+
signing_key:
|
67
|
+
specification_version: 4
|
68
|
+
summary: Smoggy AWS API
|
69
|
+
test_files: []
|
70
|
+
has_rdoc:
|