miasma-aws 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,405 @@
1
+ require 'stringio'
2
+ require 'xmlsimple'
3
+ require 'miasma'
4
+
5
+ module Miasma
6
+ module Models
7
+ class Storage
8
+ class Aws < Storage
9
+
10
+ # Service name of the API
11
+ API_SERVICE = 's3'
12
+ # Supported version of the AutoScaling API
13
+ API_VERSION = '2006-03-01'
14
+
15
+ include Contrib::AwsApiCore::ApiCommon
16
+ include Contrib::AwsApiCore::RequestUtils
17
+
18
+ # Simple init override to force HOST and adjust region for
19
+ # signatures if required
20
+ def initialize(args)
21
+ args = args.to_smash
22
+ cache_region = args[:aws_region]
23
+ args[:aws_region] = args.fetch(:aws_bucket_region, 'us-east-1')
24
+ super(args)
25
+ aws_region = cache_region
26
+ if(aws_bucket_region)
27
+ self.aws_host = "s3-#{aws_bucket_region}.amazonaws.com"
28
+ else
29
+ self.aws_host = 's3.amazonaws.com'
30
+ end
31
+ end
32
+
33
+ # Save bucket
34
+ #
35
+ # @param bucket [Models::Storage::Bucket]
36
+ # @return [Models::Storage::Bucket]
37
+ def bucket_save(bucket)
38
+ unless(bucket.persisted?)
39
+ req_args = Smash.new(
40
+ :method => :put,
41
+ :path => '/',
42
+ :endpoint => bucket_endpoint(bucket)
43
+ )
44
+ if(aws_bucket_region)
45
+ req_args[:body] = XmlSimple.xml_out(
46
+ Smash.new(
47
+ 'CreateBucketConfiguration' => {
48
+ 'LocationConstraint' => aws_bucket_region
49
+ }
50
+ ),
51
+ 'AttrPrefix' => true,
52
+ 'KeepRoot' => true
53
+ )
54
+ req_args[:headers] = Smash.new(
55
+ 'Content-Length' => req_args[:body].size.to_s
56
+ )
57
+ end
58
+ request(req_args)
59
+ bucket.id = bucket.name
60
+ bucket.valid_state
61
+ end
62
+ bucket
63
+ end
64
+
65
+ # Destroy bucket
66
+ #
67
+ # @param bucket [Models::Storage::Bucket]
68
+ # @return [TrueClass, FalseClass]
69
+ def bucket_destroy(bucket)
70
+ if(bucket.persisted?)
71
+ request(
72
+ :path => '/',
73
+ :method => :delete,
74
+ :endpoint => bucket_endpoint(bucket),
75
+ :expects => 204
76
+ )
77
+ true
78
+ else
79
+ false
80
+ end
81
+ end
82
+
83
+ # Reload the bucket
84
+ #
85
+ # @param bucket [Models::Storage::Bucket]
86
+ # @return [Models::Storage::Bucket]
87
+ def bucket_reload(bucket)
88
+ if(bucket.persisted?)
89
+ begin
90
+ result = request(
91
+ :path => '/',
92
+ :method => :head,
93
+ :endpoint => bucket_endpoint(bucket)
94
+ )
95
+ rescue Error::ApiError::RequestError => e
96
+ if(e.response.status == 404)
97
+ bucket.data.clear
98
+ bucket.dirty.clear
99
+ else
100
+ raise
101
+ end
102
+ end
103
+ end
104
+ bucket
105
+ end
106
+
107
+ # Custom bucket endpoint
108
+ #
109
+ # @param bucket [Models::Storage::Bucket]
110
+ # @return [String]
111
+ # @todo properly escape bucket name
112
+ def bucket_endpoint(bucket)
113
+ ::File.join(endpoint, bucket.name)
114
+ end
115
+
116
+ # Return all buckets
117
+ #
118
+ # @return [Array<Models::Storage::Bucket>]
119
+ def bucket_all
120
+ result = all_result_pages(nil, :body, 'ListAllMyBucketsResult', 'Buckets', 'Bucket') do |options|
121
+ request(
122
+ :path => '/',
123
+ :params => options
124
+ )
125
+ end
126
+ result.map do |bkt|
127
+ Bucket.new(
128
+ self,
129
+ :id => bkt['Name'],
130
+ :name => bkt['Name'],
131
+ :created => bkt['CreationDate']
132
+ ).valid_state
133
+ end
134
+ end
135
+
136
+ # Return filtered files
137
+ #
138
+ # @param args [Hash] filter options
139
+ # @return [Array<Models::Storage::File>]
140
+ def file_filter(bucket, args)
141
+ if(args[:prefix])
142
+ result = request(
143
+ :path => '/',
144
+ :endpoint => bucket_endpoint(bucket),
145
+ :params => Smash.new(
146
+ :prefix => args[:prefix]
147
+ )
148
+ )
149
+ [result.get(:body, 'ListBucketResult', 'Contents')].flatten.compact.map do |file|
150
+ File.new(
151
+ bucket,
152
+ :id => ::File.join(bucket.name, file['Key']),
153
+ :name => file['Key'],
154
+ :updated => file['LastModified'],
155
+ :size => file['Size'].to_i
156
+ ).valid_state
157
+ end
158
+ else
159
+ bucket_all
160
+ end
161
+ end
162
+
163
+ # Return all files within bucket
164
+ #
165
+ # @param bucket [Bucket]
166
+ # @return [Array<File>]
167
+ def file_all(bucket)
168
+ result = all_result_pages(nil, :body, 'ListBucketResult', 'Contents') do |options|
169
+ request(
170
+ :path => '/',
171
+ :params => options,
172
+ :endpoint => bucket_endpoint(bucket)
173
+ )
174
+ end
175
+ result.map do |file|
176
+ File.new(
177
+ bucket,
178
+ :id => ::File.join(bucket.name, file['Key']),
179
+ :name => file['Key'],
180
+ :updated => file['LastModified'],
181
+ :size => file['Size'].to_i
182
+ ).valid_state
183
+ end
184
+ end
185
+
186
+ # Save file
187
+ #
188
+ # @param file [Models::Storage::File]
189
+ # @return [Models::Storage::File]
190
+ def file_save(file)
191
+ if(file.dirty?)
192
+ file.load_data(file.attributes)
193
+ args = Smash.new
194
+ args[:headers] = Smash[
195
+ Smash.new(
196
+ :content_type => 'Content-Type',
197
+ :content_disposition => 'Content-Disposition',
198
+ :content_encoding => 'Content-Encoding'
199
+ ).map do |attr, key|
200
+ if(file.attributes[attr])
201
+ [key, file.attributes[attr]]
202
+ end
203
+ end.compact
204
+ ]
205
+ if(file.attributes[:body].is_a?(IO) && file.body.size >= Storage::MAX_BODY_SIZE_FOR_STRINGIFY)
206
+ upload_id = request(
207
+ args.merge(
208
+ Smash.new(
209
+ :path => file_path(file),
210
+ :endpoint => bucket_endpoint(bucket),
211
+ :params => {
212
+ :uploads => true
213
+ }
214
+ )
215
+ )
216
+ ).get(:body, 'InitiateMultipartUploadResult', 'UploadId')
217
+ count = 1
218
+ parts = []
219
+ file.body.rewind
220
+ while(content = file.body.read(Storage::READ_BODY_CHUNK_SIZE))
221
+ parts << [
222
+ count,
223
+ request(
224
+ :method => :put,
225
+ :path => file_path(file),
226
+ :endpoint => bucket_endpoint(bucket),
227
+ :headers => Smash.new(
228
+ 'Content-Length' => content.size,
229
+ 'Content-MD5' => Digest::MD5.hexdigest(content)
230
+ ),
231
+ :params => Smash.new(
232
+ 'partNumber' => count,
233
+ 'uploadId' => upload_id
234
+ ),
235
+ :body => content
236
+ ).get(:body, :headers, :etag)
237
+ ]
238
+ count += 1
239
+ end
240
+ complete = SimpleXml.xml_out(
241
+ Smash.new(
242
+ 'CompleteMultipartUpload' => {
243
+ 'Part' => parts.map{|part|
244
+ {'PartNumber' => part.first, 'ETag' => part.last}
245
+ }
246
+ }
247
+ ),
248
+ 'AttrPrefix' => true,
249
+ 'KeepRoot' => true
250
+ )
251
+ result = request(
252
+ :method => :post,
253
+ :path => file_path(file),
254
+ :endpoint => bucket_endpoint(file.bucket),
255
+ :params => Smash.new(
256
+ 'UploadId' => upload_id
257
+ ),
258
+ :headers => Smash.new(
259
+ 'Content-Length' => complete.size
260
+ ),
261
+ :body => complete
262
+ )
263
+ file.etag = result.get(:body, 'CompleteMultipartUploadResult', 'ETag')
264
+ else
265
+ if(file.attributes[:body].is_a?(IO) || file.attributes[:body].is_a?(StringIO))
266
+ args[:headers]['Content-Length'] = file.body.size.to_s
267
+ file.body.rewind
268
+ args[:body] = file.body.read
269
+ file.body.rewind
270
+ end
271
+ result = request(
272
+ args.merge(
273
+ Smash.new(
274
+ :method => :put,
275
+ :path => file_path(file),
276
+ :endpoint => bucket_endpoint(file.bucket)
277
+ )
278
+ )
279
+ )
280
+ file.etag = result.get(:headers, :etag)
281
+ end
282
+ file.id = ::File.join(file.bucket.name, file.name)
283
+ file.valid_state
284
+ end
285
+ file
286
+ end
287
+
288
+ # Destroy file
289
+ #
290
+ # @param file [Models::Storage::File]
291
+ # @return [TrueClass, FalseClass]
292
+ def file_destroy(file)
293
+ if(file.persisted?)
294
+ request(
295
+ :method => :delete,
296
+ :path => file_path(file),
297
+ :endpoint => bucket_endpoint(file.bucket),
298
+ :expects => 204
299
+ )
300
+ true
301
+ else
302
+ false
303
+ end
304
+ end
305
+
306
+ # Reload the file
307
+ #
308
+ # @param file [Models::Storage::File]
309
+ # @return [Models::Storage::File]
310
+ def file_reload(file)
311
+ if(file.persisted?)
312
+ name = file.name
313
+ result = request(
314
+ :path => file_path(file),
315
+ :endpoint => bucket_endpoint(file.bucket)
316
+ )
317
+ file.data.clear && file.dirty.clear
318
+ info = result[:headers]
319
+ file.load_data(
320
+ :id => ::File.join(file.bucket.name, name),
321
+ :name => name,
322
+ :updated => info[:last_modified],
323
+ :etag => info[:etag],
324
+ :size => info[:content_length].to_i,
325
+ :content_type => info[:content_type]
326
+ ).valid_state
327
+ end
328
+ file
329
+ end
330
+
331
+ # Create publicly accessible URL
332
+ #
333
+ # @param timeout_secs [Integer] seconds available
334
+ # @return [String] URL
335
+ def file_url(file, timeout_secs)
336
+ if(file.persisted?)
337
+ signer.generate_url(
338
+ :get, ::File.join(uri_escape(file.bucket.name), file_path(file)),
339
+ :headers => Smash.new(
340
+ 'Host' => aws_host
341
+ ),
342
+ :params => Smash.new(
343
+ 'X-Amz-Date' => Contrib::AwsApiCore.time_iso8601,
344
+ 'X-Amz-Expires' => timeout_secs
345
+ )
346
+ )
347
+ else
348
+ raise Error::ModelPersistError.new "#{file} has not been saved!"
349
+ end
350
+ end
351
+
352
+ # Fetch the contents of the file
353
+ #
354
+ # @param file [Models::Storage::File]
355
+ # @return [IO, HTTP::Response::Body]
356
+ def file_body(file)
357
+ if(file.persisted?)
358
+ result = request(
359
+ :path => file_path(file),
360
+ :endpoint => bucket_endpoint(file.bucket),
361
+ :disable_body_extraction => true
362
+ )
363
+ content = result[:body]
364
+ begin
365
+ if(content.is_a?(String))
366
+ StringIO.new(content)
367
+ else
368
+ if(content.respond_to?(:stream!))
369
+ content.stream!
370
+ end
371
+ content
372
+ end
373
+ rescue HTTP::StateError
374
+ StringIO.new(content.to_s)
375
+ end
376
+ else
377
+ StringIO.new('')
378
+ end
379
+ end
380
+
381
+ # Simple callback to allow request option adjustments prior to
382
+ # signature calculation
383
+ #
384
+ # @param opts [Smash] request options
385
+ # @return [TrueClass]
386
+ # @note this only updates when :body is defined. if a :post is
387
+ # happening (which implicitly forces :form) or :json is used
388
+ # it will not properly checksum. (but that's probably okay)
389
+ def update_request(con, opts)
390
+ con.default_headers['x-amz-content-sha256'] = Digest::SHA256.
391
+ hexdigest(opts.fetch(:body, ''))
392
+ true
393
+ end
394
+
395
+ # @return [String] escaped file path
396
+ def file_path(file)
397
+ file.name.split('/').map do |part|
398
+ uri_escape(part)
399
+ end.join('/')
400
+ end
401
+
402
+ end
403
+ end
404
+ end
405
+ end
@@ -0,0 +1,15 @@
1
+ $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
2
+ require 'miasma-aws/version'
3
+ Gem::Specification.new do |s|
4
+ s.name = 'miasma-aws'
5
+ s.version = MiasmaAws::VERSION.version
6
+ s.summary = 'Smoggy AWS API'
7
+ s.author = 'Chris Roberts'
8
+ s.email = 'code@chrisroberts.org'
9
+ s.homepage = 'https://github.com/miasma-rb/miasma-aws'
10
+ s.description = 'Smoggy AWS API'
11
+ s.license = 'Apache 2.0'
12
+ s.require_path = 'lib'
13
+ s.add_dependency 'miasma'
14
+ s.files = Dir['lib/**/*'] + %w(miasma-aws.gemspec README.md CHANGELOG.md LICENSE)
15
+ end
metadata ADDED
@@ -0,0 +1,70 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: miasma-aws
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Chris Roberts
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-01-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: miasma
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ description: Smoggy AWS API
28
+ email: code@chrisroberts.org
29
+ executables: []
30
+ extensions: []
31
+ extra_rdoc_files: []
32
+ files:
33
+ - CHANGELOG.md
34
+ - LICENSE
35
+ - README.md
36
+ - lib/miasma-aws.rb
37
+ - lib/miasma-aws/version.rb
38
+ - lib/miasma/contrib/aws.rb
39
+ - lib/miasma/contrib/aws/auto_scale.rb
40
+ - lib/miasma/contrib/aws/compute.rb
41
+ - lib/miasma/contrib/aws/load_balancer.rb
42
+ - lib/miasma/contrib/aws/orchestration.rb
43
+ - lib/miasma/contrib/aws/storage.rb
44
+ - miasma-aws.gemspec
45
+ homepage: https://github.com/miasma-rb/miasma-aws
46
+ licenses:
47
+ - Apache 2.0
48
+ metadata: {}
49
+ post_install_message:
50
+ rdoc_options: []
51
+ require_paths:
52
+ - lib
53
+ required_ruby_version: !ruby/object:Gem::Requirement
54
+ requirements:
55
+ - - ">="
56
+ - !ruby/object:Gem::Version
57
+ version: '0'
58
+ required_rubygems_version: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: '0'
63
+ requirements: []
64
+ rubyforge_project:
65
+ rubygems_version: 2.2.2
66
+ signing_key:
67
+ specification_version: 4
68
+ summary: Smoggy AWS API
69
+ test_files: []
70
+ has_rdoc: