miasma 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f4c2fc901adb9940a31de3c3379646e1225eea6
-  data.tar.gz: c45a82a7de2a1004a51a6c1a3b5fb10c60028088
+  metadata.gz: 70ab81fefcaa4ab8942ca15c53e1ad296a5d5938
+  data.tar.gz: b2042bdf52407d095767388854adad5d8783526a
 SHA512:
-  metadata.gz: 0e0147bc6672e477a407ff627ea6e37839cae93bc16c6f26499cd5f57a4069358a2e6849062d153bfd1b9940b23fe0568f0e5c6cf40b561be0bdfcd4476ff461
-  data.tar.gz: cdec65223d5ede929aad74fdf520b1a99c9a67a5d63a4559fff2cf414c5bad11eba6342da21fe9a5f27b89ff2f2b71488452e65123da99cac6a7add616902e74
+  metadata.gz: 6cb01229d765fc2f5d66ba690848af6dc1af060641e4ce4d0d3d791d27051d0a35511077a2dcfdc7ef3cc6e74aee3a395cf61703f1a507b38ee130061b409b49
+  data.tar.gz: f83fc149fb9b9bac850ef563a17eb133472d281a65ac6ce3db66c32da4bb4e12b5d367b761e1b040ad4f01305977067d07dc70cde63d9534752a6148f62fbd3f

data/CHANGELOG.md

@@ -1,2 +1,8 @@
+# v0.2.0
+* Add initial OpenStack provider support
+* Refactor of Rackspace provider support (build off OpenStack)
+* Finish defining storage modeling interfaces
+* Add initial storage provider implementation (AWS)
+
 # v0.1.0
 * Initial release

data/LICENSE

@@ -0,0 +1,13 @@
+   Copyright 2014 Chris Roberts
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -143,6 +143,7 @@ model completions.
 
 * AWS
 * Rackspace
+* OpenStack
 
 ### Models
 
@@ -150,7 +151,7 @@ model completions.
 
 |Model         |Create|Read|Update|Delete|
 |--------------|------|----|------|------|
-|AutoScale     |  X   |    |      |      |
+|AutoScale     |  X   | X  |      |      |
 |BlockStorage  |      |    |      |      |
 |Compute       |  X   | X  |      |  X   |
 |DNS           |      |    |      |      |
@@ -158,13 +159,13 @@ model completions.
 |Network       |      |    |      |      |
 |Orchestration |  X   | X  |  X   |  X   |
 |Queues        |      |    |      |      |
-|Storage       |      |    |      |      |
+|Storage       |  X   | X  |  X   |  X   |
 
 #### Rackspace
 
 |Model         |Create|Read|Update|Delete|
 |--------------|------|----|------|------|
-|AutoScale     |  X   |    |      |      |
+|AutoScale     |  X   | X  |      |      |
 |BlockStorage  |      |    |      |      |
 |Compute       |  X   | X  |      |  X   |
 |DNS           |      |    |      |      |
@@ -174,6 +175,20 @@ model completions.
 |Queues        |      |    |      |      |
 |Storage       |      |    |      |      |
 
+#### OpenStack
+
+|Model         |Create|Read|Update|Delete|
+|--------------|------|----|------|------|
+|AutoScale     |      |    |      |      |
+|BlockStorage  |      |    |      |      |
+|Compute       |  X   | X  |      |  X   |
+|DNS           |      |    |      |      |
+|LoadBalancer  |      |    |      |      |
+|Network       |      |    |      |      |
+|Orchestration |  X   | X  |  X   |  X   |
+|Queues        |      |    |      |      |
+|Storage       |      |    |      |      |
+
 ## Info
 
 * Repository: https://github.com/chrisroberts/miasma

data/lib/miasma/contrib/aws.rb

@@ -294,6 +294,7 @@ module Miasma
             attribute :aws_secret_access_key, String, :required => true
             attribute :aws_region, String, :required => true
             attribute :aws_host, String
+            attribute :aws_bucket_region, String
 
             # @return [Contrib::AwsApiCore::SignatureV4]
             attr_reader :signer
@@ -332,6 +333,11 @@ module Miasma
           )
         end
 
+        # @return [String] custom escape for aws compat
+        def uri_escape(string)
+          signer.safe_escape(string)
+        end
+
         # @return [HTTP] connection for requests (forces headers)
         def connection
           super.with_headers(
@@ -356,7 +362,7 @@ module Miasma
         def make_request(connection, http_method, request_args)
           dest, options = request_args
           path = URI.parse(dest).path
-          options = options.to_smash
+          options = options ? options.to_smash : Smash.new
           options[:params] = options.fetch(:params, Smash.new).to_smash.deep_merge('Version' => self.class::API_VERSION)
           if(http_method.to_sym == :post)
             if(options[:form])
@@ -365,6 +371,7 @@ module Miasma
               options[:form] = options.delete(:params)
             end
           end
+          update_request(connection, options)
           signature = signer.generate(
             http_method, path, options.merge(
               Smash.new(
@@ -378,6 +385,15 @@ module Miasma
           connection.auth(signature).send(http_method, dest, options)
         end
 
+        # Simple callback to allow request option adjustments prior to
+        # signature calculation
+        #
+        # @param opts [Smash] request options
+        # @return [TrueClass]
+        def update_request(con, opts)
+          true
+        end
+
       end
 
     end
@@ -387,4 +403,5 @@ module Miasma
   Models::LoadBalancer.autoload :Aws, 'miasma/contrib/aws/load_balancer'
   Models::AutoScale.autoload :Aws, 'miasma/contrib/aws/auto_scale'
   Models::Orchestration.autoload :Aws, 'miasma/contrib/aws/orchestration'
+  Models::Storage.autoload :Aws, 'miasma/contrib/aws/storage'
 end

data/lib/miasma/contrib/aws/orchestration.rb

@@ -74,8 +74,8 @@ module Miasma
               :name => stk['StackName'],
               :capabilities => [stk.get('Capabilities', 'member')].flatten(1).compact,
               :description => stk['Description'],
-              :creation_time => stk['CreationTime'],
-              :updated_time => stk['LastUpdatedTime'],
+              :created => stk['CreationTime'],
+              :updated => stk['LastUpdatedTime'],
               :notification_topics => [stk.get('NotificationARNs', 'member')].flatten(1).compact,
               :timeout_in_minutes => stk['TimeoutInMinutes'],
               :status => stk['StackStatus'],
@@ -265,7 +265,7 @@ module Miasma
               :state => res['ResourceStatus'].downcase.to_sym,
               :status => res['ResourceStatus'],
               :status_reason => res['ResourceStatusReason'],
-              :updated_time => res['Timestamp']
+              :updated => res['Timestamp']
             ).valid_state
           end
         end

data/lib/miasma/contrib/aws/storage.rb

@@ -0,0 +1,330 @@
+require 'stringio'
+require 'xmlsimple'
+require 'miasma'
+
+module Miasma
+  module Models
+    class Storage
+      class Aws < Storage
+
+        # Service name of the API
+        API_SERVICE = 's3'
+        # Supported version of the AutoScaling API
+        API_VERSION = '2006-03-01'
+
+        include Contrib::AwsApiCore::ApiCommon
+        include Contrib::AwsApiCore::RequestUtils
+
+        # Simple init override to force HOST and adjust region for
+        # signatures if required
+        def initialize(args)
+          args = args.to_smash
+          cache_region = args[:aws_region]
+          args[:aws_region] = args.fetch(:aws_bucket_region, 'us-east-1')
+          super(args)
+          aws_region = cache_region
+          if(aws_bucket_region)
+            self.aws_host = "s3-#{aws_bucket_region}.amazonaws.com"
+          else
+            self.aws_host = 's3.amazonaws.com'
+          end
+        end
+
+        # Save bucket
+        #
+        # @param bucket [Models::Storage::Bucket]
+        # @return [Models::Storage::Bucket]
+        def bucket_save(bucket)
+          unless(bucket.persisted?)
+            req_args = Smash.new(
+              :method => :put,
+              :path => '/',
+              :endpoint => bucket_endpoint(bucket)
+            )
+            if(aws_bucket_region)
+              req_args[:body] = XmlSimple.xml_out(
+                Smash.new(
+                  'CreateBucketConfiguration' => {
+                    'LocationConstraint' => aws_bucket_region
+                  }
+                ),
+                'AttrPrefix' => true,
+                'KeepRoot' => true
+              )
+              req_args[:headers] = Smash.new(
+                'Content-Length' => req_args[:body].length.to_s
+              )
+            end
+            request(req_args)
+            bucket.id = bucket.name
+            bucket.valid_state
+          end
+          bucket
+        end
+
+        # Destroy bucket
+        #
+        # @param bucket [Models::Storage::Bucket]
+        # @return [TrueClass, FalseClass]
+        def bucket_destroy(bucket)
+          if(bucket.persisted?)
+            request(
+              :path => '/',
+              :method => :delete,
+              :endpoint => bucket_endpoint(bucket),
+              :expects => 204
+            )
+            true
+          else
+            false
+          end
+        end
+
+        # Reload the bucket
+        #
+        # @param bucket [Models::Storage::Bucket]
+        # @return [Models::Storage::Bucket]
+        def bucket_reload(bucket)
+          if(bucket.persisted?)
+            begin
+              result = request(
+                :path => '/',
+                :method => :head,
+                :endpoint => bucket_endpoint(bucket)
+              )
+            rescue Error::ApiError::RequestError => e
+              if(e.response.status == 404)
+                bucket.data.clear
+                bucket.dirty.clear
+              else
+                raise
+              end
+            end
+          end
+          bucket
+        end
+
+        # Custom bucket endpoint
+        #
+        # @param bucket [Models::Storage::Bucket]
+        # @return [String]
+        # @todo properly escape bucket name
+        def bucket_endpoint(bucket)
+          ::File.join(endpoint, bucket.name)
+        end
+
+        # Return all buckets
+        #
+        # @return [Array<Models::Storage::Bucket>]
+        def bucket_all
+          result = request(:path => '/')
+          [result.get(:body, 'ListAllMyBucketsResult', 'Buckets', 'Bucket')].flatten.compact.map do |bkt|
+            Bucket.new(
+              self,
+              :id => bkt['Name'],
+              :name => bkt['Name'],
+              :created => bkt['CreationDate']
+            ).valid_state
+          end
+        end
+
+        # Return all files within bucket
+        #
+        # @param bucket [Bucket]
+        # @return [Array<File>]
+        def file_all(bucket)
+          result = request(
+            :path => '/',
+            :endpoint => bucket_endpoint(bucket)
+          )
+          [result.get(:body, 'ListBucketResult', 'Contents')].flatten.compact.map do |file|
+            File.new(
+              bucket,
+              :id => ::File.join(bucket.name, file['Key']),
+              :name => file['Key'],
+              :updated => file['LastModified'],
+              :size => file['Size'].to_i
+            ).valid_state
+          end
+        end
+
+        # Save file
+        #
+        # @param file [Models::Storage::File]
+        # @return [Models::Storage::File]
+        def file_save(file)
+          if(file.dirty?)
+            file.load_data(file.attributes)
+            args = Smash.new
+            args[:headers] = Smash[
+              Smash.new(
+                :content_type => 'Content-Type',
+                :content_disposition => 'Content-Disposition',
+                :content_encoding => 'Content-Encoding'
+              ).map do |attr, key|
+                if(file.attributes[attr])
+                  [key, file.attributes[attr]]
+                end
+              end.compact
+            ]
+            if(file.attributes[:body].is_a?(IO) && file.body.length >= 102400)
+              upload_id = request(
+                args.merge(
+                  Smash.new(
+                    :path => uri_escape(file.name),
+                    :endpoint => bucket_endpoint(bucket),
+                    :params => {
+                      :uploads => true
+                    }
+                  )
+                )
+              ).get(:body, 'InitiateMultipartUploadResult', 'UploadId')
+              count = 1
+              parts = []
+              file.body.rewind
+              while(content = file.body.read(102400))
+                parts << [
+                  count,
+                  request(
+                    :method => :put,
+                    :path => uri_escape(file.name),
+                    :endpoint => bucket_endpoint(bucket),
+                    :headers => Smash.new(
+                      'Content-Length' => content.size,
+                      'Content-MD5' => Digest::MD5.hexdigest(content)
+                    ),
+                    :params => Smash.new(
+                      'partNumber' => count,
+                      'uploadId' => upload_id
+                    ),
+                    :body => content
+                  ).get(:body, :headers, :etag)
+                ]
+                count += 1
+              end
+              complete = SimpleXml.xml_out(
+                Smash.new(
+                  'CompleteMultipartUpload' => {
+                    'Part' => parts.map{|part|
+                      {'PartNumber' => part.first, 'ETag' => part.last}
+                    }
+                  }
+                ),
+                'AttrPrefix' => true,
+                'KeepRoot' => true
+              )
+              result = request(
+                :method => :post,
+                :path => uri_escape(file.name),
+                :endpoint => bucket_endpoint(file.bucket),
+                :params => Smash.new(
+                  'UploadId' => upload_id
+                ),
+                :headers => Smash.new(
+                  'Content-Length' => complete.size
+                ),
+                :body => complete
+              )
+              file.etag = result.get(:body, 'CompleteMultipartUploadResult', 'ETag')
+            else
+              if(file.attributes[:body].is_a?(IO) || file.attributes[:body].is_a?(StringIO))
+                args[:headers]['Content-Length'] = file.body.length.to_s
+                file.body.rewind
+                args[:body] = file.body.read
+                file.body.rewind
+              end
+              result = request(
+                args.merge(
+                  Smash.new(
+                    :method => :put,
+                    :path => uri_escape(file.name),
+                    :endpoint => bucket_endpoint(file.bucket)
+                  )
+                )
+              )
+              file.etag = result.get(:headers, :etag)
+            end
+            file.id = ::File.join(file.bucket.name, file.name)
+            file.valid_state
+          end
+          file
+        end
+
+        # Destroy file
+        #
+        # @param file [Models::Storage::File]
+        # @return [TrueClass, FalseClass]
+        def file_destroy(file)
+          if(file.persisted?)
+            request(
+              :method => :delete,
+              :path => file.name,
+              :endpoint => bucket_endpoint(file.bucket),
+              :expect => 204
+            )
+            true
+          else
+            false
+          end
+        end
+
+        # Reload the file
+        #
+        # @param file [Models::Storage::File]
+        # @return [Models::Storage::File]
+        def file_reload(file)
+          if(file.persisted?)
+            name = file.name
+            result = request(
+              :path => uri_escape(file.name),
+              :endpoint => bucket_endpoint(file.bucket)
+            )
+            file.data.clear && file.dirty.clear
+            info = result[:headers]
+            file.load_data(
+              :id => ::File.join(file.bucket.name, name),
+              :name => name,
+              :updated => info[:last_modified],
+              :etag => info[:etag],
+              :size => info[:content_length].to_i,
+              :content_type => info[:content_type]
+            ).valid_state
+          end
+          file
+        end
+
+        # Fetch the contents of the file
+        #
+        # @param file [Models::Storage::File]
+        # @return [IO, HTTP::Response::Body]
+        def file_body(file)
+          if(file.persisted?)
+            result = request(
+              :path => uri_escape(file.name),
+              :endpoint => bucket_endpoint(file.bucket)
+            )
+            content = result[:body]
+            content.is_a?(String) ? StringIO.new(content) : content
+          else
+            StringIO.new('')
+          end
+        end
+
+        # Simple callback to allow request option adjustments prior to
+        # signature calculation
+        #
+        # @param opts [Smash] request options
+        # @return [TrueClass]
+        # @note this only updates when :body is defined. if a :post is
+        # happening (which implicitly forces :form) or :json is used
+        # it will not properly checksum. (but that's probably okay)
+        def update_request(con, opts)
+          con.default_headers['x-amz-content-sha256'] = Digest::SHA256.
+            hexdigest(opts.fetch(:body, ''))
+          true
+        end
+
+      end
+    end
+  end
+end