fog 1.5.0 → 1.6.0

data/lib/fog/atmos/storage.rb

@@ -0,0 +1,186 @@
+require 'fog/atmos'
+require 'fog/storage'
+
+module Fog
+  module Storage
+    class Atmos < Fog::Service
+      requires :atmos_storage_endpoint,
+               :atmos_storage_secret,
+               :atmos_storage_token
+      recognizes :persistent
+
+      model_path 'fog/atmos/models/storage'
+      model       :directory
+      collection  :directories
+      model       :file
+      collection  :files
+
+      request_path 'fog/atmos/requests/storage'
+      # request :delete_container
+      request :get_namespace
+      request :head_namespace
+      request :post_namespace
+      request :put_namespace
+      request :delete_namespace
+
+      module Utils
+        ENDPOINT_REGEX = /(https*):\/\/([a-zA-Z0-9\.\-]+)(:[0-9]+)?(\/.*)?/
+
+        def ssl?
+          protocol = @endpoint.match(ENDPOINT_REGEX)[1]
+          raise ArgumentError, 'Invalid endpoint URL' if protocol.nil?
+
+          return true if protocol == 'https'
+          return false if protocol == 'http'
+
+          raise ArgumentError, "Unknown protocol #{protocol}"
+        end
+
+        def port
+          port = @endpoint.match(ENDPOINT_REGEX)[3]
+          return ssl? ? 443 : 80 if port.nil?
+          port.split(':')[1].to_i
+        end
+
+        def host
+          @endpoint.match(ENDPOINT_REGEX)[2]
+        end
+
+        def api_path
+          @endpoint.match(ENDPOINT_REGEX)[4]
+        end
+
+        def setup_credentials(options)
+          @storage_token = options[:atmos_storage_token]
+          @storage_secret = options[:atmos_storage_secret]
+          @storage_secret_decoded = Base64.decode64(@storage_secret)
+          @endpoint = options[:atmos_storage_endpoint]
+          @prefix = self.ssl? ? 'https' : 'http'
+          @storage_host = self.host
+          @storage_port = self.port
+          @api_path = self.api_path
+        end
+      end
+
+      class Mock
+        include Utils
+
+        def initialize(options={})
+          require 'mime/types'
+          setup_credentials(options)
+        end
+
+        def request(options)
+          raise "Atmos Storage mocks not implemented"
+        end
+
+      end
+
+      class Real
+        include Utils
+
+        def initialize(options={})
+          require 'mime/types'
+
+          setup_credentials(options)
+          @connection_options = options[:connection_options] || {}
+          @hmac               = Fog::HMAC.new('sha1', @storage_secret_decoded)
+          @persistent = options.fetch(:persistent, false)
+
+          @connection = Fog::Connection.new("#{@prefix}://#{@storage_host}:#{@storage_port}",
+              @persistent, @connection_options)
+        end
+
+        def uid
+          @storage_token#.split('/')[-1]
+        end
+
+        def sign(string)
+          value = @hmac.sign(string)
+          Base64.encode64( value ).chomp()
+        end
+
+        def reload
+          @connection.reset
+        end
+
+        def request(params, &block)
+          req_path = params[:path]
+          # Force set host and port
+          params.merge!({
+                          :host     => @storage_host,
+                          :path     => "#{@api_path}/rest/#{params[:path]}",
+                        })
+          # Set default method and headers
+          params = {:method => 'GET', :headers => {}}.merge params
+
+          params[:headers]["Content-Type"] ||= "application/octet-stream"
+
+          # Add request date
+          params[:headers]["date"] = Time.now().httpdate()
+          params[:headers]["x-emc-uid"] = @storage_token
+
+          # Build signature string
+          signstring = ""
+          signstring += params[:method]
+          signstring += "\n"
+          signstring += params[:headers]["Content-Type"]
+          signstring += "\n"
+          if( params[:headers]["range"] )
+            signstring += params[:headers]["range"]
+          end
+          signstring += "\n"
+          signstring += params[:headers]["date"]
+          signstring += "\n"
+
+          signstring += "/rest/" + URI.unescape( req_path ).downcase
+          query_str = params[:query].map{|k,v| "#{k}=#{v}"}.join('&')
+          signstring += '?' + query_str unless query_str.empty?
+          signstring += "\n"
+
+          customheaders = {}
+          params[:headers].each { |key,value|
+            case key
+            when 'x-emc-date', 'x-emc-signature'
+              #skip
+            when /^x-emc-/
+              customheaders[ key.downcase ] = value
+            end
+          }
+          header_arr = customheaders.sort()
+
+          header_arr.each { |key,value|
+            # Values are lowercase and whitespace-normalized
+            signstring += key + ":" + value.strip.chomp.squeeze( " " ) + "\n"
+          }
+
+          digest = @hmac.sign(signstring.chomp())
+          signature = Base64.encode64( digest ).chomp()
+          params[:headers]["x-emc-signature"] = signature
+
+          begin
+            response = @connection.request(params, &block)
+          rescue Excon::Errors::HTTPStatusError => error
+            raise case error
+            when Excon::Errors::NotFound
+              Fog::Storage::Atmos::NotFound.slurp(error)
+            else
+              error
+            end
+          end
+          unless response.body.empty?
+            if params[:parse]
+              document = Fog::ToHashDocument.new
+              parser = Nokogiri::XML::SAX::PushParser.new(document)
+              parser << response.body
+              parser.finish
+              response.body = document.body
+            end
+          end
+          response
+        end
+
+      end
+    end
+  end
+end

data/lib/fog/aws.rb

@@ -1,5 +1,6 @@
 require 'fog/core'
 require 'fog/aws/credential_fetcher'
+require 'fog/aws/signaturev4'
 module Fog
   module AWS
 
@@ -16,6 +17,7 @@ module Fog
     service(:elasticache,     'aws/elasticache',      'Elasticache')
     service(:elb,             'aws/elb',              'ELB')
     service(:emr,             'aws/emr',              'EMR')
+    service(:glacier,         'aws/glacier',          'Glacier')
     service(:iam,             'aws/iam',              'IAM')
     service(:rds,             'aws/rds',              'RDS')
     service(:ses,             'aws/ses',              'SES')

data/lib/fog/aws/auto_scaling.rb

@@ -10,7 +10,7 @@ module Fog
       class ValidationError < Fog::Errors::Error; end
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :host, :path, :port, :scheme, :persistent, :region, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at
+      recognizes :host, :path, :port, :scheme, :persistent, :region, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
 
       request_path 'fog/aws/requests/auto_scaling'
       request :create_auto_scaling_group
@@ -79,6 +79,10 @@ module Fog
           setup_credentials(options)
 
           @connection_options = options[:connection_options] || {}
+
+          @instrumentor           = options[:instrumentor]
+          @instrumentor_name      = options[:instrumentor_name] || 'fog.aws.auto_scaling'
+          
           options[:region] ||= 'us-east-1'
           @host = options[:host] || "autoscaling.#{options[:region]}.amazonaws.com"
           @path       = options[:path]        || '/'
@@ -113,6 +117,16 @@ module Fog
             }
           )
 
+          if @instrumentor
+            @instrumentor.instrument("#{@instrumentor_name}.request", params) do
+              _request(body, idempotent, parser)
+            end
+          else
+            _request(body, idempotent, parser)
+          end
+        end
+
+        def _request(body, idempotent, parser)
           begin
             response = @connection.request({
               :body       => body,

data/lib/fog/aws/compute.rb

@@ -7,7 +7,7 @@ module Fog
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at
+      recognizes :endpoint, :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :version
 
       secrets    :aws_secret_access_key, :hmac, :aws_session_token
 
@@ -315,6 +315,9 @@ module Fog
           setup_credentials(options)
           @connection_options     = options[:connection_options] || {}
           @region                 = options[:region] ||= 'us-east-1'
+          @instrumentor           = options[:instrumentor]
+          @instrumentor_name      = options[:instrumentor_name] || 'fog.aws.compute'
+          @version                = options[:version]     ||  '2012-07-20'
 
           if @endpoint = options[:endpoint]
             endpoint = URI.parse(@endpoint)
@@ -360,12 +363,21 @@ module Fog
               :host               => @host,
               :path               => @path,
               :port               => @port,
-              :version            => '2012-06-01'
+              :version            => @version
             }
           )
 
-          begin
-            response = @connection.request({
+          if @instrumentor
+            @instrumentor.instrument("#{@instrumentor_name}.request", params) do
+              _request(body, idempotent, parser)
+            end
+          else
+            _request(body, idempotent, parser)
+          end
+        end
+
+        def _request(body, idempotent, parser)
+          @connection.request({
               :body       => body,
               :expects    => 200,
               :headers    => { 'Content-Type' => 'application/x-www-form-urlencoded' },
@@ -374,20 +386,17 @@ module Fog
               :method     => 'POST',
               :parser     => parser
             })
-          rescue Excon::Errors::HTTPStatusError => error
-            if match = error.message.match(/<Code>(.*)<\/Code><Message>(.*)<\/Message>/)
-              raise case match[1].split('.').last
-              when 'NotFound', 'Unknown'
-                Fog::Compute::AWS::NotFound.slurp(error, match[2])
-              else
-                Fog::Compute::AWS::Error.slurp(error, "#{match[1]} => #{match[2]}")
-              end
-            else
-              raise error
-            end
+        rescue Excon::Errors::HTTPStatusError => error
+          if match = error.message.match(/<Code>(.*)<\/Code><Message>(.*)<\/Message>/)
+            raise case match[1].split('.').last
+                  when 'NotFound', 'Unknown'
+                    Fog::Compute::AWS::NotFound.slurp(error, match[2])
+                  else
+                    Fog::Compute::AWS::Error.slurp(error, "#{match[1]} => #{match[2]}")
+                  end
+          else
+            raise error
           end
-
-          response
         end
 
       end

data/lib/fog/aws/elb.rb

@@ -16,7 +16,7 @@ module Fog
       class ValidationError             < Fog::Errors::Error; end
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at
+      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
 
       request_path 'fog/aws/requests/elb'
       request :configure_health_check
@@ -125,7 +125,9 @@ module Fog
           @use_iam_profile = options[:use_iam_profile]
           setup_credentials(options)
           @connection_options     = options[:connection_options] || {}
-          
+          @instrumentor           = options[:instrumentor]
+          @instrumentor_name      = options[:instrumentor_name] || 'fog.aws.elb'
+
           options[:region] ||= 'us-east-1'
           @host = options[:host] || "elasticloadbalancing.#{options[:region]}.amazonaws.com"
           @path       = options[:path]        || '/'
@@ -169,6 +171,16 @@ module Fog
           }
           )
 
+          if @instrumentor
+            @instrumentor.instrument("#{@instrumentor_name}.request", params) do
+              _request(body, idempotent, parser)
+            end
+          else
+            _request(body, idempotent, parser)
+          end
+        end
+
+        def _request(body, idempotent, parser)
           @connection.request({
             :body       => body,
             :expects    => 200,

data/lib/fog/aws/glacier.rb

@@ -0,0 +1,172 @@
+require 'fog/aws'
+
+module Fog
+  module AWS
+    class Glacier < Fog::Service
+      extend Fog::AWS::CredentialFetcher::ServiceMethods
+      
+      requires :aws_access_key_id, :aws_secret_access_key
+      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at
+
+      request_path 'fog/aws/requests/glacier'
+
+      request :abort_multipart_upload
+      request :complete_multipart_upload
+      request :create_archive
+      request :create_vault
+      request :delete_archive
+      request :delete_vault
+      request :delete_vault_notification_configuration
+      request :describe_job
+      request :describe_vault
+      request :get_job_output
+      request :get_vault_notification_configuration
+      request :initiate_job
+      request :initiate_multipart_upload
+      request :list_jobs
+      request :list_multipart_uploads
+      request :list_parts
+      request :list_vaults
+      request :set_vault_notification_configuration
+      request :upload_part
+
+      model_path 'fog/aws/models/glacier'
+      model      :vault
+      collection :vaults
+
+      MEGABYTE = 1024*1024
+
+      class TreeHash
+
+        def self.digest(body)
+          new.add_part(body)
+        end
+
+        def reduce_digests(digests)
+          while digests.length > 1
+            digests = digests.each_slice(2).collect do |pair|
+              if pair.length == 2
+                Digest::SHA256.digest(pair[0]+pair[1])
+              else
+                pair.first
+              end
+            end
+          end
+          digests.first
+        end
+
+        def initialize
+          @digests = []
+        end
+
+        def add_part(bytes)
+          part = self.digest_for_part(bytes)
+          @digests << part
+          part.unpack('H*').first
+        end
+
+        def digest_for_part(body)
+          chunk_count = [body.bytesize / MEGABYTE + (body.bytesize % MEGABYTE > 0 ? 1 : 0), 1].max
+          if body.respond_to? :byteslice
+            digests_for_part = chunk_count.times.collect {|chunk_index| Digest::SHA256.digest(body.byteslice(chunk_index * MEGABYTE, MEGABYTE))}
+          else
+            if body.respond_to? :encoding
+              old_encoding = body.encoding
+              body.force_encoding('BINARY')
+            end
+            digests_for_part = chunk_count.times.collect {|chunk_index| Digest::SHA256.digest(body.slice(chunk_index * MEGABYTE, MEGABYTE))}
+            if body.respond_to? :encoding
+              body.force_encoding(old_encoding)
+            end
+          end
+          reduce_digests(digests_for_part)
+        end
+
+        def hexdigest
+          digest.unpack('H*').first
+        end
+
+        def digest
+          reduce_digests(@digests)
+        end
+      end
+
+      class Mock
+
+        def initialize(options={})
+          Fog::Mock.not_implemented
+        end
+
+      end
+
+      class Real
+        include Fog::AWS::CredentialFetcher::ConnectionMethods
+        # Initialize connection to Glacier
+        #
+        # ==== Notes
+        # options parameter must include values for :aws_access_key_id and
+        # :aws_secret_access_key in order to create a connection
+        #
+        # ==== Examples
+        #   ses = SES.new(
+        #    :aws_access_key_id => your_aws_access_key_id,
+        #    :aws_secret_access_key => your_aws_secret_access_key
+        #   )
+        #
+        # ==== Parameters
+        # * options<~Hash> - config arguments for connection.  Defaults to {}.
+        #   * region<~String> - optional region to use. For instance, 'us-east-1' and etc.
+        #
+        # ==== Returns
+        # * Glacier object with connection to AWS.
+        def initialize(options={})
+
+          @use_iam_profile = options[:use_iam_profile]
+          @region = options[:region] || 'us-east-1'
+
+          setup_credentials(options)
+
+          @connection_options     = options[:connection_options] || {}
+          @host = options[:host] || "glacier.#{@region}.amazonaws.com"
+          @version = '2012-06-01'
+          @path       = options[:path]        || '/'
+          @persistent = options[:persistent]  || false
+          @port       = options[:port]        || 443
+          @scheme     = options[:scheme]      || 'https'
+
+          @connection = Fog::Connection.new("#{@scheme}://#{@host}:#{@port}#{@path}", @persistent, @connection_options)
+        end
+
+
+        private
+        def setup_credentials(options)
+          @aws_access_key_id      = options[:aws_access_key_id]
+          @aws_secret_access_key  = options[:aws_secret_access_key]
+          @aws_session_token      = options[:aws_session_token]
+          @aws_credentials_expire_at = options[:aws_credentials_expire_at]
+
+          @signer = Fog::AWS::SignatureV4.new( @aws_access_key_id, @aws_secret_access_key,@region,'glacier')
+        end
+
+        def request(params, &block)
+          refresh_credentials_if_expired
+
+          date = Fog::Time.now
+          params[:headers]['Date'] = date.to_date_header
+          params[:headers]['x-amz-date'] = date.to_iso8601_basic
+
+          params[:headers]['Host'] = @host
+          params[:headers]['x-amz-glacier-version'] = @version
+          params[:headers]['x-amz-security-token'] = @aws_session_token if @aws_session_token
+          params[:headers]['Authorization'] = @signer.sign params, date
+
+          response = @connection.request(params, &block)
+          if response.headers['Content-Type'] == 'application/json' && response.body.size > 0 #body will be empty if the streaming form has been used
+            response.body  = Fog::JSON.decode(response.body)
+          end
+          response
+        end
+      end
+    end
+  end
+end