fog-aws 0.0.5

data/lib/fog/aws/core.rb

@@ -0,0 +1,343 @@
+require 'fog/core'
+require 'fog/xml'
+require 'fog/json'
+require 'fog/aws/credential_fetcher'
+require 'fog/aws/region_methods'
+require 'fog/aws/signaturev4'
+
+module Fog
+  module AWS
+    extend Fog::Provider
+
+    service(:auto_scaling,    'AutoScaling')
+    service(:beanstalk,       'ElasticBeanstalk')
+    service(:cdn,             'CDN')
+    service(:compute,         'Compute')
+    service(:cloud_formation, 'CloudFormation')
+    service(:cloud_watch,     'CloudWatch')
+    service(:data_pipeline,   'DataPipeline')
+    service(:dynamodb,        'DynamoDB')
+    service(:dns,             'DNS')
+    service(:elasticache,     'Elasticache')
+    service(:elb,             'ELB')
+    service(:emr,             'EMR')
+    service(:glacier,         'Glacier')
+    service(:iam,             'IAM')
+    service(:rds,             'RDS')
+    service(:redshift,        'Redshift')
+    service(:ses,             'SES')
+    service(:simpledb,        'SimpleDB')
+    service(:sns,             'SNS')
+    service(:sqs,             'SQS')
+    service(:sts,             'STS')
+    service(:storage,         'Storage')
+
+    def self.indexed_param(key, values)
+      params = {}
+      unless key.include?('%d')
+        key << '.%d'
+      end
+      [*values].each_with_index do |value, index|
+        if value.respond_to?('keys')
+          k = format(key, index + 1)
+          value.each do | vkey, vvalue |
+            params["#{k}.#{vkey}"] = vvalue
+          end
+        else
+          params[format(key, index + 1)] = value
+        end
+      end
+      params
+    end
+
+    def self.serialize_keys(key, value, options = {})
+      case value
+      when Hash
+        value.each do | k, v |
+          options.merge!(serialize_keys("#{key}.#{k}", v))
+        end
+        return options
+      when Array
+        value.each_with_index do | it, idx |
+          options.merge!(serialize_keys("#{key}.member.#{(idx + 1)}", it))
+        end
+        return options
+      else
+        return {key => value}
+      end
+    end
+
+    def self.indexed_request_param(name, values)
+      idx = -1
+      Array(values).reduce({}) do |params, value|
+        params["#{name}.#{idx += 1}"] = value
+        params
+      end
+    end
+
+    def self.indexed_filters(filters)
+      params = {}
+      filters.keys.each_with_index do |key, key_index|
+        key_index += 1
+        params[format('Filter.%d.Name', key_index)] = key
+        [*filters[key]].each_with_index do |value, value_index|
+          value_index += 1
+          params[format('Filter.%d.Value.%d', key_index, value_index)] = value
+        end
+      end
+      params
+    end
+
+    def self.escape(string)
+      string.gsub(/([^a-zA-Z0-9_.\-~]+)/) {
+        "%" + $1.unpack("H2" * $1.bytesize).join("%").upcase
+      }
+    end
+
+    def self.signed_params_v4(params, headers, options={})
+      date = Fog::Time.now
+
+      params = params.merge('Version' => options[:version])
+
+      headers = headers.merge('Host' => options[:host], 'x-amz-date' => date.to_iso8601_basic)
+      headers['x-amz-security-token'] = options[:aws_session_token] if options[:aws_session_token]
+
+      body = ''
+      for key in params.keys.sort
+        unless (value = params[key]).nil?
+          body << "#{key}=#{escape(value.to_s)}&"
+        end
+      end
+      body.chop!
+      
+      headers['Authorization'] = options[:signer].sign({:method => options[:method], :headers => headers, :body => body, :query => {}, :path => options[:path]}, date)
+
+      return body, headers
+    end
+
+    def self.signed_params(params, options = {})
+      params.merge!({
+        'AWSAccessKeyId'    => options[:aws_access_key_id],
+        'SignatureMethod'   => 'HmacSHA256',
+        'SignatureVersion'  => '2',
+        'Timestamp'         => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        'Version'           => options[:version]
+      })
+
+      params.merge!({
+        'SecurityToken'     => options[:aws_session_token]
+      }) if options[:aws_session_token]
+
+      body = ''
+      for key in params.keys.sort
+        unless (value = params[key]).nil?
+          body << "#{key}=#{escape(value.to_s)}&"
+        end
+      end
+      string_to_sign = "POST\n#{options[:host]}:#{options[:port]}\n#{options[:path]}\n" << body.chop
+      signed_string = options[:hmac].sign(string_to_sign)
+      body << "Signature=#{escape(Base64.encode64(signed_string).chomp!)}"
+
+      body
+    end
+
+    class Mock
+      def self.arn(vendor, account_id, path, region = nil)
+        "arn:aws:#{vendor}:#{region}:#{account_id}:#{path}"
+      end
+
+      def self.availability_zone(region)
+        "#{region}#{Fog::Mock.random_selection('abcd', 1)}"
+      end
+
+      def self.box_usage
+        sprintf("%0.10f", rand / 100).to_f
+      end
+
+      def self.console_output
+        # "[ 0.000000] Linux version 2.6.18-xenU-ec2-v1.2 (root@domU-12-31-39-07-51-82) (gcc version 4.1.2 20070626 (Red Hat 4.1.2-13)) #2 SMP Wed Aug 19 09:04:38 EDT 2009"
+        Base64.decode64("WyAwLjAwMDAwMF0gTGludXggdmVyc2lvbiAyLjYuMTgteGVuVS1lYzItdjEu\nMiAocm9vdEBkb21VLTEyLTMxLTM5LTA3LTUxLTgyKSAoZ2NjIHZlcnNpb24g\nNC4xLjIgMjAwNzA2MjYgKFJlZCBIYXQgNC4xLjItMTMpKSAjMiBTTVAgV2Vk\nIEF1ZyAxOSAwOTowNDozOCBFRFQgMjAwOQ==\n")
+      end
+
+      def self.dns_name_for(ip_address)
+        "ec2-#{ip_address.gsub('.','-')}.compute-1.amazonaws.com"
+      end
+
+      def self.private_dns_name_for(ip_address)
+        "ip-#{ip_address.gsub('.','-')}.ec2.internal"
+      end
+
+      def self.image
+        path = []
+        (rand(3) + 2).times do
+          path << Fog::Mock.random_letters(rand(9) + 8)
+        end
+        {
+          "imageOwnerId"   => Fog::Mock.random_letters(rand(5) + 4),
+          "blockDeviceMapping" => [],
+          "productCodes"   => [],
+          "kernelId"       => kernel_id,
+          "ramdiskId"      => ramdisk_id,
+          "imageState"     => "available",
+          "imageId"        => image_id,
+          "architecture"   => "i386",
+          "isPublic"       => true,
+          "imageLocation"  => path.join('/'),
+          "imageType"      => "machine",
+          "rootDeviceType" => ["ebs","instance-store"][rand(2)],
+          "rootDeviceName" => "/dev/sda1"
+        }
+      end
+
+      def self.image_id
+        "ami-#{Fog::Mock.random_hex(8)}"
+      end
+
+      def self.key_fingerprint
+        fingerprint = []
+        20.times do
+          fingerprint << Fog::Mock.random_hex(2)
+        end
+        fingerprint.join(':')
+      end
+
+      def self.instance_id
+        "i-#{Fog::Mock.random_hex(8)}"
+      end
+
+      def self.ip_address
+        Fog::Mock.random_ip
+      end
+
+      def self.private_ip_address
+        ip_address.gsub(/^\d{1,3}\./,"10.")
+      end
+
+      def self.kernel_id
+        "aki-#{Fog::Mock.random_hex(8)}"
+      end
+
+      def self.key_material
+        OpenSSL::PKey::RSA.generate(1024).to_s
+      end
+
+      def self.owner_id
+        Fog::Mock.random_numbers(12)
+      end
+
+      def self.ramdisk_id
+        "ari-#{Fog::Mock.random_hex(8)}"
+      end
+
+      def self.request_id
+        request_id = []
+        request_id << Fog::Mock.random_hex(8)
+        3.times do
+          request_id << Fog::Mock.random_hex(4)
+        end
+        request_id << Fog::Mock.random_hex(12)
+        request_id.join('-')
+      end
+      class << self
+        alias_method :reserved_instances_id, :request_id
+        alias_method :reserved_instances_offering_id, :request_id
+        alias_method :sqs_message_id, :request_id
+        alias_method :sqs_sender_id, :request_id
+      end
+
+      def self.reservation_id
+        "r-#{Fog::Mock.random_hex(8)}"
+      end
+
+      def self.snapshot_id
+        "snap-#{Fog::Mock.random_hex(8)}"
+      end
+
+      def self.volume_id
+        "vol-#{Fog::Mock.random_hex(8)}"
+      end
+
+      def self.security_group_id
+        "sg-#{Fog::Mock.random_hex(8)}"
+      end
+
+      def self.network_acl_id
+        "acl-#{Fog::Mock.random_hex(8)}"
+      end
+      def self.network_acl_association_id
+        "aclassoc-#{Fog::Mock.random_hex(8)}"
+      end
+      def self.network_interface_id
+        "eni-#{Fog::Mock.random_hex(8)}"
+      end
+      def self.internet_gateway_id
+        "igw-#{Fog::Mock.random_hex(8)}"
+      end
+      def self.dhcp_options_id
+        "dopt-#{Fog::Mock.random_hex(8)}"
+      end
+      def self.vpc_id
+        "vpc-#{Fog::Mock.random_hex(8)}"
+      end
+      def self.subnet_id
+        "subnet-#{Fog::Mock.random_hex(8)}"
+      end
+      def self.zone_id
+        "zone-#{Fog::Mock.random_hex(8)}"
+      end
+      def self.change_id
+        Fog::Mock.random_letters_and_numbers(14)
+      end
+      def self.nameservers
+        [
+          'ns-2048.awsdns-64.com',
+          'ns-2049.awsdns-65.net',
+          'ns-2050.awsdns-66.org',
+          'ns-2051.awsdns-67.co.uk'
+        ]
+      end
+
+      def self.key_id(length=21)
+        #Probably close enough
+        Fog::Mock.random_selection('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789',length)
+      end
+
+      def self.rds_address(db_name,region)
+        "#{db_name}.#{Fog::Mock.random_letters(rand(12) + 4)}.#{region}.rds.amazonaws.com"
+      end
+    end
+
+    def self.parse_security_group_options(group_name, options)
+      options ||= Hash.new
+      if group_name.is_a?(Hash)
+        options = group_name
+      elsif group_name
+        if options.key?('GroupName')
+          raise Fog::Compute::AWS::Error, 'Arguments specified both group_name and GroupName in options'
+        end
+        options = options.clone
+        options['GroupName'] = group_name
+      end
+      name_specified = options.key?('GroupName') && !options['GroupName'].nil?
+      group_id_specified = options.key?('GroupId') && !options['GroupId'].nil?
+      unless name_specified || group_id_specified
+        raise Fog::Compute::AWS::Error, 'Neither GroupName nor GroupId specified'
+      end
+      if name_specified && group_id_specified
+        options.delete('GroupName')
+      end
+      options
+    end
+
+    module Errors
+      def self.match_error(error)
+        matcher = lambda {|s| s.match(/(?:.*<Code>(.*)<\/Code>)(?:.*<Message>(.*)<\/Message>)/m)}
+        [error.message, error.response.body].each(&Proc.new {|s|
+          match = matcher.call(s)
+          return {:code => match[1].split('.').last, :message => match[2]} if match
+        })
+        {} # we did not match the message or response body
+      end
+    end
+  end
+end

data/lib/fog/aws/credential_fetcher.rb

@@ -0,0 +1,63 @@
+require "fog/json"
+
+module Fog
+  module AWS
+    module CredentialFetcher
+      INSTANCE_METADATA_HOST = "http://169.254.169.254"
+      INSTANCE_METADATA_PATH = "/latest/meta-data/iam/security-credentials/"
+      module ServiceMethods
+        def fetch_credentials(options)
+          if options[:use_iam_profile]
+            begin
+              connection = options[:connection] || Excon.new(INSTANCE_METADATA_HOST)
+              role_name = connection.get(:path => INSTANCE_METADATA_PATH, :expects => 200).body
+              role_data = connection.get(:path => INSTANCE_METADATA_PATH+role_name, :expects => 200).body
+
+              session = Fog::JSON.decode(role_data)
+              credentials = {}
+              credentials[:aws_access_key_id] = session['AccessKeyId']
+              credentials[:aws_secret_access_key] = session['SecretAccessKey']
+              credentials[:aws_session_token] = session['Token']
+              credentials[:aws_credentials_expire_at] = Time.xmlschema session['Expiration']
+              #these indicate the metadata service is unavailable or has no profile setup
+              credentials
+            rescue Excon::Errors::Error => e
+              Fog::Logger.warning("Unable to fetch credentials: #{e.message}")
+              super
+            end
+          else
+            super
+          end
+        end
+      end
+
+      module ConnectionMethods
+        def refresh_credentials_if_expired
+          refresh_credentials if credentials_expired?
+        end
+
+        private
+
+        def credentials_expired?
+          @use_iam_profile &&
+            (!@aws_credentials_expire_at ||
+             (@aws_credentials_expire_at && Fog::Time.now > @aws_credentials_expire_at - 15)) #new credentials become available from around 5 minutes before expiration time
+        end
+
+        def refresh_credentials
+          if @use_iam_profile
+            new_credentials = service.fetch_credentials :use_iam_profile => @use_iam_profile
+            if new_credentials.any?
+              setup_credentials new_credentials
+              return true
+            else
+              false
+            end
+          else
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/data_pipeline.rb

@@ -0,0 +1,126 @@
+require 'fog/aws/core'
+
+module Fog
+  module AWS
+    class DataPipeline < Fog::Service
+      extend Fog::AWS::CredentialFetcher::ServiceMethods
+
+      requires :aws_access_key_id, :aws_secret_access_key
+      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
+
+      request_path 'fog/aws/requests/data_pipeline'
+      request :activate_pipeline
+      request :create_pipeline
+      request :delete_pipeline
+      request :describe_pipelines
+      request :list_pipelines
+      request :put_pipeline_definition
+      request :get_pipeline_definition
+      request :query_objects
+      request :describe_objects
+
+      model_path 'fog/aws/models/data_pipeline'
+      model       :pipeline
+      collection  :pipelines
+
+      class Mock
+        def initialize(options={})
+          Fog::Mock.not_implemented
+        end
+      end
+
+      class Real
+        attr_reader :region
+
+        include Fog::AWS::CredentialFetcher::ConnectionMethods
+        # Initialize connection to DataPipeline
+        #
+        # ==== Notes
+        # options parameter must include values for :aws_access_key_id and
+        # :aws_secret_access_key in order to create a connection
+        #
+        # ==== Examples
+        #   datapipeline = DataPipeline.new(
+        #    :aws_access_key_id => your_aws_access_key_id,
+        #    :aws_secret_access_key => your_aws_secret_access_key
+        #   )
+        #
+        # ==== Parameters
+        # * options<~Hash> - config arguments for connection.  Defaults to {}.
+        #   * region<~String> - optional region to use. For instance, 'eu-west-1', 'us-east-1' and etc.
+        #
+        # ==== Returns
+        # * DataPipeline object with connection to AWS.
+        def initialize(options={})
+          @use_iam_profile = options[:use_iam_profile]
+          @instrumentor       = options[:instrumentor]
+          @instrumentor_name  = options[:instrumentor_name] || 'fog.aws.data_pipeline'
+          @connection_options     = options[:connection_options] || {}
+          @version    = '2012-10-29'
+          @region     = options[:region]      || 'us-east-1'
+          @host       = options[:host]        || "datapipeline.#{@region}.amazonaws.com"
+          @path       = options[:path]        || '/'
+          @persistent = options[:persistent]  || false
+          @port       = options[:port]        || 443
+          @scheme     = options[:scheme]      || 'https'
+          @connection = Fog::XML::Connection.new("#{@scheme}://#{@host}:#{@port}#{@path}", @persistent, @connection_options)
+
+          setup_credentials(options)
+        end
+
+        def owner_id
+          @owner_id ||= security_groups.get('default').owner_id
+        end
+
+        def reload
+          @connection.reset
+        end
+
+        private
+
+        def setup_credentials(options)
+          @aws_access_key_id      = options[:aws_access_key_id]
+          @aws_secret_access_key  = options[:aws_secret_access_key]
+          @aws_session_token     = options[:aws_session_token]
+          @aws_credentials_expire_at = options[:aws_credentials_expire_at]
+
+          @signer = Fog::AWS::SignatureV4.new(@aws_access_key_id, @aws_secret_access_key, @region, 'datapipeline')
+        end
+
+        def request(params)
+          refresh_credentials_if_expired
+
+          # Params for all DataPipeline requests
+          params.merge!({
+            :expects => 200,
+            :method => :post,
+            :path => '/',
+          })
+
+          date = Fog::Time.now
+          params[:headers] = {
+            'Date' => date.to_date_header,
+            'Host' => @host,
+            'X-Amz-Date' => date.to_iso8601_basic,
+            'Content-Type' => 'application/x-amz-json-1.1',
+            'Content-Length' => params[:body].bytesize.to_s,
+          }.merge!(params[:headers] || {})
+          params[:headers]['x-amz-security-token'] = @aws_session_token if @aws_session_token
+          params[:headers]['Authorization'] = @signer.sign(params, date)
+
+          if @instrumentor
+            @instrumentor.instrument("#{@instrumentor_name}.request", params) do
+              _request(params)
+            end
+          else
+            _request(params)
+          end
+        end
+
+        def _request(params)
+          @connection.request(params)
+        end
+      end
+    end
+  end
+end