fog-aws 0.11.0 → 3.18.0

data/lib/fog/aws/compute.rb

@@ -1,10 +1,12 @@
 module Fog
-  module Compute
-    class AWS < Fog::Service
+  module AWS
+    class Compute < Fog::Service
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
+      class RequestLimitExceeded < Fog::Errors::Error; end
+
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :version
+      recognizes :endpoint, :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :version, :retry_request_limit_exceeded, :retry_jitter_magnitude
 
       secrets    :aws_secret_access_key, :hmac, :aws_session_token
 
@@ -54,6 +56,7 @@ module Fog
       request :attach_classic_link_vpc
       request :attach_internet_gateway
       request :attach_volume
+      request :authorize_security_group_egress
       request :authorize_security_group_ingress
       request :cancel_spot_instance_requests
       request :create_dhcp_options
@@ -98,6 +101,7 @@ module Fog
       request :describe_classic_link_instances
       request :describe_dhcp_options
       request :describe_images
+      request :describe_image_attribute
       request :describe_instances
       request :describe_instance_attribute
       request :describe_internet_gateways
@@ -119,28 +123,35 @@ module Fog
       request :describe_subnets
       request :describe_tags
       request :describe_volumes
+      request :describe_volumes_modifications
       request :describe_volume_status
       request :describe_vpcs
       request :describe_vpc_attribute
       request :describe_vpc_classic_link
+      request :describe_vpc_classic_link_dns_support
       request :detach_network_interface
       request :detach_internet_gateway
       request :detach_volume
       request :detach_classic_link_vpc
       request :disable_vpc_classic_link
+      request :disable_vpc_classic_link_dns_support
       request :disassociate_address
       request :disassociate_route_table
       request :enable_vpc_classic_link
+      request :enable_vpc_classic_link_dns_support
       request :get_console_output
       request :get_password_data
       request :import_key_pair
       request :modify_image_attribute
       request :modify_instance_attribute
+      request :modify_instance_placement
       request :modify_network_interface_attribute
       request :modify_snapshot_attribute
       request :modify_subnet_attribute
+      request :modify_volume
       request :modify_volume_attribute
       request :modify_vpc_attribute
+      request :move_address_to_vpc
       request :purchase_reserved_instances_offering
       request :reboot_instances
       request :release_address
@@ -150,6 +161,8 @@ module Fog
       request :register_image
       request :request_spot_instances
       request :reset_network_interface_attribute
+      request :restore_address_to_classic
+      request :revoke_security_group_egress
       request :revoke_security_group_ingress
       request :run_instances
       request :terminate_instances
@@ -175,15 +188,18 @@ module Fog
 
       class Mock
         MOCKED_TAG_TYPES = {
-          'ami' => 'image',
-          'i' => 'instance',
+          'acl'  => 'network_acl',
+          'ami'  => 'image',
+          'igw'  => 'internet_gateway',
+          'i'    => 'instance',
+          'rtb'  => 'route_table',
           'snap' => 'snapshot',
-          'vol' => 'volume',
-          'igw' => 'internet_gateway',
-          'acl' => 'network_acl',
-          'vpc' => 'vpc'
+          'vol'  => 'volume',
+          'vpc'  => 'vpc'
         }
 
+        VPC_BLANK_VALUE = 'none'
+
         include Fog::AWS::CredentialFetcher::ConnectionMethods
 
         def self.data
@@ -217,21 +233,24 @@ module Fog
                         'fromPort'    => -1,
                         'toPort'      => -1,
                         'ipProtocol'  => 'icmp',
-                        'ipRanges'    => []
+                        'ipRanges'    => [],
+                        'ipv6Ranges'  => []
                       },
                       {
                         'groups'      => [{'groupName' => 'default', 'userId' => owner_id, 'groupId' => security_group_id}],
                         'fromPort'    => 0,
                         'toPort'      => 65535,
                         'ipProtocol'  => 'tcp',
-                        'ipRanges'    => []
+                        'ipRanges'    => [],
+                        'ipv6Ranges'  => []
                       },
                       {
                         'groups'      => [{'groupName' => 'default', 'userId' => owner_id, 'groupId' => security_group_id}],
                         'fromPort'    => 0,
                         'toPort'      => 65535,
                         'ipProtocol'  => 'udp',
-                        'ipRanges'    => []
+                        'ipRanges'    => [],
+                        'ipv6Ranges'  => []
                       }
                     ],
                     'ownerId'             => owner_id
@@ -272,7 +291,7 @@ module Fog
                     "attributeName" => "supported-platforms"
                   },
                   {
-                    "values"        => ["none"],
+                    "values"        => [VPC_BLANK_VALUE],
                     "attributeName" => "default-vpc"
                   },
                   {
@@ -283,7 +302,9 @@ module Fog
                     "values"        => ["5"],
                     "attributeName" => "vpc-max-elastic-ips"
                   }
-                ]
+                ],
+                :spot_requests => {},
+                :volume_modifications => {}
               }
             end
           end
@@ -361,9 +382,82 @@ module Fog
           self.data[:account_attributes].find { |h| h["attributeName"] == "supported-platforms" }["values"] = values
         end
 
+        def default_vpc
+          vpc_id = describe_account_attributes.body["accountAttributeSet"].find{ |h| h["attributeName"] == "default-vpc" }["values"].first
+          vpc_id == VPC_BLANK_VALUE ? nil : vpc_id
+        end
+
+        def default_vpc=(value)
+          self.data[:account_attributes].find { |h| h["attributeName"] == "default-vpc" }["values"] = [value]
+        end
+
+        def setup_default_vpc!
+          return if default_vpc.present?
+
+          disable_ec2_classic
+
+          vpc_id = Fog::AWS::Mock.default_vpc_for(region)
+          self.default_vpc = vpc_id
+
+          data[:vpcs] << {
+            'vpcId' => vpc_id,
+            'state' => 'available',
+            'cidrBlock' => '172.31.0.0/16',
+            'dhcpOptionsId' => Fog::AWS::Mock.dhcp_options_id,
+            'tagSet' => {},
+            'instanceTenancy' => 'default',
+            'enableDnsSupport' => true,
+            'enableDnsHostnames' => true,
+            'isDefault' => true
+          }
+
+          internet_gateway_id = Fog::AWS::Mock.internet_gateway_id
+          data[:internet_gateways][internet_gateway_id] = {
+            'internetGatewayId' => internet_gateway_id,
+            'attachmentSet' => {
+              'vpcId' => vpc_id,
+              'state' => 'available'
+            },
+            'tagSet' => {}
+          }
+
+          data[:route_tables] << {
+            'routeTableId' => Fog::AWS::Mock.route_table_id,
+            'vpcId' => vpc_id,
+            'routes' => [
+              {
+                'destinationCidrBlock' => '172.31.0.0/16',
+                'gatewayId' => 'local',
+                'state' => 'active',
+                'origin' => 'CreateRouteTable'
+              },
+              {
+                'destinationCidrBlock' => '0.0.0.0/0',
+                'gatewayId' => internet_gateway_id,
+                'state' => 'active',
+                'origin' => 'CreateRoute'
+              }
+            ]
+          }
+
+          describe_availability_zones.body['availabilityZoneInfo'].map { |z| z['zoneName'] }.each_with_index do |zone, i|
+            data[:subnets] << {
+              'subnetId'                 => Fog::AWS::Mock.subnet_id,
+              'state'                    => 'available',
+              'vpcId'                    => vpc_id,
+              'cidrBlock'                => "172.31.#{i}.0/16",
+              'availableIpAddressCount'  => '251',
+              'availabilityZone'         => zone,
+              'tagSet'                   => {},
+              'mapPublicIpOnLaunch'      => true,
+              'defaultForAz'             => true
+            }
+          end
+        end
+
         def tagged_resources(resources)
           Array(resources).map do |resource_id|
-            if match = resource_id.match(/^(\w+)-[a-z0-9]{8}/i)
+            if match = resource_id.match(/^(\w+)-[a-z0-9]{8,17}/i)
               id = match.captures.first
             else
               raise(Fog::Service::NotFound.new("Unknown resource id #{resource_id}"))
@@ -384,6 +478,10 @@ module Fog
                 if self.data[:vpcs].select {|v| v['vpcId'] == resource_id }.empty?
                   raise(Fog::Service::NotFound.new("Cannot tag #{resource_id}, the vpc does not exist"))
                 end
+              when 'route_table'
+                unless self.data[:route_tables].detect { |r| r['routeTableId'] == resource_id }
+                  raise(Fog::Service::NotFound.new("Cannot tag #{resource_id}, the route table does not exist"))
+                end
               else
                 unless self.data[:"#{type}s"][resource_id]
                  raise(Fog::Service::NotFound.new("Cannot tag #{resource_id}, the #{type} does not exist"))
@@ -453,11 +551,13 @@ module Fog
 
         def initialize(options={})
 
-          @connection_options     = options[:connection_options] || {}
-          @region                 = options[:region] ||= 'us-east-1'
-          @instrumentor           = options[:instrumentor]
-          @instrumentor_name      = options[:instrumentor_name] || 'fog.aws.compute'
-          @version                = options[:version]     ||  '2014-10-01'
+          @connection_options           = options[:connection_options] || {}
+          @region                       = options[:region] ||= 'us-east-1'
+          @instrumentor                 = options[:instrumentor]
+          @instrumentor_name            = options[:instrumentor_name] || 'fog.aws.compute'
+          @version                      = options[:version]     ||  '2016-11-15'
+          @retry_request_limit_exceeded = options.fetch(:retry_request_limit_exceeded, true)
+          @retry_jitter_magnitude       = options[:retry_jitter_magnitude] || 0.1
 
           @use_iam_profile = options[:use_iam_profile]
           setup_credentials(options)
@@ -521,7 +621,11 @@ module Fog
           end
         end
 
-        def _request(body, headers, idempotent, parser)
+        def _request(body, headers, idempotent, parser, retries = 0)
+
+          max_retries = 10
+
+          begin
           @connection.request({
               :body       => body,
               :expects    => 200,
@@ -530,17 +634,44 @@ module Fog
               :method     => 'POST',
               :parser     => parser
             })
-        rescue Excon::Errors::HTTPStatusError => error
-          match = Fog::AWS::Errors.match_error(error)
-          raise if match.empty?
-          raise case match[:code]
+          rescue Excon::Errors::HTTPStatusError => error
+            match = Fog::AWS::Errors.match_error(error)
+            raise if match.empty?
+            raise case match[:code]
                 when 'NotFound', 'Unknown'
-                  Fog::Compute::AWS::NotFound.slurp(error, match[:message])
+                  Fog::AWS::Compute::NotFound.slurp(error, match[:message])
+                when 'RequestLimitExceeded'                  
+                  if @retry_request_limit_exceeded && retries < max_retries
+                    jitter = rand * 10 * @retry_jitter_magnitude
+                    wait_time = ((2.0 ** (1.0 + retries) * 100) / 1000.0) + jitter
+                    Fog::Logger.warning "Waiting #{wait_time} seconds to retry."
+                    sleep(wait_time)
+                    retries += 1
+                    retry
+                  elsif @retry_request_limit_exceeded
+                    Fog::AWS::Compute::RequestLimitExceeded.slurp(error, "Max retries exceeded (#{max_retries}) #{match[:code]} => #{match[:message]}")
+                  else
+                    Fog::AWS::Compute::RequestLimitExceeded.slurp(error, "#{match[:code]} => #{match[:message]}")
+                  end
                 else
-                  Fog::Compute::AWS::Error.slurp(error, "#{match[:code]} => #{match[:message]}")
+                  Fog::AWS::Compute::Error.slurp(error, "#{match[:code]} => #{match[:message]}")
                 end
+          end
         end
       end
     end
   end
+
+  # @deprecated
+  module Compute
+    # @deprecated
+    class AWS < Fog::AWS::Compute
+      # @deprecated
+      # @overrides Fog::Service.new (from the fog-core gem)
+      def self.new(*)
+        Fog::Logger.deprecation 'Fog::Compute::AWS is deprecated, please use Fog::AWS::Compute.'
+        super
+      end
+    end
+  end
 end

data/lib/fog/aws/credential_fetcher.rb

@@ -1,8 +1,13 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
 module Fog
   module AWS
     module CredentialFetcher
 
       INSTANCE_METADATA_HOST = "http://169.254.169.254"
+      INSTANCE_METADATA_TOKEN = "/latest/api/token"
       INSTANCE_METADATA_PATH = "/latest/meta-data/iam/security-credentials/"
       INSTANCE_METADATA_AZ = "/latest/meta-data/placement/availability-zone/"
 
@@ -11,30 +16,65 @@ module Fog
       module ServiceMethods
         def fetch_credentials(options)
           if options[:use_iam_profile] && Fog.mocking?
-            Fog::Compute::AWS::Mock.data[:iam_role_based_creds]
+            return Fog::AWS::Compute::Mock.data[:iam_role_based_creds]
           end
           if options[:use_iam_profile]
             begin
               role_data = nil
-              az_data = nil
+              region = options[:region] || ENV["AWS_DEFAULT_REGION"]
 
               if ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
                 connection = options[:connection] || Excon.new(CONTAINER_CREDENTIALS_HOST)
                 credential_path = options[:credential_path] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
-                role_data = connection.get(:path => credential_path, :expects => 200).body
+                role_data = connection.get(:path => credential_path, :idempotent => true, :expects => 200).body
+                session = Fog::JSON.decode(role_data)
+
+                if region.nil?
+                  connection = options[:metadata_connection] || Excon.new(INSTANCE_METADATA_HOST)
+                  token_header = fetch_credentials_token_header(connection, options[:disable_imds_v2])
+                  region = connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
+                end
+              elsif ENV["AWS_WEB_IDENTITY_TOKEN_FILE"]
+                params = {
+                  :Action => "AssumeRoleWithWebIdentity",
+                  :RoleArn => options[:role_arn] || ENV.fetch("AWS_ROLE_ARN"),
+                  :RoleSessionName => options[:role_session_name] || ENV["AWS_ROLE_SESSION_NAME"] || "fog-aws-#{SecureRandom.hex}",
+                  :WebIdentityToken => File.read(options[:aws_web_identity_token_file] || ENV.fetch("AWS_WEB_IDENTITY_TOKEN_FILE")),
+                  :Version => "2011-06-15",
+                }
+
+                sts_endpoint =
+                  if ENV["AWS_STS_REGIONAL_ENDPOINTS"] == "regional" && region
+                    "https://sts.#{region}.amazonaws.com"
+                  else
+                    "https://sts.amazonaws.com"
+                  end
 
-                connection = options[:metadata_connection] || Excon.new(INSTANCE_METADATA_HOST)
-                az_data = connection.get(:path => INSTANCE_METADATA_AZ, :expects => 200).body
+                connection = options[:connection] || Excon.new(sts_endpoint, :query => params)
+                document = Nokogiri::XML(connection.get(:idempotent => true, :expects => 200).body)
+
+                session = {
+                  "AccessKeyId" => document.css("AccessKeyId").children.text,
+                  "SecretAccessKey" => document.css("SecretAccessKey").children.text,
+                  "Token" => document.css("SessionToken").children.text,
+                  "Expiration" => document.css("Expiration").children.text,
+                }
+
+                if region.nil?
+                  connection = options[:metadata_connection] || Excon.new(INSTANCE_METADATA_HOST)
+                  token_header = fetch_credentials_token_header(connection, options[:disable_imds_v2])
+                  region = connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
+                end
               else
                 connection = options[:connection] || Excon.new(INSTANCE_METADATA_HOST)
-                role_name = connection.get(:path => INSTANCE_METADATA_PATH, :expects => 200).body
-                role_data = connection.get(:path => INSTANCE_METADATA_PATH+role_name, :expects => 200).body
-                az_data = connection.get(:path => INSTANCE_METADATA_AZ, :expects => 200).body
-              end
+                token_header = fetch_credentials_token_header(connection, options[:disable_imds_v2])
+                role_name = connection.get(:path => INSTANCE_METADATA_PATH, :idempotent => true, :expects => 200, :headers => token_header).body
+                role_data = connection.get(:path => INSTANCE_METADATA_PATH+role_name, :idempotent => true, :expects => 200, :headers => token_header).body
+                session = Fog::JSON.decode(role_data)
 
-              region = az_data[0..-2] # get region from az
+                region ||= connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
+              end
 
-              session = Fog::JSON.decode(role_data)
               credentials = {}
               credentials[:aws_access_key_id] = session['AccessKeyId']
               credentials[:aws_secret_access_key] = session['SecretAccessKey']
@@ -43,6 +83,7 @@ module Fog
 
               # set region by default to the one the instance is in.
               credentials[:region] = region
+              credentials[:sts_endpoint] = sts_endpoint if sts_endpoint
               #these indicate the metadata service is unavailable or has no profile setup
               credentials
             rescue Excon::Error => e
@@ -53,6 +94,26 @@ module Fog
             super
           end
         end
+
+        def fetch_credentials_token_header(connection, disable_imds_v2)
+          return nil if disable_imds_v2
+
+          token = connection.put(
+            :path => INSTANCE_METADATA_TOKEN,
+            :idempotent => true,
+            :expects => 200,
+            :retry_interval => 1,
+            :retry_limit => 3,
+            :read_timeout => 1,
+            :write_timeout => 1,
+            :connect_timeout => 1,
+            :headers => { "X-aws-ec2-metadata-token-ttl-seconds" => "300" }
+          ).body
+
+          { "X-aws-ec2-metadata-token" => token }
+        rescue Excon::Error
+          nil
+        end
       end
 
       module ConnectionMethods
@@ -62,15 +123,22 @@ module Fog
 
         private
 
+        # When defined, 'aws_credentials_refresh_threshold_seconds' controls
+        # when the credential needs to be refreshed, expressed in seconds before
+        # the current credential's expiration time
+        def credentials_refresh_threshold
+          @aws_credentials_refresh_threshold_seconds || 15
+        end
+
         def credentials_expired?
           @use_iam_profile &&
             (!@aws_credentials_expire_at ||
-             (@aws_credentials_expire_at && Fog::Time.now > @aws_credentials_expire_at - 15)) #new credentials become available from around 5 minutes before expiration time
+             (@aws_credentials_expire_at && Fog::Time.now > @aws_credentials_expire_at - credentials_refresh_threshold)) #new credentials become available from around 5 minutes before expiration time
         end
 
         def refresh_credentials
           if @use_iam_profile
-            new_credentials = service.fetch_credentials :use_iam_profile => @use_iam_profile
+            new_credentials = service.fetch_credentials :use_iam_profile => @use_iam_profile, :region => @region
             if new_credentials.any?
               setup_credentials new_credentials
               return true

data/lib/fog/aws/data_pipeline.rb

@@ -9,6 +9,7 @@ module Fog
       request_path 'fog/aws/requests/data_pipeline'
       request :activate_pipeline
       request :create_pipeline
+      request :deactivate_pipeline
       request :delete_pipeline
       request :describe_pipelines
       request :list_pipelines
@@ -22,8 +23,58 @@ module Fog
       collection  :pipelines
 
       class Mock
+        include Fog::AWS::CredentialFetcher::ConnectionMethods
+
+        def self.data
+          @data ||= Hash.new do |hash, region|
+            hash[region] = Hash.new do |region_hash, key|
+              region_hash[key] = {
+                :pipelines            => {},
+                :pipeline_definitions => {},
+              }
+            end
+          end
+        end
+
+        def self.reset
+          @data = nil
+        end
+
+        def data
+          self.class.data[@region][@aws_access_key_id]
+        end
+
+        def reset
+          self.class.reset
+        end
+
+        attr_accessor :region
+
         def initialize(options={})
-          Fog::Mock.not_implemented
+          @region                = options[:region] || "us-east-1"
+          @aws_access_key_id     = options[:aws_access_key_id]
+          @aws_secret_access_key = options[:aws_secret_access_key]
+        end
+
+        def stringify_keys(object)
+          case object
+          when Hash
+            object.inject({}) { |h,(k,v)| h[k.to_s] = stringify_keys(v); h }
+          when Array
+            object.map { |v| stringify_keys(v) }
+          else
+            object
+          end
+        end
+
+        def find_pipeline(id)
+          pipeline = self.data[:pipelines].values.detect { |p| p["pipelineId"] == id }
+
+          if pipeline.nil? || pipeline[:deleted]
+            raise Fog::AWS::DataPipeline::NotFound.new("Pipeline with id: #{id} does not exist")
+          end
+
+          pipeline
         end
       end
 
@@ -116,7 +167,19 @@ module Fog
         end
 
         def _request(params)
-          @connection.request(params)
+          response = @connection.request(params)
+
+          unless response.body.empty?
+            response.body = Fog::JSON.decode(response.body)
+          end
+
+          response
+        rescue Excon::Error::BadRequest => error
+          match = Fog::AWS::Errors.match_error(error)
+          raise if match.empty?
+          if %w(PipelineNotFoundException PipelineDeletedException).include?(match[:code])
+            raise Fog::AWS::DataPipeline::NotFound.slurp(error, match[:message])
+          end
         end
       end
     end