fog-aws 3.8.0 → 3.12.0

data/README.md

@@ -1,7 +1,7 @@
 # Fog::Aws
 
 ![Gem Version](https://badge.fury.io/rb/fog-aws.svg)
-[![Build Status](https://travis-ci.org/fog/fog-aws.svg?branch=master)](https://travis-ci.org/fog/fog-aws)
+[![Build Status](https://github.com/fog/fog-aws/actions/workflows/ruby.yml/badge.svg)](https://github.com/fog/fog-aws/actions/workflows/ruby.yml)
 [![Test Coverage](https://codeclimate.com/github/fog/fog-aws/badges/coverage.svg)](https://codeclimate.com/github/fog/fog-aws)
 [![Code Climate](https://codeclimate.com/github/fog/fog-aws.svg)](https://codeclimate.com/github/fog/fog-aws)
 

data/fog-aws.gemspec

@@ -14,17 +14,18 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/fog/fog-aws"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0")
+  spec.files         = Dir['lib/**/*.rb', 'tests/**/*', 'CHANGELOG.md', 'CONTRIBUTING.md',
+                           'CONTRIBUTORS.md', 'LICENSE.md', 'README.md', 'fog-aws.gemspec',]
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
   spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'github_changelog_generator', '~> 1.14'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'github_changelog_generator', '~> 1.16'
   spec.add_development_dependency 'rake',    '>= 12.3.3'
-  spec.add_development_dependency 'rubyzip', '~> 1.3.0'
+  spec.add_development_dependency 'rubyzip', '~> 2.3.0'
   spec.add_development_dependency 'shindo',  '~> 0.3'
 
   spec.add_dependency 'fog-core',  '~> 2.1'

data/lib/fog/aws/compute.rb

@@ -233,21 +233,24 @@ module Fog
                         'fromPort'    => -1,
                         'toPort'      => -1,
                         'ipProtocol'  => 'icmp',
-                        'ipRanges'    => []
+                        'ipRanges'    => [],
+                        'ipv6Ranges'  => []
                       },
                       {
                         'groups'      => [{'groupName' => 'default', 'userId' => owner_id, 'groupId' => security_group_id}],
                         'fromPort'    => 0,
                         'toPort'      => 65535,
                         'ipProtocol'  => 'tcp',
-                        'ipRanges'    => []
+                        'ipRanges'    => [],
+                        'ipv6Ranges'  => []
                       },
                       {
                         'groups'      => [{'groupName' => 'default', 'userId' => owner_id, 'groupId' => security_group_id}],
                         'fromPort'    => 0,
                         'toPort'      => 65535,
                         'ipProtocol'  => 'udp',
-                        'ipRanges'    => []
+                        'ipRanges'    => [],
+                        'ipv6Ranges'  => []
                       }
                     ],
                     'ownerId'             => owner_id

data/lib/fog/aws/credential_fetcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module Fog
   module AWS
     module CredentialFetcher
@@ -11,8 +13,6 @@ module Fog
 
       CONTAINER_CREDENTIALS_HOST = "http://169.254.170.2"
 
-      STS_GLOBAL_ENDPOINT = "https://sts.amazonaws.com"
-
       module ServiceMethods
         def fetch_credentials(options)
           if options[:use_iam_profile] && Fog.mocking?
@@ -21,7 +21,7 @@ module Fog
           if options[:use_iam_profile]
             begin
               role_data = nil
-              region = options[:region]
+              region = options[:region] || ENV["AWS_DEFAULT_REGION"]
 
               if ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
                 connection = options[:connection] || Excon.new(CONTAINER_CREDENTIALS_HOST)
@@ -38,11 +38,19 @@ module Fog
                 params = {
                   :Action => "AssumeRoleWithWebIdentity",
                   :RoleArn => options[:role_arn] || ENV.fetch("AWS_ROLE_ARN"),
-                  :RoleSessionName => options[:role_session_name] || ENV.fetch("AWS_ROLE_SESSION_NAME"),
+                  :RoleSessionName => options[:role_session_name] || ENV["AWS_ROLE_SESSION_NAME"] || "fog-aws-#{SecureRandom.hex}",
                   :WebIdentityToken => File.read(options[:aws_web_identity_token_file] || ENV.fetch("AWS_WEB_IDENTITY_TOKEN_FILE")),
                   :Version => "2011-06-15",
                 }
-                connection = options[:connection] || Excon.new(STS_GLOBAL_ENDPOINT, :query => params)
+
+                sts_endpoint =
+                  if ENV["AWS_STS_REGIONAL_ENDPOINTS"] == "regional" && region
+                    "https://sts.#{region}.amazonaws.com"
+                  else
+                    "https://sts.amazonaws.com"
+                  end
+
+                connection = options[:connection] || Excon.new(sts_endpoint, :query => params)
                 document = Nokogiri::XML(connection.get(:idempotent => true, :expects => 200).body)
 
                 session = {
@@ -63,18 +71,19 @@ module Fog
                 role_name = connection.get(:path => INSTANCE_METADATA_PATH, :idempotent => true, :expects => 200, :headers => token_header).body
                 role_data = connection.get(:path => INSTANCE_METADATA_PATH+role_name, :idempotent => true, :expects => 200, :headers => token_header).body
                 session = Fog::JSON.decode(role_data)
-                
+
                 region ||= connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2]
               end
-              
+
               credentials = {}
               credentials[:aws_access_key_id] = session['AccessKeyId']
               credentials[:aws_secret_access_key] = session['SecretAccessKey']
               credentials[:aws_session_token] = session['Token']
               credentials[:aws_credentials_expire_at] = Time.xmlschema session['Expiration']
-              
+
               # set region by default to the one the instance is in.
               credentials[:region] = region
+              credentials[:sts_endpoint] = sts_endpoint if sts_endpoint
               #these indicate the metadata service is unavailable or has no profile setup
               credentials
             rescue Excon::Error => e

data/lib/fog/aws/models/compute/security_group.rb

@@ -62,7 +62,8 @@ module Fog
         # options::
         #   A hash that can contain any of the following keys:
         #    :cidr_ip (defaults to "0.0.0.0/0")
-        #    :group - ("account:group_name" or "account:group_id"), cannot be used with :cidr_ip
+        #    :cidr_ipv6 cannot be used with :cidr_ip
+        #    :group - ("account:group_name" or "account:group_id"), cannot be used with :cidr_ip or :cidr_ipv6
         #    :ip_protocol (defaults to "tcp")
         #
         # == Returns:
@@ -178,7 +179,8 @@ module Fog
         # options::
         #   A hash that can contain any of the following keys:
         #    :cidr_ip (defaults to "0.0.0.0/0")
-        #    :group - ("account:group_name" or "account:group_id"), cannot be used with :cidr_ip
+        #    :cidr_ipv6 cannot be used with :cidr_ip
+        #    :group - ("account:group_name" or "account:group_id"), cannot be used with :cidr_ip or :cidr_ipv6
         #    :ip_protocol (defaults to "tcp")
         #
         # == Returns:
@@ -327,9 +329,15 @@ module Fog
           }
 
           if options[:group].nil?
-            ip_permission['IpRanges'] = [
-              { 'CidrIp' => options[:cidr_ip] || '0.0.0.0/0' }
-            ]
+            if options[:cidr_ipv6].nil?
+              ip_permission['IpRanges'] = [
+                { 'CidrIp' => options[:cidr_ip] || '0.0.0.0/0' }
+              ]
+            else
+              ip_permission['Ipv6Ranges'] = [
+                { 'CidrIpv6' => options[:cidr_ipv6] }
+              ]
+            end
           else
             ip_permission['Groups'] = [
               group_info(options[:group])

data/lib/fog/aws/models/compute/server.rb

@@ -50,6 +50,7 @@ module Fog
         attribute :subnet_id,                :aliases => 'subnetId'
         attribute :tenancy
         attribute :tags,                     :aliases => 'tagSet'
+        attribute :tag_specifications,       :aliases => 'tagSpecifications'
         attribute :user_data
         attribute :virtualization_type,      :aliases => 'virtualizationType'
         attribute :vpc_id,                   :aliases => 'vpcId'
@@ -166,6 +167,7 @@ module Fog
             'SecurityGroupId'             => security_group_ids,
             'SubnetId'                    => subnet_id,
             'UserData'                    => user_data,
+            'TagSpecifications'           => tag_specifications,
           }
           options.delete_if {|key, value| value.nil?}
 

data/lib/fog/aws/models/storage/file.rb

@@ -4,8 +4,11 @@ module Fog
   module AWS
     class Storage
       class File < Fog::Model
-        MIN_MULTIPART_CHUNK_SIZE = 5242880
-        MAX_SINGLE_PUT_SIZE = 5368709120
+        # @deprecated use {Fog::AWS::Storage::MIN_MULTIPART_CHUNK_SIZE} instead
+        MIN_MULTIPART_CHUNK_SIZE = Fog::AWS::Storage::MIN_MULTIPART_CHUNK_SIZE
+        # @deprecated use {Fog::AWS::Storage::MAX_SINGLE_PUT_SIZE} instead
+        MAX_SINGLE_PUT_SIZE = Fog::AWS::Storage::MAX_SINGLE_PUT_SIZE
+        # @deprecated not used for anything
         MULTIPART_COPY_THRESHOLD = 15728640
 
         # @see AWS Object docs http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
@@ -30,6 +33,7 @@ module Fog
         attribute :version,             :aliases => 'x-amz-version-id'
         attribute :kms_key_id,          :aliases => 'x-amz-server-side-encryption-aws-kms-key-id'
         attribute :tags,                :aliases => 'x-amz-tagging'
+        attribute :website_redirect_location, :aliases => 'x-amz-website-redirect-location'
 
         UploadPartData = Struct.new(:part_number, :upload_options, :etag)
 
@@ -64,7 +68,7 @@ module Fog
         #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
         attr_reader :multipart_chunk_size
         def multipart_chunk_size=(mp_chunk_size)
-          raise ArgumentError.new("minimum multipart_chunk_size is #{MIN_MULTIPART_CHUNK_SIZE}") if mp_chunk_size < MIN_MULTIPART_CHUNK_SIZE
+          service.validate_chunk_size(mp_chunk_size, 'multipart_chunk_size')
           @multipart_chunk_size = mp_chunk_size
         end
 
@@ -144,10 +148,9 @@ module Fog
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
 
-          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE * 2 if !multipart_chunk_size && self.content_length.to_i > MAX_SINGLE_PUT_SIZE
+          self.multipart_chunk_size = service.max_copy_chunk_size if multipart_chunk_size.nil?
 
-          if multipart_chunk_size && self.content_length.to_i >= multipart_chunk_size
+          if multipart_chunk_size > 0 && self.content_length.to_i >= multipart_chunk_size
             upload_part_options = options.select { |key, _| ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym) }
             upload_part_options = upload_part_options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" })
             multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
@@ -249,6 +252,7 @@ module Fog
         # @option options [String] storage_class sets x-amz-storage-class HTTP header. Defaults to 'STANDARD'. Or, 'REDUCED_REDUNDANCY'
         # @option options [String] encryption sets HTTP encryption header. Set to 'AES256' to encrypt files at rest on S3
         # @option options [String] tags sets x-amz-tagging HTTP header. For example, 'Org-Id=1' or 'Org-Id=1&Service=MyService'
+        # @option options [String] website_redirect_location sets x-amz-website-redirect-location HTTP header. For example, 'website_redirect_location=http://www.rubydoc.info/github/fog/fog-aws'
         # @return [Boolean] true if no errors
         #
         def save(options = {})
@@ -266,12 +270,11 @@ module Fog
           options.merge!(metadata)
           options['x-amz-storage-class'] = storage_class if storage_class
           options['x-amz-tagging'] = tags if tags
+          options['x-amz-website-redirect-location'] = website_redirect_location if website_redirect_location
           options.merge!(encryption_headers)
 
-          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE if !multipart_chunk_size && Fog::Storage.get_body_size(body) > MAX_SINGLE_PUT_SIZE
-
-          if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
+          self.multipart_chunk_size = service.max_put_chunk_size if multipart_chunk_size.nil?
+          if multipart_chunk_size > 0 && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
             data = multipart_save(options)
             merge_attributes(data.body)
           else

data/lib/fog/aws/models/storage/files.rb

@@ -17,6 +17,15 @@ module Fog
 
         model Fog::AWS::Storage::File
 
+        DASHED_HEADERS = %w(
+          Cache-Control
+          Content-Disposition
+          Content-Encoding
+          Content-Length
+          Content-MD5
+          Content-Type
+        ).freeze
+
         def all(options = {})
           requires :directory
           options = {
@@ -114,8 +123,29 @@ module Fog
         end
 
         def normalize_headers(data)
-          data.headers['Last-Modified'] = Time.parse(data.get_header('Last-Modified'))
-          data.headers['ETag'] = data.get_header('ETag').gsub('"','')
+          data.headers['Last-Modified'] = Time.parse(fetch_and_delete_header(data, 'Last-Modified'))
+
+          etag = fetch_and_delete_header(data, 'ETag').gsub('"','')
+          data.headers['ETag'] = etag
+
+          DASHED_HEADERS.each do |header|
+            value = fetch_and_delete_header(data, header)
+            data.headers[header] = value if value
+          end
+        end
+
+        private
+
+        def fetch_and_delete_header(response, header)
+          value = response.get_header(header)
+
+          return unless value
+
+          response.headers.keys.each do |key|
+            response.headers.delete(key) if key.downcase == header.downcase
+          end
+
+          value
         end
       end
     end

data/lib/fog/aws/parsers/compute/describe_security_groups.rb

@@ -5,9 +5,10 @@ module Fog
         class DescribeSecurityGroups < Fog::Parsers::Base
           def reset
             @group = {}
-            @ip_permission = { 'groups' => [], 'ipRanges' => []}
-            @ip_permission_egress = { 'groups' => [], 'ipRanges' => []}
+            @ip_permission = { 'groups' => [], 'ipRanges' => [], 'ipv6Ranges' => []}
+            @ip_permission_egress = { 'groups' => [], 'ipRanges' => [], 'ipv6Ranges' => []}
             @ip_range = {}
+            @ipv6_range = {}
             @security_group = { 'ipPermissions' => [], 'ipPermissionsEgress' => [], 'tagSet' => {} }
             @response = { 'securityGroupInfo' => [] }
             @tag = {}
@@ -24,6 +25,8 @@ module Fog
               @in_ip_permissions_egress = true
             when 'ipRanges'
               @in_ip_ranges = true
+            when 'ipv6Ranges'
+              @in_ipv6_ranges = true
             when 'tagSet'
               @in_tag_set = true
             end
@@ -44,6 +47,8 @@ module Fog
               case name
               when 'cidrIp'
                 @ip_range[name] = value
+              when 'cidrIpv6'
+                @ipv6_range[name] = value
               when 'fromPort', 'toPort'
                 if @in_ip_permissions_egress
                   @ip_permission_egress[name] = value.to_i
@@ -72,6 +77,8 @@ module Fog
                 end
               when 'ipRanges'
                 @in_ip_ranges = false
+              when 'ipv6Ranges'
+                @in_ipv6_ranges = false
               when 'item'
                 if @in_groups
                   if @in_ip_permissions_egress
@@ -87,12 +94,19 @@ module Fog
                     @ip_permission['ipRanges'] << @ip_range
                   end
                   @ip_range = {}
+                elsif @in_ipv6_ranges
+                  if @in_ip_permissions_egress
+                    @ip_permission_egress['ipv6Ranges'] << @ipv6_range
+                  else
+                    @ip_permission['ipv6Ranges'] << @ipv6_range
+                  end
+                  @ipv6_range = {}
                 elsif @in_ip_permissions
                   @security_group['ipPermissions'] << @ip_permission
-                  @ip_permission = { 'groups' => [], 'ipRanges' => []}
+                  @ip_permission = { 'groups' => [], 'ipRanges' => [], 'ipv6Ranges' => []}
                 elsif @in_ip_permissions_egress
                   @security_group['ipPermissionsEgress'] << @ip_permission_egress
-                  @ip_permission_egress = { 'groups' => [], 'ipRanges' => []}
+                  @ip_permission_egress = { 'groups' => [], 'ipRanges' => [], 'ipv6Ranges' => []}
                 else
                   @response['securityGroupInfo'] << @security_group
                   @security_group = { 'ipPermissions' => [], 'ipPermissionsEgress' => [], 'tagSet' => {} }

data/lib/fog/aws/requests/compute/authorize_security_group_ingress.rb

@@ -30,6 +30,9 @@ module Fog
         #       * 'IpRanges'<~Array>:
         #         * ip_range<~Hash>:
         #           * 'CidrIp'<~String> - CIDR range
+        #       * 'Ipv6Ranges'<~Array>:
+        #         * ip_range<~Hash>:
+        #           * 'CidrIpv6'<~String> - CIDR range
         #       * 'ToPort'<~Integer> - End of port range (or -1 for ICMP wildcard)
         #
         # === Returns
@@ -72,6 +75,10 @@ module Fog
               range_index += 1
               params[format('IpPermissions.%d.IpRanges.%d.CidrIp', key_index, range_index)] = ip_range['CidrIp']
             end
+            (permission['Ipv6Ranges'] || []).each_with_index do |ip_range, range_index|
+              range_index += 1
+              params[format('IpPermissions.%d.Ipv6Ranges.%d.CidrIpv6', key_index, range_index)] = ip_range['CidrIpv6']
+            end
           end
           params.reject {|k, v| v.nil? }
         end
@@ -186,6 +193,14 @@ module Fog
               'groups'     => [],
               'ipRanges'   => [{'cidrIp' => options['CidrIp']}]
             }
+          elsif options['CidrIpv6']
+            normalized_permissions << {
+              'ipProtocol' => options['IpProtocol'],
+              'fromPort'   => Integer(options['FromPort']),
+              'toPort'     => Integer(options['ToPort']),
+              'groups'     => [],
+              'ipv6Ranges' => [{'cidrIpv6' => options['CidrIpv6']}]
+            }
           elsif options['IpPermissions']
             options['IpPermissions'].each do |permission|
 

data/lib/fog/aws/requests/compute/describe_security_groups.rb

@@ -27,6 +27,8 @@ module Fog
         #         * 'ipProtocol'<~String> - Ip protocol, must be in ['tcp', 'udp', 'icmp']
         #         * 'ipRanges'<~Array>:
         #           * 'cidrIp'<~String> - CIDR range
+        #         * 'ipv6Ranges'<~Array>:
+        #           * 'cidrIpv6'<~String> - CIDR ipv6 range
         #         * 'toPort'<~Integer> - End of port range (or -1 for ICMP wildcard)
         #       * 'ownerId'<~String> - AWS Access Key Id of the owner of the security group
         #     * 'NextToken'<~String> - The token to retrieve the next page of results

data/lib/fog/aws/requests/compute/run_instances.rb

@@ -44,6 +44,11 @@ module Fog
         #     * 'PrivateIpAddresses.Primary'<~Bool> - Indicates whether the private IP address is the primary private IP address.
         #     * 'SecondaryPrivateIpAddressCount'<~Bool> - The number of private IP addresses to assign to the network interface.
         #     * 'AssociatePublicIpAddress'<~String> - Indicates whether to assign a public IP address to an instance in a VPC. The public IP address is assigned to a specific network interface
+        #   * 'TagSpecifications'<~Array>: array of hashes
+        #     * 'ResourceType'<~String> - Type of resource to apply tags on, e.g: instance or volume
+        #     * 'Tags'<~Array> - List of hashs reprensenting tag to be set
+        #       * 'Key'<~String> - Tag name
+        #       * 'Value'<~String> - Tag value
         #   * 'ClientToken'<~String> - unique case-sensitive token for ensuring idempotency
         #   * 'DisableApiTermination'<~Boolean> - specifies whether or not to allow termination of the instance from the api
         #   * 'SecurityGroup'<~Array> or <~String> - Name of security group(s) for instances (not supported for VPC)
@@ -144,6 +149,45 @@ module Fog
               end
             end
           end
+          if tag_specifications = options.delete('TagSpecifications')
+            # From https://docs.aws.amazon.com/sdk-for-ruby/v2/api/Aws/EC2/Client.html#run_instances-instance_method
+            # And https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html
+            # Discussed at https://github.com/fog/fog-aws/issues/603
+            #
+            # Example
+            #
+            # TagSpecifications: [
+            #     {
+            #       ResourceType: "instance",
+            #       Tags: [
+            #         {
+            #           Key: "Project",
+            #           Value: "MyProject",
+            #         },
+            #       ],
+            #     },
+            #     {
+            #       ResourceType: "volume",
+            #       Tags: [
+            #         {
+            #           Key: "Project",
+            #           Value: "MyProject",
+            #         },
+            #       ],
+            #     },
+            # ]
+            tag_specifications.each_with_index do |val, idx|
+              resource_type = val["ResourceType"]
+              tags = val["Tags"]
+              options["TagSpecification.#{idx}.ResourceType"] = resource_type
+              tags.each_with_index do |tag, tag_idx|
+                aws_tag_key = "TagSpecification.#{idx}.Tag.#{tag_idx}.Key"
+                aws_tag_value = "TagSpecification.#{idx}.Tag.#{tag_idx}.Value"
+                options[aws_tag_key] = tag["Key"]
+                options[aws_tag_value] = tag["Value"]
+              end
+            end
+          end
 
           idempotent = !(options['ClientToken'].nil? || options['ClientToken'].empty?)