fog-aws 3.8.0 → 3.12.0

data/lib/fog/aws/requests/storage/delete_multiple_objects.rb

@@ -36,13 +36,20 @@ module Fog
           data << "<Quiet>true</Quiet>" if headers.delete(:quiet)
           version_ids = headers.delete('versionId')
           object_names.each do |object_name|
-            data << "<Object>"
-            data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
             object_version = version_ids.nil? ? nil : version_ids[object_name]
             if object_version
-              data << "<VersionId>#{CGI.escapeHTML(object_version)}</VersionId>"
+              object_version = object_version.is_a?(String) ? [object_version] : object_version
+              object_version.each do |version_id|
+                data << "<Object>"
+                data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
+                data << "<VersionId>#{CGI.escapeHTML(version_id)}</VersionId>"
+                data << "</Object>"
+              end
+            else
+              data << "<Object>"
+              data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
+              data << "</Object>"
             end
-            data << "</Object>"
           end
           data << "</Delete>"
 
@@ -72,10 +79,13 @@ module Fog
             response.body = { 'DeleteResult' => [] }
             version_ids = headers.delete('versionId')
             object_names.each do |object_name|
-              object_version = version_ids.nil? ? nil : version_ids[object_name]
-              response.body['DeleteResult'] << delete_object_helper(bucket,
-                                                                object_name,
-                                                                object_version)
+              object_version = version_ids.nil? ? [nil] : version_ids[object_name]
+              object_version = object_version.is_a?(String) ? [object_version] : object_version
+              object_version.each do |version_id|
+                response.body['DeleteResult'] << delete_object_helper(bucket,
+                                                                  object_name,
+                                                                  version_id)
+              end
             end
           else
             response.status = 404

data/lib/fog/aws/requests/storage/get_object.rb

@@ -50,7 +50,7 @@ module Fog
 
           idempotent = true
           if block_given?
-            params[:response_block] = Proc.new
+            params[:response_block] = Proc.new(&block)
             idempotent = false
           end
 

data/lib/fog/aws/storage.rb

@@ -14,6 +14,9 @@ module Fog
         'https' => 443
       }
 
+      MIN_MULTIPART_CHUNK_SIZE = 5242880
+      MAX_SINGLE_PUT_SIZE = 5368709120
+
       VALID_QUERY_KEYS = %w[
         acl
         cors
@@ -43,7 +46,7 @@ module Fog
       ]
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname
+      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size
 
       secrets    :aws_secret_access_key, :hmac
 
@@ -117,6 +120,17 @@ module Fog
       module Utils
         attr_accessor :region
 
+        # Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB.
+        # Other S3-compatible storages (like, Ceph) do not have such limit.
+        # Ceph shows much better performance when file is copied as a whole, in a single request.
+        # fog-aws user can use these settings to configure chunk sizes.
+        # A non-positive value will tell fog-aws to use a single put/copy request regardless of file size.
+        #
+        # @return [Integer]
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html
+        attr_reader :max_put_chunk_size
+        attr_reader :max_copy_chunk_size
+
         def cdn
           @cdn ||= Fog::AWS::CDN.new(
             :aws_access_key_id => @aws_access_key_id,
@@ -171,6 +185,12 @@ module Fog
           params_to_url(params)
         end
 
+        # @param value [int]
+        # @param description [str]
+        def validate_chunk_size(value, description)
+          raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE
+        end
+
         private
 
         def validate_signature_version!
@@ -179,6 +199,16 @@ module Fog
           end
         end
 
+        def init_max_put_chunk_size!(options = {})
+          @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size')
+        end
+
+        def init_max_copy_chunk_size!(options = {})
+          @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size')
+        end
+
         def v4_signed_params_for_url(params, expires)
           now = Fog::Time.now
 
@@ -284,10 +314,10 @@ module Fog
               path_style = params.fetch(:path_style, @path_style)
               if !path_style
                 if COMPLIANT_BUCKET_NAMES !~ bucket_name
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 elsif scheme == 'https' && !path_style && bucket_name =~ /\./
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 end
               end
@@ -298,6 +328,8 @@ module Fog
                 host = params.fetch(:cname, bucket_name)
               elsif path_style
                 path = bucket_to_path bucket_name, path
+              elsif host.start_with?("#{bucket_name}.")
+                # no-op
               else
                 host = [bucket_name, host].join('.')
               end
@@ -450,6 +482,10 @@ module Fog
 
 
           @path_style = options[:path_style] || false
+
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
           @signature_version = options.fetch(:aws_signature_version, 4)
           validate_signature_version!
           setup_credentials(options)
@@ -513,6 +549,9 @@ module Fog
           validate_signature_version!
           @path_style = options[:path_style]  || false
 
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
           @region = options[:region] || DEFAULT_REGION
 
           if @endpoint = options[:endpoint]

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "3.8.0"
+    VERSION = "3.12.0"
   end
 end

data/tests/credentials_tests.rb

@@ -83,10 +83,63 @@ Shindo.tests('AWS | credentials', ['aws']) do
         aws_secret_access_key: 'dummysecret',
         aws_session_token: 'dummytoken',
         region: 'us-west-1',
+        sts_endpoint: "https://sts.amazonaws.com",
         aws_credentials_expire_at: expires_at
       ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
     end
 
+    ENV['AWS_ROLE_SESSION_NAME'] = nil
+
+    tests('#fetch_credentials token based without session name') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        sts_endpoint: "https://sts.amazonaws.com",
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, region: 'us-west-1') }
+    end
+
+    ENV["AWS_STS_REGIONAL_ENDPOINTS"] = "regional"
+
+    tests('#fetch_credentials with no region specified') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        sts_endpoint: "https://sts.amazonaws.com",
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
+
+    tests('#fetch_credentials with regional STS endpoint') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        sts_endpoint: "https://sts.us-west-1.amazonaws.com",
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, region: 'us-west-1') }
+    end
+
+    ENV["AWS_DEFAULT_REGION"] = "us-west-1"
+
+    tests('#fetch_credentials with regional STS endpoint with region in env') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        sts_endpoint: "https://sts.us-west-1.amazonaws.com",
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
+
+    ENV["AWS_STS_REGIONAL_ENDPOINTS"] = nil
+    ENV["AWS_DEFAULT_REGION"] = nil
     ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
 
     compute = Fog::AWS::Compute.new(use_iam_profile: true)

data/tests/helpers/succeeds_helper.rb

@@ -1,8 +1,8 @@
 module Shindo
   class Tests
-    def succeeds
+    def succeeds(&block)
       test('succeeds') do
-        !instance_eval(&Proc.new).nil?
+        !instance_eval(&Proc.new(&block)).nil?
       end
     end
   end

data/tests/models/storage/directory_tests.rb

@@ -1,4 +1,117 @@
 Shindo.tests("Storage[:aws] | directory", ["aws"]) do
+  tests('Fog::Storage[:aws]', "#request_params") do
+    def slice(hash, *args)
+      hash.select { |k, _| args.include?(k) }
+    end
+
+    instance = Fog::Storage[:aws]
+    method = instance.method(:request_params)
+
+    params = {bucket_name: 'profile-uploads', host: 'profile-uploads.s3.us-west-2.amazonaws.com'}
+    tests("given #{params}, request_params[:host]").returns("profile-uploads.s3.us-west-2.amazonaws.com") do
+      method.call(params)[:host]
+    end
+
+    params = {bucket_name: 'profile-uploads.johnsmith.net', cname: 'profile-uploads.johnsmith.net', virtual_host: true}
+    tests("given #{params}, request_params[:host]").returns("profile-uploads.johnsmith.net") do
+      method.call(params)[:host]
+    end
+
+    params = {bucket_name: 'profile-uploads.johnsmith.net', cname: 'profile-uploads.johnsmith.net', virtual_host: false}
+    tests("given #{params}, request_params[:host], request_params[:path]").
+      returns({host: "s3.amazonaws.com", path: "/profile-uploads.johnsmith.net/"}) do
+      slice(method.call(params), :host, :path)
+    end
+
+    params = {bucket_name: 'profile-uploads.johnsmith.net', bucket_cname: 'profile-uploads.johnsmith.net'}
+    tests("given #{params}, request_params[:host]").returns("profile-uploads.johnsmith.net") do
+      method.call(params)[:host]
+    end
+
+    params = {bucket_name: 'profile-uploads'}
+    tests("given #{params}, request_params[:path], request_params[:host]").
+      returns({path: "/", host: "profile-uploads.s3.amazonaws.com"}) do
+        slice(method.call(params), :path, :host)
+      end
+
+    params = {bucket_name: 'profile-uploads', path_style: true}
+    tests("given #{params}, request_params[:path], request_params[:host]").
+      returns({path: "/profile-uploads/", host: "s3.amazonaws.com"}) do
+        slice(method.call(params), :path, :host)
+      end
+
+    params = {bucket_name: 'profile-uploads', path_style: false}
+    tests("given #{params}, request_params[:path], request_params[:host]").
+      returns({path: "/", host: "profile-uploads.s3.amazonaws.com"}) do
+        slice(method.call(params), :path, :host)
+      end
+
+    params = {scheme: 'https', bucket_name: 'profile.uploads', path_style: false}
+    tests("given #{params}, request_params[:path], request_params[:host]").
+      returns(path: "/profile.uploads/", host: "s3.amazonaws.com") do
+        slice(method.call(params), :path, :host)
+      end
+
+    params = {:headers=>{:"Content-Type"=>"application/json"}}
+    tests("given #{params}, request_params[:headers]").returns({:"Content-Type"=>"application/json"}) do
+      method.call(params)[:headers]
+    end
+
+    params = {headers: {}}
+    tests("given #{params}, request_params[:headers]").returns({}) do
+      method.call(params)[:headers]
+    end
+
+    params = {scheme: 'http'}
+    tests("given #{params}, request_params[:scheme]").returns('http') do
+      method.call(params)[:scheme]
+    end
+
+    params = {}
+    tests("given #{params}, request_params[:scheme]").returns('https') do
+      method.call(params)[:scheme]
+    end
+
+    params = {scheme: 'http', port: 8080}
+    tests("given #{params} (default scheme), request_params[:port]").returns(8080) do
+      method.call(params)[:port]
+    end
+
+    params = {scheme: 'https', port: 443}
+    tests("given #{params}, request_params[:port]").returns(nil) do
+      method.call(params)[:port]
+    end
+
+    params = {}
+    tests("given #{params}, request_params[:host]").returns("s3.amazonaws.com") do
+      method.call(params)[:host]
+    end
+
+    params = {region: 'us-east-1'}
+    tests("given #{params}, request_params[:host]").returns("s3.amazonaws.com") do
+      method.call(params)[:host]
+    end
+
+    params = {region: 'us-west-2'}
+    tests("given #{params}, request_params[:host]").returns("s3.us-west-2.amazonaws.com") do
+      method.call(params)[:host]
+    end
+
+    params= {region: 'us-east-1', host: 's3.us-west-2.amazonaws.com'}
+    tests("given #{params}, request_params[:host]").returns("s3.us-west-2.amazonaws.com") do
+      method.call(params)[:host]
+    end
+
+    params = {object_name: 'image.png'}
+    tests("given #{params}, request_params[:host]").returns("/image.png") do
+      method.call(params)[:path]
+    end
+
+    params = {object_name: 'image.png', path: '/images/image.png'}
+    tests("given #{params}, request_params[:host]").returns("/images/image.png") do
+      method.call(params)[:path]
+    end
+  end
 
   directory_attributes = {
     :key => uniq_id('fogdirectorytests')
@@ -85,7 +198,5 @@ Shindo.tests("Storage[:aws] | directory", ["aws"]) do
         @instance.versioning?
       end
     end
-
   end
-
 end

data/tests/models/storage/files_tests.rb

@@ -50,6 +50,38 @@ Shindo.tests("Storage[:aws] | files", ["aws"]) do
       end
     end
 
+    tests('#normalize_headers') do
+      files = @directory.files
+      response = Excon::Response.new
+      current_time = Time.new(2021, 02, 21)
+
+      response.headers['last-modified'] = current_time.to_s
+      response.headers['etag'] = '12345'
+      response.headers['ETAG'] = '12345'
+      response.headers['Cache-Control'] = 'no-cache'
+      response.headers['Content-disposition'] = 'attachment'
+      response.headers['content-length'] = 100
+      response.headers['content-Encoding'] = 'gzip'
+      response.headers['content-md5'] = 'ABCDEAB'
+      response.headers['content-Md5'] = 'ABCDEAB'
+      response.headers['ConTent-Type'] = 'application/json'
+
+      expected = {
+        'Last-Modified' => current_time,
+        'ETag' => '12345',
+        'Cache-Control' => 'no-cache',
+        'Content-Disposition' => 'attachment',
+        'Content-Length' => 100,
+        'Content-Encoding' => 'gzip',
+        'Content-MD5' => 'ABCDEAB',
+        'Content-Type' => 'application/json'
+      }
+
+      tests('header keys are normalized').returns(expected) do
+        files.normalize_headers(response)
+        response.headers
+      end
+    end
   end
 
   @directory.versions.each(&:destroy)

data/tests/requests/compute/security_group_tests.rb

@@ -19,6 +19,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
         'groups'      => [{ 'groupName' => Fog::Nullable::String, 'userId' => String, 'groupId' => String }],
         'ipProtocol'  => String,
         'ipRanges'    => [Fog::Nullable::Hash],
+        'ipv6Ranges'  => [Fog::Nullable::Hash],
         'toPort'      => Fog::Nullable::Integer,
       }],
       'ipPermissionsEgress' => [],
@@ -54,16 +55,19 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
       {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
         "fromPort"=>1,
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "toPort"=>65535},
       {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
         "fromPort"=>1,
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"udp",
         "toPort"=>65535},
       {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}],
         "fromPort"=>-1,
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"icmp",
         "toPort"=>-1}
     ]
@@ -88,6 +92,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
         [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
           {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "fromPort"=>1,
         "toPort"=>65535},
@@ -95,6 +100,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
         [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
           {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"udp",
         "fromPort"=>1,
         "toPort"=>65535},
@@ -102,6 +108,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
         [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default},
           {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"icmp",
         "fromPort"=>-1,
         "toPort"=>-1}
@@ -133,6 +140,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
     expected_permissions += [
       {"groups"=>[],
         "ipRanges"=>[{"cidrIp"=>"10.0.0.0/8"}],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "fromPort"=>22,
         "toPort"=>22}
@@ -164,7 +172,8 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
       'IpPermissions' => [
         {
           'IpProtocol' => 'tcp', 'FromPort' => '80', 'ToPort' => '80',
-          'IpRanges' => [{ 'CidrIp' => '192.168.0.0/24' }]
+          'IpRanges' => [{ 'CidrIp' => '192.168.0.0/24' }],
+          'Ipv6Ranges' => []
         }
       ]
     }
@@ -177,6 +186,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
     expected_permissions += [
       {"groups"=>[],
         "ipRanges"=>[{"cidrIp"=>"192.168.0.0/24"}],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "fromPort"=>80,
         "toPort"=>80}
@@ -204,6 +214,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do
     expected_permissions += [
       {"groups"=>[{"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}],
         "ipRanges"=>[],
+        "ipv6Ranges"=>[],
         "ipProtocol"=>"tcp",
         "fromPort"=>8000,
         "toPort"=>8000}