fog-aws 3.0.0 → 3.10.0

data/lib/fog/aws/requests/sts/assume_role_with_web_identity.rb

@@ -6,12 +6,13 @@ module Fog
 
         def assume_role_with_web_identity(role_arn, web_identity_token, role_session_name, options={})
           request_unsigned(
-            'Action'          => 'AssumeRoleWithWebIdentity',
-            'RoleArn'         => role_arn,
-            'RoleSessionName' => role_session_name,
-            'DurationSeconds' => options[:duration] || 3600,
-            :idempotent       => true,
-            :parser           => Fog::Parsers::AWS::STS::AssumeRoleWithWebIdentity.new
+            'Action'            => 'AssumeRoleWithWebIdentity',
+            'RoleArn'           => role_arn,
+            'RoleSessionName'   => role_session_name,
+            'WebIdentityToken'  => web_identity_token,
+            'DurationSeconds'   => options[:duration] || 3600,
+            :idempotent         => true,
+            :parser             => Fog::Parsers::AWS::STS::AssumeRoleWithWebIdentity.new
           )
         end
       end

data/lib/fog/aws/service_mapper.rb

@@ -14,19 +14,19 @@ module Fog
           when :beanstalk
             Fog::AWS::ElasticBeanstalk
           when :cdn
-            Fog::CDN::AWS
+            Fog::AWS::CDN
           when :cloud_formation
             Fog::AWS::CloudFormation
           when :cloud_watch
             Fog::AWS::CloudWatch
           when :compute
-            Fog::Compute::AWS
+            Fog::AWS::Compute
           when :data_pipeline
             Fog::AWS::DataPipeline
           when :ddb, :dynamodb
             Fog::AWS::DynamoDB
           when :dns
-            Fog::DNS::AWS
+            Fog::AWS::DNS
           when :elasticache
             Fog::AWS::Elasticache
           when :elb
@@ -46,7 +46,7 @@ module Fog
           when :sqs
             Fog::AWS::SQS
           when :eu_storage, :storage
-            Fog::Storage::AWS
+            Fog::AWS::Storage
           when :rds
             Fog::AWS::RDS
           when :sns
@@ -126,4 +126,4 @@ module Fog
       end
     end
   end
-end
+end

data/lib/fog/aws/storage.rb

@@ -1,11 +1,12 @@
 module Fog
-  module Storage
-    class AWS < Fog::Service
+  module AWS
+    class Storage < Fog::Service
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       COMPLIANT_BUCKET_NAMES = /^(?:[a-z]|\d(?!\d{0,2}(?:\.\d{1,3}){3}$))(?:[a-z0-9]|\.(?![\.\-])|\-(?![\.])){1,61}[a-z0-9]$/
 
       DEFAULT_REGION = 'us-east-1'
+      ACCELERATION_HOST = 's3-accelerate.amazonaws.com'
 
       DEFAULT_SCHEME = 'https'
       DEFAULT_SCHEME_PORT = {
@@ -42,7 +43,7 @@ module Fog
       ]
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :instrumentor, :instrumentor_name, :aws_signature_version
+      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname
 
       secrets    :aws_secret_access_key, :hmac
 
@@ -83,6 +84,7 @@ module Fog
       request :get_object_http_url
       request :get_object_https_url
       request :get_object_url
+      request :get_object_tagging
       request :get_request_payment
       request :get_service
       request :head_bucket
@@ -106,9 +108,11 @@ module Fog
       request :put_object
       request :put_object_acl
       request :put_object_url
+      request :put_object_tagging
       request :put_request_payment
       request :sync_clock
       request :upload_part
+      request :upload_part_copy
 
       module Utils
         attr_accessor :region
@@ -130,7 +134,7 @@ module Fog
         end
 
         def url(params, expires)
-          Fog::Logger.deprecation("Fog::Storage::AWS => #url is deprecated, use #https_url instead [light_black](#{caller.first})[/]")
+          Fog::Logger.deprecation("Fog::AWS::Storage => #url is deprecated, use #https_url instead [light_black](#{caller.first})[/]")
           https_url(params, expires)
         end
 
@@ -221,10 +225,10 @@ module Fog
           case region.to_s
           when DEFAULT_REGION, ''
             's3.amazonaws.com'
-          when 'cn-north-1'
-            's3.cn-north-1.amazonaws.com.cn'
+          when %r{\Acn-.*}
+            "s3.#{region}.amazonaws.com.cn"
           else
-            "s3-#{region}.amazonaws.com"
+            "s3.#{region}.amazonaws.com"
           end
         end
 
@@ -280,16 +284,22 @@ module Fog
               path_style = params.fetch(:path_style, @path_style)
               if !path_style
                 if COMPLIANT_BUCKET_NAMES !~ bucket_name
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 elsif scheme == 'https' && !path_style && bucket_name =~ /\./
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 end
               end
 
-              if path_style
+              # uses the bucket name as host if `virtual_host: true`, you can also
+              # manually specify the cname if required.
+              if params[:virtual_host]
+                host = params.fetch(:cname, bucket_name)
+              elsif path_style
                 path = bucket_to_path bucket_name, path
+              elsif host.start_with?("#{bucket_name}.")
+                # no-op
               else
                 host = [bucket_name, host].join('.')
               end
@@ -316,7 +326,8 @@ module Fog
         def params_to_url(params)
           query = params[:query] && params[:query].map do |key, value|
             if value
-              [key, escape(value.to_s)].join('=')
+              # URL parameters need / to be escaped
+              [key, Fog::AWS.escape(value.to_s)].join('=')
             else
               key
             end
@@ -498,7 +509,9 @@ module Fog
           @instrumentor_name  = options[:instrumentor_name] || 'fog.aws.storage'
           @connection_options     = options[:connection_options] || {}
           @persistent = options.fetch(:persistent, false)
+          @acceleration = options.fetch(:acceleration, false)
           @signature_version = options.fetch(:aws_signature_version, 4)
+          @enable_signature_v4_streaming = options.fetch(:enable_signature_v4_streaming, true)
           validate_signature_version!
           @path_style = options[:path_style]  || false
 
@@ -515,6 +528,7 @@ module Fog
             @port       = options[:port]        || DEFAULT_SCHEME_PORT[@scheme]
           end
 
+          @host = ACCELERATION_HOST if @acceleration
           setup_credentials(options)
         end
 
@@ -579,20 +593,24 @@ module Fog
           if @signature_version == 4
             params[:headers]['x-amz-date'] = date.to_iso8601_basic
             if params[:body].respond_to?(:read)
-              # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
-              # We ignore the bit about setting the content-encoding to aws-chunked because
-              # this can cause s3 to serve files with a blank content encoding which causes problems with some CDNs
-              # AWS have confirmed that s3 can infer that the content-encoding is aws-chunked from the x-amz-content-sha256 header
-              #
-              params[:headers]['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
-              params[:headers]['x-amz-decoded-content-length'] = params[:headers].delete 'Content-Length'
+              if @enable_signature_v4_streaming
+                # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+                # We ignore the bit about setting the content-encoding to aws-chunked because
+                # this can cause s3 to serve files with a blank content encoding which causes problems with some CDNs
+                # AWS have confirmed that s3 can infer that the content-encoding is aws-chunked from the x-amz-content-sha256 header
+                #
+                params[:headers]['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
+                params[:headers]['x-amz-decoded-content-length'] = params[:headers].delete 'Content-Length'
+              else
+                params[:headers]['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
+              end
             else
               params[:headers]['x-amz-content-sha256'] ||= OpenSSL::Digest::SHA256.hexdigest(params[:body] || '')
             end
             signature_components = @signer.signature_components(params, date, params[:headers]['x-amz-content-sha256'])
             params[:headers]['Authorization'] = @signer.components_to_header(signature_components)
 
-            if params[:body].respond_to?(:read)
+            if params[:body].respond_to?(:read) && @enable_signature_v4_streaming
               body = params.delete :body
               params[:request_block] = S3Streamer.new(body, signature_components['X-Amz-Signature'], @signer, date)
             end
@@ -763,4 +781,17 @@ DATA
       end
     end
   end
+
+  # @deprecated
+  module Storage
+    # @deprecated
+    class AWS < Fog::AWS::Storage
+      # @deprecated
+      # @overrides Fog::Service.new (from the fog-core gem)
+      def self.new(*)
+        Fog::Logger.deprecation 'Fog::Storage::AWS is deprecated, please use Fog::AWS::Storage.'
+        super
+      end
+    end
+  end
 end

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "3.0.0"
+    VERSION = "3.10.0"
   end
 end

data/tests/credentials_tests.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Shindo.tests('AWS | credentials', ['aws']) do
   old_mock_value = Excon.defaults[:mock]
   fog_was_mocked = Fog.mocking?
@@ -5,8 +7,10 @@ Shindo.tests('AWS | credentials', ['aws']) do
   Fog.unmock!
   begin
     Excon.defaults[:mock] = true
-    Excon.stub({:method => :get, :path => "/latest/meta-data/iam/security-credentials/"}, {:status => 200, :body => 'arole'})
-    Excon.stub({:method => :get, :path => "/latest/meta-data/placement/availability-zone/"}, {:status => 200, :body => 'us-west-1a'})
+    Excon.stub({ method: :put, path: '/latest/api/token' }, { status: 200, body: 'token1234' })
+
+    Excon.stub({ method: :get, path: '/latest/meta-data/iam/security-credentials/' }, { status: 200, body: 'arole' })
+    Excon.stub({ method: :get, path: '/latest/meta-data/placement/availability-zone/' }, { status: 200, body: 'us-west-1a' })
 
     expires_at = Time.at(Time.now.to_i + 500)
     credentials = {
@@ -16,69 +20,128 @@ Shindo.tests('AWS | credentials', ['aws']) do
       'Expiration' => expires_at.xmlschema
     }
 
-    Excon.stub({:method => :get, :path => "/latest/meta-data/iam/security-credentials/arole"}, {:status => 200, :body => Fog::JSON.encode(credentials)})
+    Excon.stub({ method: :get, path: '/latest/meta-data/iam/security-credentials/arole' }, { status: 200, body: Fog::JSON.encode(credentials) })
+
+    tests('#fetch_credentials') do
+      returns(aws_access_key_id: 'dummykey',
+              aws_secret_access_key: 'dummysecret',
+              aws_session_token: 'dummytoken',
+              region: 'us-west-1',
+              aws_credentials_expire_at: expires_at) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
+
+    tests('#fetch_credentials when the v2 token 404s') do
+      Excon.stub({ method: :put, path: '/latest/api/token' }, { status: 404, body: 'not found' })
+      returns(aws_access_key_id: 'dummykey',
+              aws_secret_access_key: 'dummysecret',
+              aws_session_token: 'dummytoken',
+              region: 'us-west-1',
+              aws_credentials_expire_at: expires_at) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
+
+    tests('#fetch_credentials when the v2 disabled') do
+      returns(aws_access_key_id: 'dummykey',
+              aws_secret_access_key: 'dummysecret',
+              aws_session_token: 'dummytoken',
+              region: 'us-west-1',
+              aws_credentials_expire_at: expires_at) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, disable_imds_v2: true) }
+    end
+
+    ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = '/v1/credentials?id=task_id'
+    Excon.stub({ method: :get, path: '/v1/credentials?id=task_id' }, { status: 200, body: Fog::JSON.encode(credentials) })
 
-    tests("#fetch_credentials") do
-      returns({:aws_access_key_id => 'dummykey',
-                :aws_secret_access_key => 'dummysecret',
-                :aws_session_token => 'dummytoken',
-                :region => "us-west-1",
-                :aws_credentials_expire_at => expires_at}) { Fog::Compute::AWS.fetch_credentials(:use_iam_profile => true) }
+    tests('#fetch_credentials') do
+      returns(aws_access_key_id: 'dummykey',
+              aws_secret_access_key: 'dummysecret',
+              aws_session_token: 'dummytoken',
+              region: 'us-west-1',
+              aws_credentials_expire_at: expires_at) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
     end
 
-    ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = '/v1/credentials?id=task_id'
-    Excon.stub({:method => :get, :path => '/v1/credentials?id=task_id'}, {:status => 200, :body => Fog::JSON.encode(credentials)})
+    ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = nil
 
-    tests("#fetch_credentials") do
-      returns({:aws_access_key_id => 'dummykey',
-                :aws_secret_access_key => 'dummysecret',
-                :aws_session_token => 'dummytoken',
-                :region => "us-west-1",
-                :aws_credentials_expire_at => expires_at}) { Fog::Compute::AWS.fetch_credentials(:use_iam_profile => true) }
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = File.dirname(__FILE__) + '/lorem.txt'
+    ENV['AWS_ROLE_ARN'] = "dummyrole"
+    ENV['AWS_ROLE_SESSION_NAME'] = "dummyrolesessionname"
+    document = 
+      '<AssumeRoleWithWebIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">'\
+        '<AssumeRoleWithWebIdentityResult>'\
+          '<Credentials>'\
+            '<SessionToken>dummytoken</SessionToken>'\
+            '<SecretAccessKey>dummysecret</SecretAccessKey>'\
+            "<Expiration>#{expires_at.xmlschema}</Expiration>"\
+            '<AccessKeyId>dummykey</AccessKeyId>'\
+          '</Credentials>'\
+        '</AssumeRoleWithWebIdentityResult>'\
+      '</AssumeRoleWithWebIdentityResponse>'
+    
+    Excon.stub({method: :get, path: "/", idempotent: true}, { status: 200, body: document})
+
+    tests('#fetch_credentials token based') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
     end
 
-    ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = nil
+    ENV['AWS_ROLE_SESSION_NAME'] = nil
 
-    compute = Fog::Compute::AWS.new(:use_iam_profile => true)
+    tests('#fetch_credentials token based without session name') do
+      returns(
+        aws_access_key_id: 'dummykey',
+        aws_secret_access_key: 'dummysecret',
+        aws_session_token: 'dummytoken',
+        region: 'us-west-1',
+        aws_credentials_expire_at: expires_at
+      ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
+    end
 
-    tests("#refresh_credentials_if_expired") do
-      returns(nil){compute.refresh_credentials_if_expired}
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
+
+    compute = Fog::AWS::Compute.new(use_iam_profile: true)
+
+    tests('#refresh_credentials_if_expired') do
+      returns(nil) { compute.refresh_credentials_if_expired }
     end
 
     credentials['AccessKeyId'] = 'newkey'
     credentials['SecretAccessKey'] = 'newsecret'
     credentials['Expiration'] = (expires_at + 10).xmlschema
 
-    Excon.stub({:method => :get, :path => "/latest/meta-data/iam/security-credentials/arole"}, {:status => 200, :body => Fog::JSON.encode(credentials)})
+    Excon.stub({ method: :get, path: '/latest/meta-data/iam/security-credentials/arole' }, { status: 200, body: Fog::JSON.encode(credentials) })
 
     Fog::Time.now = expires_at + 1
-    tests("#refresh_credentials_if_expired") do
-      returns(true){compute.refresh_credentials_if_expired}
-      returns("newkey"){ compute.instance_variable_get(:@aws_access_key_id)}
+    tests('#refresh_credentials_if_expired') do
+      returns(true) { compute.refresh_credentials_if_expired }
+      returns('newkey') { compute.instance_variable_get(:@aws_access_key_id) }
     end
     Fog::Time.now = Time.now
 
-    default_credentials = Fog::Compute::AWS.fetch_credentials({})
-    tests("#fetch_credentials when the url 404s") do
-      Excon.stub({:method => :get, :path => "/latest/meta-data/iam/security-credentials/"}, {:status => 404, :body => 'not bound'})
-      Excon.stub({:method => :get, :path => "/latest/meta-data/placement/availability-zone/"}, {:status => 400, :body => 'not found'})
-      returns(default_credentials) {Fog::Compute::AWS.fetch_credentials(:use_iam_profile => true)}
+    default_credentials = Fog::AWS::Compute.fetch_credentials({})
+    tests('#fetch_credentials when the url 404s') do
+      Excon.stub({ method: :put, path: '/latest/api/token' }, { status: 404, body: 'not found' })
+      Excon.stub({ method: :get, path: '/latest/meta-data/iam/security-credentials/' }, { status: 404, body: 'not bound' })
+      Excon.stub({ method: :get, path: '/latest/meta-data/placement/availability-zone/' }, { status: 400, body: 'not found' })
+      returns(default_credentials) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
     end
 
     mocked_credentials = {
-      :aws_access_key_id => "access-key-id",
-      :aws_secret_access_key => "secret-access-key",
-      :aws_session_token => "session-token",
-      :aws_credentials_expire_at => Time.at(Time.now.to_i + 500).xmlschema
+      aws_access_key_id: 'access-key-id',
+      aws_secret_access_key: 'secret-access-key',
+      aws_session_token: 'session-token',
+      aws_credentials_expire_at: Time.at(Time.now.to_i + 500).xmlschema
     }
-    tests("#fetch_credentials when mocking") do
+    tests('#fetch_credentials when mocking') do
       Fog.mock!
-      Fog::Compute::AWS::Mock.data[:iam_role_based_creds] = mocked_credentials
-      returns(mocked_credentials) {Fog::Compute::AWS.fetch_credentials(:use_iam_profile => true)}
+      Fog::AWS::Compute::Mock.data[:iam_role_based_creds] = mocked_credentials
+      returns(mocked_credentials) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) }
     end
-
   ensure
-    ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = nil
+    ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = nil
+    ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil
     Excon.stubs.clear
     Excon.defaults[:mock] = old_mock_value
     Fog.mock! if fog_was_mocked

data/tests/helper.rb

@@ -1,7 +1,7 @@
 begin
   require 'simplecov'
   SimpleCov.start
-  SimpleCov.command_name "Shindo"
+  SimpleCov.command_name 'Shindo'
 rescue LoadError => e
   $stderr.puts "not recording test coverage: #{e.inspect}"
 end
@@ -10,7 +10,7 @@ require File.expand_path('../../lib/fog/aws', __FILE__)
 
 Bundler.require(:test)
 
-Excon.defaults.merge!(:debug_request => true, :debug_response => true)
+Excon.defaults.merge!(debug_request: true, debug_response: true)
 
 require File.expand_path(File.join(File.dirname(__FILE__), 'helpers', 'mock_helper'))
 

data/tests/helpers/collection_helper.rb

@@ -1,6 +1,5 @@
 def collection_tests(collection, params = {}, mocks_implemented = true)
   tests('success') do
-
     tests("#new(#{params.inspect})").succeeds do
       pending if Fog.mocking? && !mocks_implemented
       collection.new(params)
@@ -34,7 +33,7 @@ def collection_tests(collection, params = {}, mocks_implemented = true)
       pending if Fog.mocking? && !mocks_implemented
 
       methods = [
-        'all?', 'any?', 'find',  'detect', 'collect', 'map',
+        'all?', 'any?', 'find', 'detect', 'collect', 'map',
         'find_index', 'flat_map', 'collect_concat', 'group_by',
         'none?', 'one?'
       ]
@@ -43,7 +42,7 @@ def collection_tests(collection, params = {}, mocks_implemented = true)
         if collection.respond_to?(enum_method)
           tests("##{enum_method}").succeeds do
             block_called = false
-            collection.send(enum_method) {|x| block_called = true }
+            collection.send(enum_method) { block_called = true }
             block_called
           end
         end
@@ -55,7 +54,7 @@ def collection_tests(collection, params = {}, mocks_implemented = true)
         if collection.respond_to?(enum_method)
           tests("##{enum_method}").succeeds do
             block_called = false
-            collection.send(enum_method) {|x| block_called = true; 0 }
+            collection.send(enum_method) { block_called = true; 0 }
             block_called
           end
         end