fog-aws 3.5.0 → 3.28.0

data/lib/fog/aws/storage.rb

@@ -14,6 +14,17 @@ module Fog
         'https' => 443
       }
 
+      DEFAULT_CONNECTION_OPTIONS = {
+        retry_limit: 5,
+        retry_interval: 1,
+        retry_errors: [
+          Excon::Error::Timeout, Excon::Error::Socket, Excon::Error::Server
+        ]
+      }
+
+      MIN_MULTIPART_CHUNK_SIZE = 5242880
+      MAX_SINGLE_PUT_SIZE = 5368709120
+
       VALID_QUERY_KEYS = %w[
         acl
         cors
@@ -43,7 +54,7 @@ module Fog
       ]
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :virtual_host, :cname
+      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size, :aws_credentials_refresh_threshold_seconds, :disable_content_md5_validation, :sts_endpoint
 
       secrets    :aws_secret_access_key, :hmac
 
@@ -84,6 +95,7 @@ module Fog
       request :get_object_http_url
       request :get_object_https_url
       request :get_object_url
+      request :get_object_tagging
       request :get_request_payment
       request :get_service
       request :head_bucket
@@ -107,12 +119,26 @@ module Fog
       request :put_object
       request :put_object_acl
       request :put_object_url
+      request :put_object_tagging
       request :put_request_payment
       request :sync_clock
       request :upload_part
+      request :upload_part_copy
 
       module Utils
         attr_accessor :region
+        attr_accessor :disable_content_md5_validation
+
+        # Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB.
+        # Other S3-compatible storages (like, Ceph) do not have such limit.
+        # Ceph shows much better performance when file is copied as a whole, in a single request.
+        # fog-aws user can use these settings to configure chunk sizes.
+        # A non-positive value will tell fog-aws to use a single put/copy request regardless of file size.
+        #
+        # @return [Integer]
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html
+        attr_reader :max_put_chunk_size
+        attr_reader :max_copy_chunk_size
 
         def cdn
           @cdn ||= Fog::AWS::CDN.new(
@@ -168,6 +194,12 @@ module Fog
           params_to_url(params)
         end
 
+        # @param value [int]
+        # @param description [str]
+        def validate_chunk_size(value, description)
+          raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE
+        end
+
         private
 
         def validate_signature_version!
@@ -176,6 +208,16 @@ module Fog
           end
         end
 
+        def init_max_put_chunk_size!(options = {})
+          @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size')
+        end
+
+        def init_max_copy_chunk_size!(options = {})
+          @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size')
+        end
+
         def v4_signed_params_for_url(params, expires)
           now = Fog::Time.now
 
@@ -225,7 +267,7 @@ module Fog
           when %r{\Acn-.*}
             "s3.#{region}.amazonaws.com.cn"
           else
-            "s3-#{region}.amazonaws.com"
+            "s3.#{region}.amazonaws.com"
           end
         end
 
@@ -281,10 +323,10 @@ module Fog
               path_style = params.fetch(:path_style, @path_style)
               if !path_style
                 if COMPLIANT_BUCKET_NAMES !~ bucket_name
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 elsif scheme == 'https' && !path_style && bucket_name =~ /\./
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 end
               end
@@ -295,6 +337,8 @@ module Fog
                 host = params.fetch(:cname, bucket_name)
               elsif path_style
                 path = bucket_to_path bucket_name, path
+              elsif host.start_with?("#{bucket_name}.")
+                # no-op
               else
                 host = [bucket_name, host].join('.')
               end
@@ -447,6 +491,12 @@ module Fog
 
 
           @path_style = options[:path_style] || false
+
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
+          @disable_content_md5_validation = options[:disable_content_md5_validation] || false
+
           @signature_version = options.fetch(:aws_signature_version, 4)
           validate_signature_version!
           setup_credentials(options)
@@ -461,6 +511,8 @@ module Fog
         end
 
         def setup_credentials(options)
+          @aws_credentials_refresh_threshold_seconds = options[:aws_credentials_refresh_threshold_seconds]
+
           @aws_access_key_id = options[:aws_access_key_id]
           @aws_secret_access_key = options[:aws_secret_access_key]
           @aws_session_token     = options[:aws_session_token]
@@ -502,13 +554,20 @@ module Fog
           @use_iam_profile = options[:use_iam_profile]
           @instrumentor       = options[:instrumentor]
           @instrumentor_name  = options[:instrumentor_name] || 'fog.aws.storage'
-          @connection_options     = options[:connection_options] || {}
+          @connection_options =
+            DEFAULT_CONNECTION_OPTIONS.merge(options[:connection_options] || {})
           @persistent = options.fetch(:persistent, false)
           @acceleration = options.fetch(:acceleration, false)
           @signature_version = options.fetch(:aws_signature_version, 4)
+          @enable_signature_v4_streaming = options.fetch(:enable_signature_v4_streaming, false)
           validate_signature_version!
           @path_style = options[:path_style]  || false
 
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
+          @disable_content_md5_validation = options[:disable_content_md5_validation] || false
+
           @region = options[:region] || DEFAULT_REGION
 
           if @endpoint = options[:endpoint]
@@ -534,6 +593,8 @@ module Fog
 
 
         def setup_credentials(options)
+          @aws_credentials_refresh_threshold_seconds = options[:aws_credentials_refresh_threshold_seconds]
+
           @aws_access_key_id     = options[:aws_access_key_id]
           @aws_secret_access_key = options[:aws_secret_access_key]
           @aws_session_token     = options[:aws_session_token]
@@ -587,20 +648,24 @@ module Fog
           if @signature_version == 4
             params[:headers]['x-amz-date'] = date.to_iso8601_basic
             if params[:body].respond_to?(:read)
-              # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
-              # We ignore the bit about setting the content-encoding to aws-chunked because
-              # this can cause s3 to serve files with a blank content encoding which causes problems with some CDNs
-              # AWS have confirmed that s3 can infer that the content-encoding is aws-chunked from the x-amz-content-sha256 header
-              #
-              params[:headers]['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
-              params[:headers]['x-amz-decoded-content-length'] = params[:headers].delete 'Content-Length'
+              if @enable_signature_v4_streaming
+                # See http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+                # We ignore the bit about setting the content-encoding to aws-chunked because
+                # this can cause s3 to serve files with a blank content encoding which causes problems with some CDNs
+                # AWS have confirmed that s3 can infer that the content-encoding is aws-chunked from the x-amz-content-sha256 header
+                #
+                params[:headers]['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
+                params[:headers]['x-amz-decoded-content-length'] = params[:headers].delete 'Content-Length'
+              else
+                params[:headers]['x-amz-content-sha256'] = 'UNSIGNED-PAYLOAD'
+              end
             else
               params[:headers]['x-amz-content-sha256'] ||= OpenSSL::Digest::SHA256.hexdigest(params[:body] || '')
             end
             signature_components = @signer.signature_components(params, date, params[:headers]['x-amz-content-sha256'])
             params[:headers]['Authorization'] = @signer.components_to_header(signature_components)
 
-            if params[:body].respond_to?(:read)
+            if params[:body].respond_to?(:read) && @enable_signature_v4_streaming
               body = params.delete :body
               params[:request_block] = S3Streamer.new(body, signature_components['X-Amz-Signature'], @signer, date)
             end

data/lib/fog/aws/support.rb

@@ -4,7 +4,7 @@ module Fog
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :host, :path, :port, :scheme, :instrumentor, :instrumentor_name, :region, :persistent, :aws_session_token
+      recognizes :host, :path, :port, :scheme, :instrumentor, :instrumentor_name, :region, :persistent, :aws_session_token, :aws_credentials_expire_at, :sts_endpoint
 
       model_path 'fog/aws/models/support'
       request_path 'fog/aws/requests/support'

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "3.5.0"
+    VERSION = '3.28.0'.freeze
   end
 end

data/lib/fog/aws.rb

@@ -1,3 +1,4 @@
+require 'base64'
 require 'fog/core'
 require 'fog/xml'
 require 'fog/json'
@@ -26,6 +27,7 @@ module Fog
     autoload :ECS,              File.expand_path('../aws/ecs', __FILE__)
     autoload :EFS,              File.expand_path('../aws/efs', __FILE__)
     autoload :ELB,              File.expand_path('../aws/elb', __FILE__)
+    autoload :ELBV2,            File.expand_path('../aws/elbv2', __FILE__)
     autoload :EMR,              File.expand_path('../aws/emr', __FILE__)
     autoload :ElasticBeanstalk, File.expand_path('../aws/beanstalk', __FILE__)
     autoload :Elasticache,      File.expand_path('../aws/elasticache', __FILE__)
@@ -58,6 +60,7 @@ module Fog
     service(:ecs,             'ECS')
     service(:efs,             'EFS')
     service(:elb,             'ELB')
+    service(:elbv2,           'ELBV2')
     service(:emr,             'EMR')
     service(:federation,      'Federation')
     service(:glacier,         'Glacier')
@@ -218,18 +221,24 @@ module Fog
 
     def self.regions
       @regions ||= [
+        'af-south-1',
+        'ap-east-1',
         'ap-northeast-1', 'ap-northeast-2', 'ap-northeast-3',
         'ap-south-1',
-        'ap-southeast-1', 'ap-southeast-2',
+        'ap-southeast-1', 'ap-southeast-2', 'ap-southeast-3', 'ap-southeast-4',
         'ca-central-1',
+        'ca-west-1',
         'cn-north-1',
         'cn-northwest-1',
         'eu-central-1',
         'eu-north-1',
-        'eu-west-1', 'eu-west-2', 'eu-west-3',
+        'eu-west-1', 'eu-west-2', 'eu-west-3', 'eu-south-1', 'eu-south-2',
+        'il-central-1',
+        'me-south-1', 'me-central-1',
         'us-east-1', 'us-east-2',
         'us-west-1', 'us-west-2',
         'sa-east-1',
+        'us-gov-east-1',
         'us-gov-west-1'
       ]
     end