fog-aws 1.4.1 → 3.27.0

data/lib/fog/aws/models/storage/directory.rb

@@ -2,8 +2,8 @@ require 'fog/aws/models/storage/files'
 require 'fog/aws/models/storage/versions'
 
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class Directory < Fog::Model
         VALID_ACLS = ['private', 'public-read', 'public-read-write', 'authenticated-read']
 
@@ -12,7 +12,6 @@ module Fog
         identity  :key,           :aliases => ['Name', 'name']
 
         attribute :creation_date, :aliases => 'CreationDate', :type => 'time'
-        attribute :location,      :aliases => 'LocationConstraint', :type => 'string'
 
         def acl=(new_acl)
           unless VALID_ACLS.include?(new_acl)
@@ -30,8 +29,32 @@ module Fog
           false
         end
 
+        # @param options [Hash] (defaults to: {}) — a customizable set of options.
+        #   Consider tuning this values for big buckets.
+        # @option options timeout [Integer] — default: Fog.timeout — Maximum number of
+        #   seconds to wait for the bucket to be empty.
+        # @option options interval [Proc|Integer] — default: Fog.interval — Seconds to wait before
+        #   retrying to check if the bucket is empty.
+        def destroy!(options = {})
+          requires :key
+          options = {
+            timeout: Fog.timeout,
+            interval: Fog.interval,
+          }.merge(options)
+
+          attempts = 0
+          begin
+            clear!
+            Fog.wait_for(options[:timeout], options[:interval]) { objects_keys.size == 0 }
+            service.delete_bucket(key)
+            true
+          rescue Excon::Errors::HTTPStatusError
+            false
+          end
+        end
+
         def location
-          @location ||= (bucket_location || AWS::DEFAULT_REGION)
+          @location ||= (bucket_location || Storage::DEFAULT_REGION)
         end
 
         # NOTE: you can't change the region once the bucket is created
@@ -40,7 +63,7 @@ module Fog
         end
 
         def files
-          @files ||= Fog::Storage::AWS::Files.new(:directory => self, :service => service)
+          @files ||= Fog::AWS::Storage::Files.new(:directory => self, :service => service)
         end
 
         def payer
@@ -67,7 +90,7 @@ module Fog
         end
 
         def versions
-          @versions ||= Fog::Storage::AWS::Versions.new(:directory => self, :service => service)
+          @versions ||= Fog::AWS::Storage::Versions.new(:directory => self, :service => service)
         end
 
         def public=(new_public)
@@ -119,6 +142,17 @@ module Fog
           data = service.get_bucket_location(key)
           data.body['LocationConstraint']
         end
+
+        def objects_keys
+          requires :key
+          bucket_query = service.get_bucket(key)
+          bucket_query.body["Contents"].map {|c| c["Key"]}
+        end
+
+        def clear!
+          requires :key
+          service.delete_multiple_objects(key, objects_keys) if objects_keys.size > 0
+        end
       end
     end
   end

data/lib/fog/aws/models/storage/file.rb

@@ -1,9 +1,16 @@
 require 'fog/aws/models/storage/versions'
 
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class File < Fog::Model
+        # @deprecated use {Fog::AWS::Storage::MIN_MULTIPART_CHUNK_SIZE} instead
+        MIN_MULTIPART_CHUNK_SIZE = Fog::AWS::Storage::MIN_MULTIPART_CHUNK_SIZE
+        # @deprecated use {Fog::AWS::Storage::MAX_SINGLE_PUT_SIZE} instead
+        MAX_SINGLE_PUT_SIZE = Fog::AWS::Storage::MAX_SINGLE_PUT_SIZE
+        # @deprecated not used for anything
+        MULTIPART_COPY_THRESHOLD = 15728640
+
         # @see AWS Object docs http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
 
         identity  :key,             :aliases => 'Key'
@@ -25,15 +32,57 @@ module Fog
         attribute :encryption_key,      :aliases => 'x-amz-server-side-encryption-customer-key'
         attribute :version,             :aliases => 'x-amz-version-id'
         attribute :kms_key_id,          :aliases => 'x-amz-server-side-encryption-aws-kms-key-id'
+        attribute :tags,                :aliases => 'x-amz-tagging'
+        attribute :website_redirect_location, :aliases => 'x-amz-website-redirect-location'
+
+        UploadPartData = Struct.new(:part_number, :upload_options, :etag)
+
+        class PartList
+          def initialize(parts = [])
+            @parts = parts
+            @mutex = Mutex.new
+          end
+
+          def push(part)
+            @mutex.synchronize { @parts.push(part) }
+          end
+
+          def shift
+            @mutex.synchronize { @parts.shift }
+          end
+
+          def clear!
+            @mutex.synchronize { @parts.clear }
+          end
+
+          def size
+            @mutex.synchronize { @parts.size }
+          end
+
+          def to_a
+            @mutex.synchronize { @parts.dup }
+          end
+        end
 
         # @note Chunk size to use for multipart uploads.
         #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
         attr_reader :multipart_chunk_size
         def multipart_chunk_size=(mp_chunk_size)
-          raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880
+          service.validate_chunk_size(mp_chunk_size, 'multipart_chunk_size')
           @multipart_chunk_size = mp_chunk_size
         end
 
+        # @note Number of threads used to copy files.
+        def concurrency=(concurrency)
+          raise ArgumentError.new('minimum concurrency is 1') if concurrency.to_i < 1
+
+          @concurrency = concurrency.to_i
+        end
+
+        def concurrency
+          @concurrency || 1
+        end
+
         def acl
           requires :directory, :key
           service.get_object_acl(directory.key, key).body['AccessControlList']
@@ -59,15 +108,16 @@ module Fog
         # @return [File]
         #
         def body
-          return attributes[:body] if attributes[:body]
-          return '' unless last_modified
+          return attributes[:body] if attributes.key?(:body)
 
           file = collection.get(identity)
-          if file
-            attributes[:body] = file.body
-          else
-            attributes[:body] = ''
-          end
+
+          attributes[:body] =
+            if file
+              file.body
+            else
+              ''
+            end
         end
 
         # Set body attribute.
@@ -98,7 +148,17 @@ module Fog
         #
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+
+          self.multipart_chunk_size = service.max_copy_chunk_size if multipart_chunk_size.nil?
+
+          if multipart_chunk_size > 0 && self.content_length.to_i >= multipart_chunk_size
+            upload_part_options = options.select { |key, _| ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym) }
+            upload_part_options = upload_part_options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" })
+            multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          else
+            service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          end
+
           target_directory = service.directories.new(:key => target_directory_key)
           target_directory.files.head(target_file_key)
         end
@@ -120,7 +180,8 @@ module Fog
 
         remove_method :metadata
         def metadata
-          attributes.reject {|key, value| !(key.to_s =~ /^x-amz-/)}
+          attributes.reject {|key, value| !(key.to_s =~ /^x-amz-/) }
+                    .reject {|key, value| ['x-amz-id-2', 'x-amz-request-id'].include?(key) }
         end
 
         remove_method :metadata=
@@ -185,13 +246,15 @@ module Fog
         # @param [Hash] options
         # @option options [String] acl sets x-amz-acl HTTP header. Valid values include, private | public-read | public-read-write | authenticated-read | bucket-owner-read | bucket-owner-full-control
         # @option options [String] cache_control sets Cache-Control header. For example, 'No-cache'
-        # @option options [String] content_disposition sets Content-Disposition HTTP header. For exampple, 'attachment; filename=testing.txt'
+        # @option options [String] content_disposition sets Content-Disposition HTTP header. For example, 'attachment; filename=testing.txt'
         # @option options [String] content_encoding sets Content-Encoding HTTP header. For example, 'x-gzip'
         # @option options [String] content_md5 sets Content-MD5. For example, '79054025255fb1a26e4bc422aef54eb4'
         # @option options [String] content_type Content-Type. For example, 'text/plain'
         # @option options [String] expires sets number of seconds before AWS Object expires.
         # @option options [String] storage_class sets x-amz-storage-class HTTP header. Defaults to 'STANDARD'. Or, 'REDUCED_REDUNDANCY'
         # @option options [String] encryption sets HTTP encryption header. Set to 'AES256' to encrypt files at rest on S3
+        # @option options [String] tags sets x-amz-tagging HTTP header. For example, 'Org-Id=1' or 'Org-Id=1&Service=MyService'
+        # @option options [String] website_redirect_location sets x-amz-website-redirect-location HTTP header. For example, 'website_redirect_location=http://www.rubydoc.info/github/fog/fog-aws'
         # @return [Boolean] true if no errors
         #
         def save(options = {})
@@ -208,19 +271,19 @@ module Fog
           options['Expires'] = expires if expires
           options.merge!(metadata)
           options['x-amz-storage-class'] = storage_class if storage_class
+          options['x-amz-tagging'] = tags if tags
+          options['x-amz-website-redirect-location'] = website_redirect_location if website_redirect_location
           options.merge!(encryption_headers)
 
-          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = 5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120
-
-          if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
+          self.multipart_chunk_size = service.max_put_chunk_size if multipart_chunk_size.nil?
+          if multipart_chunk_size > 0 && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
             data = multipart_save(options)
             merge_attributes(data.body)
           else
             data = service.put_object(directory.key, key, body, options)
-            merge_attributes(data.headers.reject {|key, value| ['Content-Length', 'Content-Type'].include?(key)})
+            merge_attributes(data.headers.reject {|key, value| ['Connection', 'Content-Length', 'Content-Type'].include?(key)})
           end
-          self.etag.gsub!('"','') if self.etag
+          self.etag = self.etag.gsub('"','') if self.etag
           self.content_length = Fog::Storage.get_body_size(body)
           self.content_type ||= Fog::Storage.get_content_type(body)
           true
@@ -240,11 +303,11 @@ module Fog
         end
 
         # File version if exists or creates new version.
-        # @return [Fog::Storage::AWS::Version]
+        # @return [Fog::AWS::Storage::Version]
         #
         def versions
           @versions ||= begin
-            Fog::Storage::AWS::Versions.new(
+            Fog::AWS::Storage::Versions.new(
               :file         => self,
               :service   => service
             )
@@ -273,12 +336,12 @@ module Fog
             body.rewind  rescue nil
           end
           while (chunk = body.read(multipart_chunk_size)) do
-            part_upload = service.upload_part(directory.key, key, upload_id, part_tags.size + 1, chunk, part_headers(chunk, options))
+            part_upload = service.upload_part(directory.key, key, upload_id, part_tags.size + 1, chunk, part_headers(chunk))
             part_tags << part_upload.headers["ETag"]
           end
 
           if part_tags.empty? #it is an error to have a multipart upload with no parts
-            part_upload = service.upload_part(directory.key, key, upload_id, 1, '', part_headers('', options))
+            part_upload = service.upload_part(directory.key, key, upload_id, 1, '', part_headers(''))
             part_tags << part_upload.headers["ETag"]
           end
 
@@ -291,6 +354,30 @@ module Fog
           service.complete_multipart_upload(directory.key, key, upload_id, part_tags)
         end
 
+        def multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          # Initiate the upload
+          res = service.initiate_multipart_upload(target_directory_key, target_file_key, options)
+          upload_id = res.body["UploadId"]
+
+          # Store ETags of upload parts
+          part_tags = []
+          pending = PartList.new(create_part_list(upload_part_options))
+          thread_count = self.concurrency
+          completed = PartList.new
+          errors = upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+
+          raise errors.first if errors.any?
+
+          part_tags = completed.to_a.sort_by { |part| part.part_number }.map(&:etag)
+        rescue => e
+          # Abort the upload & reraise
+          service.abort_multipart_upload(target_directory_key, target_file_key, upload_id) if upload_id
+          raise
+        else
+          # Complete the upload
+          service.complete_multipart_upload(target_directory_key, target_file_key, upload_id, part_tags)
+        end
+
         def encryption_headers
           if encryption && encryption_key
             encryption_customer_key_headers
@@ -301,11 +388,18 @@ module Fog
           end
         end
 
-        def part_headers(chunk, options)
-          md5 = Base64.encode64(OpenSSL::Digest::MD5.digest(chunk)).strip
-          encryption_keys = encryption_customer_key_headers.keys
-          encryption_headers = options.select { |key| encryption_keys.include?(key) }
-          { 'Content-MD5' => md5 }.merge(encryption_headers)
+        def part_headers(chunk)
+          base_headers = part_checksum_headers(chunk)
+
+          # Only SSE-C headers needed in the UploadPart request. [1]
+          # x-amz-server-side-encryption and
+          # x-amz-server-side-encryption-aws-kms-key-id are only needed
+          # in the CreateMultipartUpload request. [2]
+          # [1] https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html
+          # [2] https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html
+          base_headers.merge!(encryption_customer_key_headers) if encryption && encryption_key
+
+          base_headers
         end
 
         def encryption_customer_key_headers
@@ -315,6 +409,57 @@ module Fog
             'x-amz-server-side-encryption-customer-key-md5' => Base64.encode64(OpenSSL::Digest::MD5.digest(encryption_key.to_s)).chomp!
           }
         end
+
+        def part_checksum_headers(chunk)
+          if service.disable_content_md5_validation
+            {}
+          else
+            { 'Content-MD5' => Base64.encode64(OpenSSL::Digest::MD5.digest(chunk)).strip }
+          end
+        end
+
+        def create_part_list(upload_part_options)
+          current_pos = 0
+          count = 0
+          pending = []
+
+          while current_pos < self.content_length do
+            start_pos = current_pos
+            end_pos = [current_pos + self.multipart_chunk_size, self.content_length - 1].min
+            range = "bytes=#{start_pos}-#{end_pos}"
+            part_options = upload_part_options.dup
+            part_options['x-amz-copy-source-range'] = range
+            pending << UploadPartData.new(count + 1, part_options, nil)
+            count += 1
+            current_pos = end_pos + 1
+          end
+
+          pending
+        end
+
+        def upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+          threads = []
+
+          thread_count.times do
+            thread = Thread.new do
+              begin
+                while part = pending.shift
+                  part_upload = service.upload_part_copy(target_directory_key, target_file_key, upload_id, part.part_number, part.upload_options)
+                  part.etag = part_upload.body['ETag']
+                  completed.push(part)
+                end
+              rescue => error
+                pending.clear!
+                error
+              end
+            end
+
+            thread.abort_on_exception = true
+            threads << thread
+          end
+
+          threads.map(&:value).compact
+        end
       end
     end
   end

data/lib/fog/aws/models/storage/files.rb

@@ -1,8 +1,8 @@
 require 'fog/aws/models/storage/file'
 
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class Files < Fog::Collection
         extend Fog::Deprecation
         deprecate :get_url, :get_https_url
@@ -15,7 +15,16 @@ module Fog
         attribute :max_keys,        :aliases => ['MaxKeys', 'max-keys']
         attribute :prefix,          :aliases => 'Prefix'
 
-        model Fog::Storage::AWS::File
+        model Fog::AWS::Storage::File
+
+        DASHED_HEADERS = %w(
+          Cache-Control
+          Content-Disposition
+          Content-Encoding
+          Content-Length
+          Content-MD5
+          Content-Type
+        ).freeze
 
         def all(options = {})
           requires :directory
@@ -70,7 +79,7 @@ module Fog
           when /<Code>NoSuchKey<\/Code>/
             nil
           when /<Code>NoSuchBucket<\/Code>/
-            raise(Fog::Storage::AWS::NotFound.new("Directory #{directory.identity} does not exist."))
+            raise(Fog::AWS::Storage::NotFound.new("Directory #{directory.identity} does not exist."))
           else
             raise(error)
           end
@@ -114,8 +123,30 @@ module Fog
         end
 
         def normalize_headers(data)
-          data.headers['Last-Modified'] = Time.parse(data.get_header('Last-Modified'))
-          data.headers['ETag'] = data.get_header('ETag').gsub('"','')
+          data.headers['Last-Modified'] = Time.parse(fetch_and_delete_header(data, 'Last-Modified'))
+
+          if (etag = fetch_and_delete_header(data, 'ETag'))
+            data.headers['ETag'] = etag.gsub('"', '')
+          end
+
+          DASHED_HEADERS.each do |header|
+            value = fetch_and_delete_header(data, header)
+            data.headers[header] = value if value
+          end
+        end
+
+        private
+
+        def fetch_and_delete_header(response, header)
+          value = response.get_header(header)
+
+          return unless value
+
+          response.headers.keys.each do |key|
+            response.headers.delete(key) if key.downcase == header.downcase
+          end
+
+          value
         end
       end
     end

data/lib/fog/aws/models/storage/version.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class Version < Fog::Model
         identity  :version,             :aliases => 'VersionId'
 

data/lib/fog/aws/models/storage/versions.rb

@@ -1,13 +1,13 @@
 require 'fog/aws/models/storage/version'
 
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class Versions < Fog::Collection
         attribute :file
         attribute :directory
 
-        model Fog::Storage::AWS::Version
+        model Fog::AWS::Storage::Version
 
         def all(options = {})
           data = if file

data/lib/fog/aws/parsers/cdn/distribution.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class Distribution < Fog::Parsers::Base
           def reset
             @response = { 'DistributionConfig' => { 'CNAME' => [], 'Logging' => {}, 'TrustedSigners' => [] } }

data/lib/fog/aws/parsers/cdn/get_distribution_list.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class GetDistributionList < Fog::Parsers::Base
           def reset
             @distribution_summary = { 'CNAME' => [], 'TrustedSigners' => [] }

data/lib/fog/aws/parsers/cdn/get_invalidation.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class GetInvalidation < Fog::Parsers::Base
           def reset
             @response = { 'InvalidationBatch' => { 'Path' => [] } }

data/lib/fog/aws/parsers/cdn/get_invalidation_list.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class GetInvalidationList < Fog::Parsers::Base
           def reset
             @invalidation_summary = { }

data/lib/fog/aws/parsers/cdn/get_streaming_distribution_list.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class GetStreamingDistributionList < Fog::Parsers::Base
           def reset
             @distribution_summary = { 'CNAME' => [], 'TrustedSigners' => [] }

data/lib/fog/aws/parsers/cdn/post_invalidation.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class PostInvalidation < Fog::Parsers::Base
           def reset
             @response = { 'InvalidationBatch' => { 'Path' => [] } }

data/lib/fog/aws/parsers/cdn/streaming_distribution.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
+    module AWS
       module CDN
-        module AWS
 
         class StreamingDistribution < Fog::Parsers::Base
           def reset

data/lib/fog/aws/parsers/compute/allocate_address.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class AllocateAddress < Fog::Parsers::Base
           def end_element(name)
             case name

data/lib/fog/aws/parsers/compute/assign_private_ip_addresses.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class AssignPrivateIpAddresses < Fog::Parsers::Base
           def end_element(name)
             case name

data/lib/fog/aws/parsers/compute/associate_address.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class AssociateAddress < Fog::Parsers::Base
           def end_element(name)
             case name

data/lib/fog/aws/parsers/compute/associate_route_table.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class AssociateRouteTable < Fog::Parsers::Base
           def end_element(name)
             case name

data/lib/fog/aws/parsers/compute/attach_network_interface.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class AttachNetworkInterface < Fog::Parsers::Base
           def end_element(name)
             case name

data/lib/fog/aws/parsers/compute/attach_volume.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class AttachVolume < Fog::Parsers::Base
           def end_element(name)
             case name

data/lib/fog/aws/parsers/compute/basic.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class Basic < Fog::Parsers::Base
           def end_element(name)
             case name

data/lib/fog/aws/parsers/compute/cancel_spot_instance_requests.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class CancelSpotInstanceRequests < Fog::Parsers::Base
           def reset
             @spot_instance_request = {}

data/lib/fog/aws/parsers/compute/copy_image.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module Compute
-      module AWS
+    module AWS
+      module Compute
         class CopyImage < Fog::Parsers::Base
           def end_element(name)
             case name