fog-aws 3.0.0 → 3.10.0

data/lib/fog/aws/models/storage/directory.rb

@@ -2,8 +2,8 @@ require 'fog/aws/models/storage/files'
 require 'fog/aws/models/storage/versions'
 
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class Directory < Fog::Model
         VALID_ACLS = ['private', 'public-read', 'public-read-write', 'authenticated-read']
 
@@ -12,7 +12,6 @@ module Fog
         identity  :key,           :aliases => ['Name', 'name']
 
         attribute :creation_date, :aliases => 'CreationDate', :type => 'time'
-        attribute :location,      :aliases => 'LocationConstraint', :type => 'string'
 
         def acl=(new_acl)
           unless VALID_ACLS.include?(new_acl)
@@ -30,8 +29,32 @@ module Fog
           false
         end
 
+        # @param options [Hash] (defaults to: {}) — a customizable set of options.
+        #   Consider tuning this values for big buckets.
+        # @option options timeout [Integer] — default: Fog.timeout — Maximum number of
+        #   seconds to wait for the bucket to be empty.
+        # @option options interval [Proc|Integer] — default: Fog.interval — Seconds to wait before
+        #   retrying to check if the bucket is empty.
+        def destroy!(options = {})
+          requires :key
+          options = {
+            timeout: Fog.timeout,
+            interval: Fog.interval,
+          }.merge(options)
+
+          attempts = 0
+          begin
+            clear!
+            Fog.wait_for(options[:timeout], options[:interval]) { objects_keys.size == 0 }
+            service.delete_bucket(key)
+            true
+          rescue Excon::Errors::HTTPStatusError
+            false
+          end
+        end
+
         def location
-          @location ||= (bucket_location || AWS::DEFAULT_REGION)
+          @location ||= (bucket_location || Storage::DEFAULT_REGION)
         end
 
         # NOTE: you can't change the region once the bucket is created
@@ -40,7 +63,7 @@ module Fog
         end
 
         def files
-          @files ||= Fog::Storage::AWS::Files.new(:directory => self, :service => service)
+          @files ||= Fog::AWS::Storage::Files.new(:directory => self, :service => service)
         end
 
         def payer
@@ -67,7 +90,7 @@ module Fog
         end
 
         def versions
-          @versions ||= Fog::Storage::AWS::Versions.new(:directory => self, :service => service)
+          @versions ||= Fog::AWS::Storage::Versions.new(:directory => self, :service => service)
         end
 
         def public=(new_public)
@@ -119,6 +142,17 @@ module Fog
           data = service.get_bucket_location(key)
           data.body['LocationConstraint']
         end
+
+        def objects_keys
+          requires :key
+          bucket_query = service.get_bucket(key)
+          bucket_query.body["Contents"].map {|c| c["Key"]}
+        end
+
+        def clear!
+          requires :key
+          service.delete_multiple_objects(key, objects_keys) if objects_keys.size > 0
+        end
       end
     end
   end

data/lib/fog/aws/models/storage/file.rb

@@ -1,9 +1,13 @@
 require 'fog/aws/models/storage/versions'
 
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class File < Fog::Model
+        MIN_MULTIPART_CHUNK_SIZE = 5242880
+        MAX_SINGLE_PUT_SIZE = 5368709120
+        MULTIPART_COPY_THRESHOLD = 15728640
+
         # @see AWS Object docs http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
 
         identity  :key,             :aliases => 'Key'
@@ -25,15 +29,56 @@ module Fog
         attribute :encryption_key,      :aliases => 'x-amz-server-side-encryption-customer-key'
         attribute :version,             :aliases => 'x-amz-version-id'
         attribute :kms_key_id,          :aliases => 'x-amz-server-side-encryption-aws-kms-key-id'
+        attribute :tags,                :aliases => 'x-amz-tagging'
+
+        UploadPartData = Struct.new(:part_number, :upload_options, :etag)
+
+        class PartList
+          def initialize(parts = [])
+            @parts = parts
+            @mutex = Mutex.new
+          end
+
+          def push(part)
+            @mutex.synchronize { @parts.push(part) }
+          end
+
+          def shift
+            @mutex.synchronize { @parts.shift }
+          end
+
+          def clear!
+            @mutex.synchronize { @parts.clear }
+          end
+
+          def size
+            @mutex.synchronize { @parts.size }
+          end
+
+          def to_a
+            @mutex.synchronize { @parts.dup }
+          end
+        end
 
         # @note Chunk size to use for multipart uploads.
         #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
         attr_reader :multipart_chunk_size
         def multipart_chunk_size=(mp_chunk_size)
-          raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880
+          raise ArgumentError.new("minimum multipart_chunk_size is #{MIN_MULTIPART_CHUNK_SIZE}") if mp_chunk_size < MIN_MULTIPART_CHUNK_SIZE
           @multipart_chunk_size = mp_chunk_size
         end
 
+        # @note Number of threads used to copy files.
+        def concurrency=(concurrency)
+          raise ArgumentError.new('minimum concurrency is 1') if concurrency.to_i < 1
+
+          @concurrency = concurrency.to_i
+        end
+
+        def concurrency
+          @concurrency || 1
+        end
+
         def acl
           requires :directory, :key
           service.get_object_acl(directory.key, key).body['AccessControlList']
@@ -98,7 +143,18 @@ module Fog
         #
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+
+          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
+          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE * 2 if !multipart_chunk_size && self.content_length.to_i > MAX_SINGLE_PUT_SIZE
+
+          if multipart_chunk_size && self.content_length.to_i >= multipart_chunk_size
+            upload_part_options = options.select { |key, _| ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym) }
+            upload_part_options = upload_part_options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" })
+            multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          else
+            service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          end
+
           target_directory = service.directories.new(:key => target_directory_key)
           target_directory.files.head(target_file_key)
         end
@@ -192,6 +248,7 @@ module Fog
         # @option options [String] expires sets number of seconds before AWS Object expires.
         # @option options [String] storage_class sets x-amz-storage-class HTTP header. Defaults to 'STANDARD'. Or, 'REDUCED_REDUNDANCY'
         # @option options [String] encryption sets HTTP encryption header. Set to 'AES256' to encrypt files at rest on S3
+        # @option options [String] tags sets x-amz-tagging HTTP header. For example, 'Org-Id=1' or 'Org-Id=1&Service=MyService'
         # @return [Boolean] true if no errors
         #
         def save(options = {})
@@ -208,10 +265,11 @@ module Fog
           options['Expires'] = expires if expires
           options.merge!(metadata)
           options['x-amz-storage-class'] = storage_class if storage_class
+          options['x-amz-tagging'] = tags if tags
           options.merge!(encryption_headers)
 
           # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = 5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120
+          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE if !multipart_chunk_size && Fog::Storage.get_body_size(body) > MAX_SINGLE_PUT_SIZE
 
           if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
             data = multipart_save(options)
@@ -240,11 +298,11 @@ module Fog
         end
 
         # File version if exists or creates new version.
-        # @return [Fog::Storage::AWS::Version]
+        # @return [Fog::AWS::Storage::Version]
         #
         def versions
           @versions ||= begin
-            Fog::Storage::AWS::Versions.new(
+            Fog::AWS::Storage::Versions.new(
               :file         => self,
               :service   => service
             )
@@ -291,6 +349,30 @@ module Fog
           service.complete_multipart_upload(directory.key, key, upload_id, part_tags)
         end
 
+        def multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          # Initiate the upload
+          res = service.initiate_multipart_upload(target_directory_key, target_file_key, options)
+          upload_id = res.body["UploadId"]
+
+          # Store ETags of upload parts
+          part_tags = []
+          pending = PartList.new(create_part_list(upload_part_options))
+          thread_count = self.concurrency
+          completed = PartList.new
+          errors = upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+
+          raise errors.first if errors.any?
+
+          part_tags = completed.to_a.sort_by { |part| part.part_number }.map(&:etag)
+        rescue => e
+          # Abort the upload & reraise
+          service.abort_multipart_upload(target_directory_key, target_file_key, upload_id) if upload_id
+          raise
+        else
+          # Complete the upload
+          service.complete_multipart_upload(target_directory_key, target_file_key, upload_id, part_tags)
+        end
+
         def encryption_headers
           if encryption && encryption_key
             encryption_customer_key_headers
@@ -315,6 +397,49 @@ module Fog
             'x-amz-server-side-encryption-customer-key-md5' => Base64.encode64(OpenSSL::Digest::MD5.digest(encryption_key.to_s)).chomp!
           }
         end
+
+        def create_part_list(upload_part_options)
+          current_pos = 0
+          count = 0
+          pending = []
+
+          while current_pos < self.content_length do
+            start_pos = current_pos
+            end_pos = [current_pos + self.multipart_chunk_size, self.content_length - 1].min
+            range = "bytes=#{start_pos}-#{end_pos}"
+            part_options = upload_part_options.dup
+            part_options['x-amz-copy-source-range'] = range
+            pending << UploadPartData.new(count + 1, part_options, nil)
+            count += 1
+            current_pos = end_pos + 1
+          end
+
+          pending
+        end
+
+        def upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+          threads = []
+
+          thread_count.times do
+            thread = Thread.new do
+              begin
+                while part = pending.shift
+                  part_upload = service.upload_part_copy(target_directory_key, target_file_key, upload_id, part.part_number, part.upload_options)
+                  part.etag = part_upload.body['ETag']
+                  completed.push(part)
+                end
+              rescue => error
+                pending.clear!
+                error
+              end
+            end
+
+            thread.abort_on_exception = true
+            threads << thread
+          end
+
+          threads.map(&:value).compact
+        end
       end
     end
   end

data/lib/fog/aws/models/storage/files.rb

@@ -1,8 +1,8 @@
 require 'fog/aws/models/storage/file'
 
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class Files < Fog::Collection
         extend Fog::Deprecation
         deprecate :get_url, :get_https_url
@@ -15,7 +15,16 @@ module Fog
         attribute :max_keys,        :aliases => ['MaxKeys', 'max-keys']
         attribute :prefix,          :aliases => 'Prefix'
 
-        model Fog::Storage::AWS::File
+        model Fog::AWS::Storage::File
+
+        DASHED_HEADERS = %w(
+          Cache-Control
+          Content-Disposition
+          Content-Encoding
+          Content-Length
+          Content-MD5
+          Content-Type
+        ).freeze
 
         def all(options = {})
           requires :directory
@@ -70,7 +79,7 @@ module Fog
           when /<Code>NoSuchKey<\/Code>/
             nil
           when /<Code>NoSuchBucket<\/Code>/
-            raise(Fog::Storage::AWS::NotFound.new("Directory #{directory.identity} does not exist."))
+            raise(Fog::AWS::Storage::NotFound.new("Directory #{directory.identity} does not exist."))
           else
             raise(error)
           end
@@ -114,8 +123,29 @@ module Fog
         end
 
         def normalize_headers(data)
-          data.headers['Last-Modified'] = Time.parse(data.get_header('Last-Modified'))
-          data.headers['ETag'] = data.get_header('ETag').gsub('"','')
+          data.headers['Last-Modified'] = Time.parse(fetch_and_delete_header(data, 'Last-Modified'))
+
+          etag = fetch_and_delete_header(data, 'ETag').gsub('"','')
+          data.headers['ETag'] = etag
+
+          DASHED_HEADERS.each do |header|
+            value = fetch_and_delete_header(data, header)
+            data.headers[header] = value if value
+          end
+        end
+
+        private
+
+        def fetch_and_delete_header(response, header)
+          value = response.get_header(header)
+
+          return unless value
+
+          response.headers.keys.each do |key|
+            response.headers.delete(key) if key.downcase == header.downcase
+          end
+
+          value
         end
       end
     end

data/lib/fog/aws/models/storage/version.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class Version < Fog::Model
         identity  :version,             :aliases => 'VersionId'
 

data/lib/fog/aws/models/storage/versions.rb

@@ -1,13 +1,13 @@
 require 'fog/aws/models/storage/version'
 
 module Fog
-  module Storage
-    class AWS
+  module AWS
+    class Storage
       class Versions < Fog::Collection
         attribute :file
         attribute :directory
 
-        model Fog::Storage::AWS::Version
+        model Fog::AWS::Storage::Version
 
         def all(options = {})
           data = if file

data/lib/fog/aws/parsers/cdn/distribution.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class Distribution < Fog::Parsers::Base
           def reset
             @response = { 'DistributionConfig' => { 'CNAME' => [], 'Logging' => {}, 'TrustedSigners' => [] } }

data/lib/fog/aws/parsers/cdn/get_distribution_list.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class GetDistributionList < Fog::Parsers::Base
           def reset
             @distribution_summary = { 'CNAME' => [], 'TrustedSigners' => [] }

data/lib/fog/aws/parsers/cdn/get_invalidation.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class GetInvalidation < Fog::Parsers::Base
           def reset
             @response = { 'InvalidationBatch' => { 'Path' => [] } }

data/lib/fog/aws/parsers/cdn/get_invalidation_list.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class GetInvalidationList < Fog::Parsers::Base
           def reset
             @invalidation_summary = { }

data/lib/fog/aws/parsers/cdn/get_streaming_distribution_list.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class GetStreamingDistributionList < Fog::Parsers::Base
           def reset
             @distribution_summary = { 'CNAME' => [], 'TrustedSigners' => [] }

data/lib/fog/aws/parsers/cdn/post_invalidation.rb

@@ -1,7 +1,7 @@
 module Fog
   module Parsers
-    module CDN
-      module AWS
+    module AWS
+      module CDN
         class PostInvalidation < Fog::Parsers::Base
           def reset
             @response = { 'InvalidationBatch' => { 'Path' => [] } }