aws-sdk 1.2.3 → 1.2.4

data/lib/aws/s3/bucket.rb

@@ -102,19 +102,28 @@ module AWS
 
       # Deletes the current bucket.
       # 
-      # @note the bucket *must* be empty.
+      # @note This operation will fail if the bucket is not empty.
+      #
       # @return [nil]
+      #
       def delete
         client.delete_bucket(:bucket_name => @name)
         nil
       end
 
-      # Deletes any objects and versions which may be in the bucket,
-      # then deletes the bucket.
+      # Attempts to delete all objects (or object versions) from the bucket
+      # and then attempts to delete the bucket.
+      #
+      # @note This operation may fail if you do not have privileges to delete 
+      #   all objects from the bucket.
+      #
       # @return [nil]
+      #
       def delete!
-        versions.each{|version| version.delete }
-        delete
+        versions.each_batch do |versions|
+          objects.delete(versions)
+        end
+        self.delete
       end
 
       # @return [String] bucket owner id

data/lib/aws/s3/bucket_version_collection.rb

@@ -19,8 +19,6 @@ module AWS
     # @see PrefixedCollection
     class BucketVersionCollection
 
-      include Core::Model
-      include Enumerable
       include PrefixAndDelimiterCollection
 
       # @param [Bucket] bucket
@@ -35,14 +33,18 @@ module AWS
       # @return [ObjectVersion] Returns the most recently created object
       #   version in the entire bucket.
       def latest
-        self.find{|version| true }
+        first
+        #self.find{|version| true }
       end
 
       # Yields once for each version in the bucket.
       #
       # @yield [object_version]
+      #
       # @yieldparam [ObjectVersion] object_version
+      #
       # @return nil
+      #
       def each options = {}, █ super; end
 
       # @private
@@ -50,10 +52,8 @@ module AWS
       def each_member_in_page(page, &block)
         super
         page.versions.each do |version|
-          object_version =
-            ObjectVersion.new(bucket.objects[version.key],
-                              version.version_id,
-                              :delete_marker => version.delete_marker?)
+          object_version = ObjectVersion.new(bucket.objects[version.key],
+            version.version_id, :delete_marker => version.delete_marker?)
           yield(object_version)
         end
       end
@@ -72,10 +72,6 @@ module AWS
       protected
       def pagination_markers; super + [:version_id_marker]; end
 
-      # @private
-      protected
-      def page_size(resp); super + resp.versions.size; end
-
     end
   end
 end

data/lib/aws/s3/client.rb

@@ -15,6 +15,7 @@ require 'rexml/document'
 require 'pathname'
 require 'stringio'
 require 'json'
+require 'digest/md5'
 
 module AWS
   class S3
@@ -561,17 +562,19 @@ module AWS
       #   marker.
       object_method(:get_object, :get,
                     :header_options => {
-                      :range => "Range",
                       :if_modified_since => "If-Modified-Since",
                       :if_unmodified_since => "If-Unmodified-Since",
                       :if_match => "If-Match",
                       :if_none_match => "If-None-Match"
                     }) do
         configure_request do |req, options|
+
           super(req, options)
+
           if options[:version_id]
             req.add_param('versionId', options[:version_id])
           end
+
           ["If-Modified-Since",
            "If-Unmodified-Since"].each do |date_header|
             case value = req.headers[date_header]
@@ -581,6 +584,13 @@ module AWS
               req.headers[date_header] = value.rfc2822
             end
           end
+
+          if options[:range]
+            range = options[:range]
+            range = "bytes=#{range.first}-#{range.last}" if range.is_a?(Range)
+            req.headers['Range'] = range
+          end
+
         end
 
         process_response do |resp|
@@ -705,6 +715,31 @@ module AWS
         end
       end
 
+      bucket_method(:delete_objects, :post, 'delete', XML::DeleteObjects) do
+        configure_request do |req, options|
+
+          super(req, options)
+
+          quiet = options.key?(:quiet) ? options[:quiet] : true
+
+          objects = options[:objects].inject('') do |xml,o|
+            xml << "<Object><Key>#{o[:key]}</Key>"
+            xml << "<VersionId>#{o[:version_id]}</VersionId>" if o[:version_id]
+            xml << "</Object>"
+          end
+
+          xml = '<?xml version="1.0" encoding="UTF-8"?>'
+          xml << "<Delete><Quiet>#{quiet}</Quiet>#{objects}</Delete>"
+
+          req.body = xml
+
+          md5 = Base64.encode64(Digest::MD5.digest(xml)).strip
+
+          req.headers['content-md5'] = md5
+
+        end
+      end
+
       object_method(:upload_part, :put,
                     :header_options => {
                       :content_md5 => 'Content-MD5'
@@ -799,6 +834,7 @@ module AWS
       object_method(:copy_object, :put,
                     :header_options => {
                       :copy_source => 'x-amz-copy-source',
+                      :cache_control => 'Cache-Control',
                       :metadata_directive => 'x-amz-metadata-directive',
                       :storage_class => 'x-amz-storage-class',
                       :server_side_encryption => 'x-amz-server-side-encryption',

data/lib/aws/s3/client/xml.rb

@@ -159,10 +159,17 @@ module AWS
             element("StorageClass") { symbol_value }
             element("Initiated") { datetime_value }
           end
-
           include HasCommonPrefixes
         end
 
+        DeleteObjects = Core::XmlGrammar.customize do
+          element("Deleted") do
+            element("DeleteMarker") { boolean_value }
+            list
+          end
+          element("Error") { list; rename('errors') }  
+        end
+
         # keep default behavior
         CompleteMultipartUpload = Core::XmlGrammar.customize
 

data/lib/aws/s3/errors.rb

@@ -31,6 +31,20 @@ module AWS
     # @private
     module Errors
 
+      class BatchDeleteError < StandardError
+
+        def initialize error_counts
+          @error_counts = error_counts
+          total = error_counts.values.inject(0) {|sum,count| sum + count }
+          super("Failed to delete #{total} objects")
+        end
+
+        # @return [Hash] Returns a hash of error codes and how many
+        #   objects failed with that code.
+        attr_reader :error_counts
+
+      end
+
       # This error is special, because S3 does not (and must not
       # according to RFC 2616) return a body with the HTTP response.
       # The interface is the same as for any other client error.

data/lib/aws/s3/multipart_upload_collection.rb

@@ -64,9 +64,6 @@ module AWS
       protected
       def pagination_markers; super + [:upload_id_marker]; end
 
-      protected
-      def page_size(resp); super + resp.uploads.size; end
-
     end
 
   end

data/lib/aws/s3/object_collection.rb

@@ -102,6 +102,153 @@ module AWS
         super(prefix, mode)
       end
 
+      # Deletes the objects provided in as few requests as possible.
+      #
+      #   # delete 2 objects (by key) in a single request
+      #   bucket.objects.delete('abc', 'xyz')
+      #
+      # You can delete objects also by passing their S3Object representation:
+      #
+      #   to_delete = []
+      #   to_delete << buckets.objects['foo']
+      #   to_delete << buckets.objects['bar']
+      #
+      #   bucket.objects.delete(to_delete)
+      #
+      # @param [Mixed] objects One or more objects to delete.  Each object
+      #   can be one of the following:
+      #
+      #   * An object key (string)
+      #   * A hash with :key and :version_id (for versioned objects)
+      #   * An {S3Object} instance
+      #   * An {ObjectVersion} instance
+      #
+      # @raise [BatchDeleteError] If any of the objects failed to delete, 
+      #   a BatchDeleteError will be raised with a summary of the errors.
+      #
+      # @return [nil]
+      #
+      def delete *objects
+
+        objects = objects.flatten.collect do |obj|
+          case obj
+          when String        then { :key => obj }
+          when Hash          then obj
+          when S3Object      then { :key => obj.key }
+          when ObjectVersion then { :key => obj.key, :version_id => obj.version_id }
+          else  
+            msg = "objects must be keys (strings or hashes with :key and " +
+                  ":version_id), S3Objects or ObjectVersions, got " +
+                  object.class.name
+            raise ArgumentError, msg
+          end
+        end
+
+        batch_helper = BatchHelper.new(1000) do |batch|
+          client_opts = {}
+          client_opts[:bucket_name] = bucket.name
+          client_opts[:quiet] = true
+          client_opts[:objects] = batch
+          client.delete_objects(client_opts)
+        end
+
+        error_counts = {}
+        batch_helper.after_batch do |response|
+          response.errors.each do |error|
+            error_counts[error.code] ||= 0
+            error_counts[error.code] += 1
+          end
+        end
+
+        objects.each do |object|
+          batch_helper.add(object)
+        end
+
+        batch_helper.complete!
+
+        raise Errors::BatchDeleteError.new(error_counts) unless 
+          error_counts.empty?
+
+        nil
+
+      end
+
+      # Deletes each object in the collection that returns a true value 
+      # from block passed to this method.  Deletes are batched for efficiency.
+      #
+      #   # delete text files in the 2009 "folder"
+      #   bucket.objects.with_prefix('2009/').delete_if {|o| o.key =~ /\.txt$/ }
+      #
+      # @yieldparam [S3Object] object
+      #
+      # @raise [BatchDeleteError] If any of the objects failed to delete, 
+      #   a BatchDeleteError will be raised with a summary of the errors.
+      #
+      def delete_if &block
+
+        error_counts = {}
+
+        batch_helper = BatchHelper.new(1000) do |objects| 
+          begin
+            delete(objects)
+          rescue Errors::BatchDeleteError => error
+            error.error_counts.each_pair do |code,count|
+              error_counts[code] ||= 0
+              error_counts[code] += count
+            end
+          end
+        end
+
+        each do |object|
+          batch_helper.add(object) if yield(object)
+        end
+
+        batch_helper.complete!
+
+        raise Errors::BatchDeleteError.new(error_counts) unless 
+          error_counts.empty?
+
+        nil
+
+      end
+
+      # Deletes all objects represented by this collection.
+      #
+      # @example Delete all objects from a bucket
+      #
+      #   bucket.objects.delete_all
+      #
+      # @example Delete objects with a given prefix
+      #
+      #   bucket.objects.with_prefix('2009/').delete_all
+      #
+      # @raise [BatchDeleteError] If any of the objects failed to delete, 
+      #   a BatchDeleteError will be raised with a summary of the errors.
+      #
+      # @return [Array] Returns an array of results
+      #
+      def delete_all
+
+        error_counts = {}
+
+        each_batch do |objects|
+          begin
+            delete(objects)
+          rescue Errors::BatchDeleteError => error
+            error.error_counts.each_pair do |code,count|
+              error_counts[code] ||= 0
+              error_counts[code] += count
+            end
+          end
+        end
+
+        raise Errors::BatchDeleteError.new(error_counts) unless 
+          error_counts.empty?
+
+        nil
+
+      end
+
       # Iterates the collection, yielding instances of S3Object.
       #
       # Use break or raise an exception to terminate the enumeration.
@@ -127,7 +274,7 @@ module AWS
 
       # @private
       protected
-      def list_request(options)
+      def list_request options
         client.list_objects(options)
       end
 
@@ -139,14 +286,43 @@ module AWS
 
       # @private
       protected
-      def page_size resp
-        super + resp.contents.size
+      def next_markers page
+        { :marker => (last = page.contents.last and last.key) }
       end
 
+      # processes items in batches of 1k items
       # @private
-      protected
-      def next_markers page
-        { :marker => (last = page.contents.last and last.key) }
+      class BatchHelper
+        
+        def initialize batch_size, &block
+          @batch_size = batch_size
+          @block = block
+          @batch = []
+        end
+
+        def after_batch &block
+          @after_batch = block
+        end
+
+        def add item
+          @batch << item
+          if @batch.size == @batch_size
+            process_batch
+            @batch = []
+          end
+          item
+        end
+
+        def complete!
+          process_batch unless @batch.empty?
+        end
+
+        protected
+        def process_batch
+          response = @block.call(@batch)
+          @after_batch.call(response) if @after_batch
+        end
+
       end
 
     end

data/lib/aws/s3/paginated_collection.rb

@@ -17,47 +17,33 @@ module AWS
     # @private
     module PaginatedCollection
 
-      def each(options = {}, &block)
-        each_page(options) do |page|
-          each_member_in_page(page, &block)
-        end
-        nil
-      end
+      include Core::Collection::Limitable
 
       protected
-      def each_member_in_page(page, &block); end
+      def _each_item markers, limit, options = {}, &block
+
+        options = list_options(options)
+        options.merge!(markers) unless markers.nil? or markers.empty?
+        options[limit_param] = limit || 1000
+
+        response = list_request(options)
+
+        each_member_in_page(response, &block)
+      
+        response.truncated? ? next_markers(response) : nil
 
-      protected
-      def each_page(options = {}, &block)
-        opts = list_options(options)
-        limit = options[:limit]
-        batch_size = options[:batch_size] || 1000
-        markers = {}
-        received = 0
-
-        loop do
-          page_opts = opts.dup
-          page_opts.merge!(markers)
-          page_opts[limit_param] =
-            limit ? [limit - received, batch_size].min : batch_size
-
-          page = list_request(page_opts)
-          markers = next_markers(page)
-          received += page_size(page)
-
-          yield(page)
-
-          return unless page.truncated?
-        end
       end
 
+      protected
+      def each_member_in_page(page, &block); end
+
       protected
       def list_request(options)
         raise NotImplementedError
       end
 
       protected
-      def list_options(options)
+      def list_options options
         opts = {}
         opts[:bucket_name] = bucket.name if respond_to?(:bucket)
         opts
@@ -74,20 +60,15 @@ module AWS
       end
 
       protected
-      def next_markers(page)
-        pagination_markers.inject({}) do |markers, marker|
-          markers[marker] = page.send("next_#{marker}")
+      def next_markers page
+        pagination_markers.inject({}) do |markers, marker_name|
+          if marker = page.send("next_#{marker_name}")
+            markers[marker_name] = marker if marker
+          end
           markers
         end
       end
 
-      protected
-      def page_size(resp)
-        (resp.respond_to?(:common_prefixes) and
-         prefixes = resp.common_prefixes and
-         prefixes.size) or 0
-      end
-
     end
 
   end