right_aws 3.0.4 → 3.0.5

data/History.txt

@@ -346,3 +346,13 @@ the source key.
   - Fixed:
     - #125 - fixes redirect bug in file PUT requests (Cary)
     - some other minor fixes
+
+=== 3.0.5
+  Release Notes:
+  - Added: API '2012-06-15' support for CreateVolume and DescribeVolumes API calls (to support IOPS)
+  - Fixed:
+    - Single-threaded multipart upload support (https://github.com/rightscale/right_aws/pull/116)
+    - S3 multi object delete (https://github.com/rightscale/right_aws/pull/106)
+    - Support for "ami_version" added in emr interface (https://github.com/rightscale/right_aws/pull/129)
+    - S3: Added block references to several methods (https://github.com/rightscale/right_aws/pull/130)
+    - Some other minor changes

data/lib/awsbase/right_awsbase.rb

@@ -43,7 +43,12 @@ module RightAws
       Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
     end
 
-    # Escape a string accordingly Amazon rules
+    # Calculates 'Content-MD5' header value for some content
+    def self.content_md5(content)
+      Base64.encode64(Digest::MD5::new.update(content).digest).strip
+    end
+
+    # Escape a string accordingly Amazon rulles
     # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
     def self.amz_escape(param)
       param = param.flatten.join('') if param.is_a?(Array) # ruby 1.9.x Array#to_s fix
@@ -443,6 +448,7 @@ module RightAws
                       "AWSAccessKeyId" => @aws_access_key_id,
                       "Version"        => custom_options[:api_version] || @params[:api_version] }
       service_hash.merge!(options)
+      service_hash["SecurityToken"] = @params[:token] if @params[:token]
       # Sign request options
       service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:host_to_sign], @params[:service])
       # Use POST if the length of the query string is too large

data/lib/awsbase/version.rb

@@ -2,7 +2,7 @@ module RightAws #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 3  unless defined?(MAJOR)
     MINOR = 0  unless defined?(MINOR)
-    TINY  = 4  unless defined?(TINY)
+    TINY  = 5  unless defined?(TINY)
 
     STRING = [MAJOR, MINOR, TINY].join('.') unless defined?(STRING)
   end

data/lib/ec2/right_ec2.rb

@@ -91,9 +91,12 @@ module RightAws
                                  'm2.xlarge' ,
                                  'm2.2xlarge',
                                  'm2.4xlarge',
+                                 'm3.xlarge' ,
+                                 'm3.2xlarge',
                                 'cc1.4xlarge',
                                 'cg1.4xlarge',
-                                'cc2.8xlarge']
+                                'cc2.8xlarge',
+                                'hi1.4xlarge' ]
     
     @@bench = AwsBenchmarkingBlock.new
     def self.bench_xml
@@ -120,6 +123,7 @@ module RightAws
     # * <tt>:logger</tt>: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT
     # * <tt>:signature_version</tt>:  The signature version : '0','1' or '2'(default)
     # * <tt>:cache</tt>: true/false: caching for: ec2_describe_images, describe_instances,
+    # * <tt>:token</tt>: Option SecurityToken for temporary credentials
     # describe_images_by_owner, describe_images_by_executable_by, describe_availability_zones,
     # describe_security_groups, describe_key_pairs, describe_addresses, 
     # describe_volumes, describe_snapshots methods, default: false.
@@ -154,12 +158,15 @@ module RightAws
       # 'RemoteFunctionName' -> :remote_funtion_name
       cache_name = remote_function_name.right_underscore.to_sym
       list, options = AwsUtils::split_items_and_params(list_and_options)
+      custom_options = {}
       # Resource IDs to fetch
       request_hash  = amazonize_list(remote_item_name, list)
       # Other custom options
       options.each do |key, values|
         next if values.right_blank?
         case key
+        when :options
+          custom_options = values
         when :filters then
           request_hash.merge!(amazonize_list(['Filter.?.Name', 'Filter.?.Value.?'], values))
         else
@@ -167,12 +174,17 @@ module RightAws
         end
       end
       cache_for = (list.right_blank? && options.right_blank?) ? cache_name : nil
-      link = generate_request(remote_function_name, request_hash)
+      link = generate_request(remote_function_name, request_hash, custom_options)
       request_cache_or_info(cache_for, link,  parser_class, @@bench, cache_for, &block)
     rescue Exception
       on_exception
     end
 
+    def merge_new_options_into_list_and_options(list_and_options, new_options)
+      list, options = AwsUtils::split_items_and_params(list_and_options)
+      list << options.merge(new_options)
+    end
+
   #-----------------------------------------------------------------
   #      Keys
   #-----------------------------------------------------------------

data/lib/ec2/right_ec2_ebs.rb

@@ -29,6 +29,9 @@ module RightAws
     #      EBS: Volumes
     #-----------------------------------------------------------------
 
+    VOLUME_API_VERSION = (API_VERSION > '2012-06-15') ? API_VERSION : '2012-06-15'
+    VOLUME_TYPES       = ['standard', 'io1']
+
     # Describe EBS volumes.
     #
     # Accepts a list of volumes and/or a set of filters as the last parameter.
@@ -47,6 +50,14 @@ module RightAws
     #        :aws_id                => "vol-60957009",
     #        :aws_created_at        => "2008-06-18T08:19:20.000Z",
     #        :aws_instance_id       => "i-c014c0a9"},
+    #       {:aws_id         => "vol-71de8b1f",
+    #        :aws_size       => 5,
+    #        :snapshot_id    => nil,
+    #        :zone           => "us-east-1a",
+    #        :aws_status     => "available",
+    #        :aws_created_at => "2012-06-21T18:47:34.000Z",
+    #        :volume_type    => "io1",
+    #        :iop            => "5"},#
     #       {:aws_size       => 1,
     #        :zone           => "merlot",
     #        :snapshot_id    => nil,
@@ -59,6 +70,7 @@ module RightAws
     #  P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeVolumes.html
     #
     def describe_volumes(*list_and_options)
+      list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => VOLUME_API_VERSION})
       describe_resources_with_list_and_options('DescribeVolumes', 'VolumeId', QEc2DescribeVolumesParser, list_and_options)
     end
 
@@ -73,12 +85,29 @@ module RightAws
     #       :aws_created_at => "2008-06-24T18:13:32.000Z",
     #       :aws_size       => 94}
     #
-    def create_volume(snapshot_id, size, zone)
+    #  ec2.create_volume(nil, 5, 'us-east-1a', :iops => '5', :volume_type => 'io1') #=>
+    #      {:aws_id=>"vol-71de8b1f",
+    #       :aws_size=>5,
+    #       :snapshot_id=>nil,
+    #       :zone=>"us-east-1a",
+    #       :aws_status=>"creating",
+    #       :aws_created_at=>"2012-06-21T18:47:34.000Z",
+    #       :volume_type=>"io1",
+    #       :iops=>"5"}
+    #
+    def create_volume(snapshot_id, size, zone, options={})
       hash = { "Size"              => size.to_s,
                "AvailabilityZone"  => zone.to_s }
       # Get rig of empty snapshot: e8s guys do not like it
       hash["SnapshotId"] = snapshot_id.to_s unless snapshot_id.right_blank?
-      link = generate_request("CreateVolume", hash )
+      # Add IOPS support (default behavior) but skip it when an old API version call is requested
+      options[:options]                 ||= {}
+      options[:options][:api_version]   ||= VOLUME_API_VERSION
+      if options[:options][:api_version] >= VOLUME_API_VERSION
+        hash["VolumeType"] = options[:volume_type] unless options[:volume_type].right_blank?
+        hash["Iops"]       = options[:iops]        unless options[:iops].right_blank?
+      end
+      link = generate_request("CreateVolume", hash, options[:options])
       request_info(link, QEc2CreateVolumeParser.new(:logger => @logger))
     rescue Exception
       on_exception
@@ -364,6 +393,8 @@ module RightAws
         when 'size'             then @result[:aws_size]       = @text.to_i ###
         when 'snapshotId'       then @result[:snapshot_id]    = @text.right_blank? ? nil : @text ###
         when 'availabilityZone' then @result[:zone]           = @text ###
+        when 'volumeType'       then @result[:volume_type]    = @text
+        when 'iops'             then @result[:iops]           = @text
         end
       end
       def reset
@@ -403,6 +434,8 @@ module RightAws
         when 'snapshotId'       then @item[:snapshot_id]     = @text.right_blank? ? nil : @text
         when 'availabilityZone' then @item[:zone]            = @text
         when 'deleteOnTermination' then @item[:delete_on_termination] = (@text == 'true')
+        when 'volumeType'       then @item[:volume_type]     = @text
+        when 'iops'             then @item[:iops]            = @text
         else
           case full_tag_name
           when %r{/volumeSet/item/volumeId$}   then @item[:aws_id]                = @text

data/lib/ec2/right_ec2_instances.rb

@@ -25,6 +25,8 @@ module RightAws
 
   class Ec2
 
+    INSTANCE_API_VERSION = (API_VERSION > '2012-07-20') ? API_VERSION : '2012-07-20'
+
   #-----------------------------------------------------------------
   #      Instances
   #-----------------------------------------------------------------
@@ -108,6 +110,7 @@ module RightAws
     # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeInstances.html
     #
     def describe_instances(*list_and_options)
+      list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => INSTANCE_API_VERSION})
       describe_resources_with_list_and_options('DescribeInstances', 'InstanceId', QEc2DescribeInstancesParser, list_and_options) do |parser|
         get_desc_instances(parser.result)
       end
@@ -158,7 +161,6 @@ module RightAws
                                    :group_names                          => group_names,
                                    :key_name                             => key_name,
                                    :instance_type                        => instance_type,
-                                   :addressing_type                      => addressing_type,
                                    :kernel_id                            => kernel_id,
                                    :ramdisk_id                           => ramdisk_id,
                                    :availability_zone                    => availability_zone,
@@ -174,9 +176,10 @@ module RightAws
 
     # Launch new EC2 instances.
     # 
-    # Options: :image_id, :addressing_type, :min_count, max_count, :key_name, :kernel_id, :ramdisk_id,
+    # Options: :image_id, :min_count, max_count, :key_name, :kernel_id, :ramdisk_id,
     # :availability_zone, :monitoring_enabled, :subnet_id, :disable_api_termination, :instance_initiated_shutdown_behavior,
-    # :block_device_mappings, :placement_group_name, :license_pool, :group_ids, :group_names, :private_ip_address
+    # :block_device_mappings, :placement_group_name, :license_pool, :group_ids, :group_names, :private_ip_address,
+    # :ebs_optimized
     # 
     # Returns a list of launched instances or an exception.
     #
@@ -223,9 +226,9 @@ module RightAws
     def launch_instances(image_id, options={})
       options[:user_data] = options[:user_data].to_s
       params = map_api_keys_and_values( options,
-        :key_name, :addressing_type, :kernel_id,
+        :key_name, :kernel_id,
         :ramdisk_id, :subnet_id, :instance_initiated_shutdown_behavior,
-        :private_ip_address, :additional_info, :license_pool,
+        :private_ip_address, :additional_info, :license_pool, :ebs_optimized,
         :image_id                => { :value => image_id },
         :min_count               => { :value => options[:min_count] || 1 },
         :max_count               => { :value => options[:max_count] || options[:min_count] || 1 },
@@ -243,7 +246,12 @@ module RightAws
                                       :value => Proc.new{ options[:monitoring_enabled] && options[:monitoring_enabled].to_s }})
       # Log debug information
       @logger.info("Launching instance of image #{image_id}. Options: #{params.inspect}")
-      link = generate_request("RunInstances", params)
+      # Add IOPS support (default behavior) but skip it when an old API version call is requested
+      options[:options]                 ||= {}
+      options[:options][:api_version]   ||= INSTANCE_API_VERSION
+      params.delete("EbsOptimized") if options[:options][:api_version] < INSTANCE_API_VERSION
+      #
+      link = generate_request("RunInstances", params, options[:options])
       instances = request_info(link, QEc2DescribeInstancesParser.new(:logger => @logger))
       get_desc_instances(instances)
     rescue Exception
@@ -383,7 +391,7 @@ module RightAws
       when 'GroupId'  then request_hash.merge!(amazonize_list('GroupId', value))
       else                 request_hash["#{attribute}.Value"] = value
       end
-      link = generate_request('ModifyInstanceAttribute', request_hash)
+      link = generate_request('ModifyInstanceAttribute', request_hash, :api_version => INSTANCE_API_VERSION)
       request_info(link, RightBoolResponseParser.new(:logger => @logger))
     rescue Exception
       on_exception
@@ -588,9 +596,10 @@ module RightAws
         when 'requesterId'           then @item[:requester_id]             = @text
         when 'virtualizationType'    then @item[:virtualization_type]      = @text
         when 'clientToken'           then @item[:client_token]      = @text
-        when 'sourceDestCheck'       then @item[:source_dest_check] = @text == 'true' ? true : false
+        when 'sourceDestCheck'       then @item[:source_dest_check] = @text == 'true'
         when 'tenancy'               then @item[:placement_tenancy] = @text
         when 'hypervisor'            then @item[:hypervisor]        = @text
+        when 'ebsOptimized'          then @item[:ebs_optimized]     = @text == 'true'
         else
           case full_tag_name
           # EC2 Groups
@@ -618,7 +627,7 @@ module RightAws
             when 'volumeId'            then @block_device_mapping[:ebs_volume_id]              = @text
             when 'status'              then @block_device_mapping[:ebs_status]                 = @text
             when 'attachTime'          then @block_device_mapping[:ebs_attach_time]            = @text
-            when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination]  = @text == 'true' ? true : false
+            when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination]  = @text == 'true'
             when 'item'                then @item[:block_device_mappings]                     << @block_device_mapping
             end
           when %r{/instancesSet/item$} then @reservation[:instances_set] << @item
@@ -675,9 +684,9 @@ module RightAws
         when %r{/ramdisk/value$}        then @result = @text
         when %r{/userData/value$}       then @result = @text
         when %r{/rootDeviceName/value$} then @result = @text
-        when %r{/disableApiTermination/value}              then @result = @text == 'true' ? true : false
+        when %r{/disableApiTermination/value}              then @result = @text == 'true'
         when %r{/instanceInitiatedShutdownBehavior/value$} then @result = @text
-        when %r{/sourceDestCheck/value$}                   then @result = @text == 'true' ? true : false
+        when %r{/sourceDestCheck/value$}                   then @result = @text == 'true'
         when %r{/groupSet/item} # no trailing $
           case name
           when 'groupId'   then @group[:group_id]   = @text
@@ -692,7 +701,7 @@ module RightAws
           when 'volumeId'            then @block_device_mapping[:ebs_volume_id]              = @text
           when 'status'              then @block_device_mapping[:ebs_status]                 = @text
           when 'attachTime'          then @block_device_mapping[:ebs_attach_time]            = @text
-          when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination]  = @text == 'true' ? true : false
+          when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination]  = @text == 'true'
           when 'item'                then @result                                           << @block_device_mapping
           end
         end

data/lib/emr/right_emr_interface.rb

@@ -124,6 +124,7 @@ module RightAws
       :additional_info => 'AdditionalInfo',
       :log_uri => 'LogUri',
       :name => 'Name',
+      :ami_version => 'AmiVersion',
       # JobFlowInstancesConfig
       :ec2_key_name          => 'Instances.Ec2KeyName',
       :hadoop_version => 'Instances.HadoopVersion',

data/lib/s3/right_s3.rb

@@ -259,11 +259,12 @@ module RightAws
         #  key = RightAws::S3::Key.create(bucket, 'logs/today/1.log')
         #  key.head
         #
-      def key(key_name, head=false)
+      def key(key_name, head=false, &blck)
         raise 'Key name can not be empty.' if key_name.right_blank?
         key_instance = nil
           # if this key exists - find it ....
         keys({'prefix'=>key_name}, head).each do |key|
+          blck.call if block_given?
           if key.name == key_name.to_s
             key_instance = key
             break
@@ -282,9 +283,9 @@ module RightAws
         #
         #  bucket.put('logs/today/1.log', 'Olala!') #=> true
         #
-      def put(key, data=nil, meta_headers={}, perms=nil, headers={})
+      def put(key, data=nil, meta_headers={}, perms=nil, headers={}, &blck)
         key = Key.create(self, key.to_s, data, meta_headers) unless key.is_a?(Key) 
-        key.put(data, perms, headers)
+        key.put(data, perms, headers, &blck)
       end
 
         # Retrieve data object from Amazon. 
@@ -518,11 +519,33 @@ module RightAws
         #   ...
         #  key.put('Olala!')   #=> true
         #
-      def put(data=nil, perms=nil, headers={})
+      def put(data=nil, perms=nil, headers={}, &blck)
         headers['x-amz-acl'] = perms if perms
         @data = data || @data
         meta  = self.class.add_meta_prefix(@meta_headers)
-        @bucket.s3.interface.put(@bucket.name, @name, @data, meta.merge(headers))
+        @bucket.s3.interface.put(@bucket.name, @name, @data, meta.merge(headers), &blck)
+      end
+
+        # Store object data on S3 using the Multipart Upload API. This is useful if you do not know the file size
+        # upfront (for example reading from pipe or socket) or if you are transmitting data over an unreliable network.
+        #
+        # Parameter +data+ is an object which responds to :read or an object which can be converted to a String prior to upload.
+        # Parameter +part_size+ determines the size of each part sent (must be > 5MB per Amazon's API requirements)
+        #
+        # If data is a stream the caller is responsible for calling close() on the stream after this methods returns
+        #
+        # Returns +true+.
+        #
+        #  upload_data = StringIO.new('My sample data')
+        #  key = RightAws::S3::Key.create(bucket, 'logs/today/1.log')
+        #  key.data = upload_data
+        #  key.put_multipart(:part_size => 5*1024*1024)             #=> true
+        #
+      def put_multipart(data=nil, perms=nil, headers={}, part_size=nil)
+        headers['x-amz-acl'] = perms if perms
+        @data = data || @data
+        meta  = self.class.add_meta_prefix(@meta_headers)
+        @bucket.s3.interface.store_object_multipart({:bucket => @bucket.name, :key => @name, :data => @data, :headers => meta.merge(headers), :part_size => part_size})
       end
       
         # Rename an object. Returns new object name.

data/lib/s3/right_s3_interface.rb

@@ -26,6 +26,8 @@ module RightAws
   class S3Interface < RightAwsBase
     
     USE_100_CONTINUE_PUT_SIZE = 1_000_000
+    MINIMUM_PART_SIZE         = 5 * 1024 * 1024
+    DEFAULT_RETRY_COUNT       = 5
     
     include RightAwsBaseInterface
     
@@ -41,15 +43,20 @@ module RightAws
     S3_REQUEST_PARAMETERS = [ 'acl',
         'location',
         'logging', # this one is beta, no support for now
+        'partNumber',
         'response-content-type',
         'response-content-language',
         'response-expires',
         'response-cache-control',
         'response-content-disposition',
         'response-content-encoding',
-        'torrent' ].sort
-
+        'torrent',
+        'uploadId',
+        'uploads',
+        'delete'].sort
+    MULTI_OBJECT_DELETE_MAX_KEYS = 1000
 
+    
     @@bench = AwsBenchmarkingBlock.new
     def self.bench_xml
       @@bench.xml
@@ -428,7 +435,7 @@ module RightAws
       # mode.
       #
       
-    def put(bucket, key, data=nil, headers={})
+    def put(bucket, key, data=nil, headers={}, &blck)
       # On Windows, if someone opens a file in text mode, we must reset it so
       # to binary mode for streaming to work properly
       if(data.respond_to?(:binmode))
@@ -439,7 +446,7 @@ module RightAws
         headers['expect'] = '100-continue'
       end
       req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}", :data=>data))
-      request_info(req_hash, RightHttp2xxParser.new)
+      request_info(req_hash, RightHttp2xxParser.new, &blck)
     rescue
       on_exception
     end
@@ -524,6 +531,134 @@ module RightAws
       r[:verified_md5] ? (return r) : (raise AwsError.new("Uploaded object failed MD5 checksum verification: #{r.inspect}"))
     end
     
+    # New experimental API for uploading objects using the multipart upload API.
+    # store_object_multipart is similar in function to the store_object method, but breaks the input into parts and transmits each
+    # part separately.  The multipart upload API has the benefit of being be able to retransmit a part in isolation without needing to
+    # restart the entire upload.  This makes it ideal for uploading large files over unreliable networks.  It also does not
+    # require the file size to be known before starting the upload, making it useful for stream data as it is created (say via reading a pipe or socket).
+    # The hash of the response headers contains useful information like the location (the URI for the newly created object), bucket, key, and etag).
+    #
+    # The optional argument of :headers allows the caller to specify arbitrary request header values.
+    #
+    # s3.store_object_multipart(:bucket => "foobucket", :key => "foo", :data => "polemonium" )
+    #   => {:location=>"https://s3.amazonaws.com/right_s3_awesome_test_bucket_000B1_officedrop/test%2Flarge_multipart_file",
+    #       :e_tag=>"\"72b81ac08aed4d4d1055c11f56c2a258-1\"",
+    #       :key=>"test/large_multipart_file",
+    #       :bucket=>"right_s3_awesome_test_bucket_000B1_officedrop"}
+    #
+    # f = File.new("some_file", "r")
+    # s3.store_object_multipart(:bucket => "foobucket", :key => "foo", :data => f )
+    #   => {:location=>"https://s3.amazonaws.com/right_s3_awesome_test_bucket_000B1_officedrop/test%2Flarge_multipart_file",
+    #       :e_tag=>"\"72b81ac08aed4d4d1055c11f56c2a258-1\"",
+    #       :key=>"test/large_multipart_file",
+    #       :bucket=>"right_s3_awesome_test_bucket_000B1_officedrop"}
+    def store_object_multipart(params)
+      AwsUtils.allow_only([:bucket, :key, :data, :headers, :part_size, :retry_count], params)
+      AwsUtils.mandatory_arguments([:bucket, :key, :data], params)
+      params[:headers] = {} unless params[:headers]
+
+      params[:data].binmode if(params[:data].respond_to?(:binmode)) # On Windows, if someone opens a file in text mode, we must reset it to binary mode for streaming to work properly
+
+      # detect whether we are using straight read or converting to string first
+      unless(params[:data].respond_to?(:read))
+        params[:data] = StringIO.new(params[:data].to_s)
+      end
+
+      # make sure part size is > 5 MB minimum
+      params[:part_size] ||= MINIMUM_PART_SIZE
+      if params[:part_size] < MINIMUM_PART_SIZE
+        raise AwsError.new("Part size for a multipart upload must be greater than or equal to #{5 * 1024 * 1024} bytes.  #{params[:part_size]} bytes was provided.")
+      end
+
+      # make sure retry_count is positive
+      params[:retry_count] ||= DEFAULT_RETRY_COUNT
+      if params[:retry_count] < 0
+        raise AwsError.new("Retry count must be positive.  #{params[:retry_count]} bytes was provided.")
+      end
+
+      # Set 100-continue for large part sizes
+      if (params[:part_size] >= USE_100_CONTINUE_PUT_SIZE)
+        params[:headers]['expect'] = '100-continue'
+      end
+
+      # initiate upload
+      initiate_hash = generate_rest_request('POST', params[:headers].merge(:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?uploads"))
+      initiate_resp = request_info(initiate_hash, S3MultipartUploadInitiateResponseParser.new)
+      upload_id = initiate_resp[:upload_id]
+
+      # split into parts and upload each one, re-trying if necessary
+      #   upload occurs serially at this time.
+      part_etags = []
+      part_data = ""
+      index = 1
+      until params[:data].eof?
+        part_data = params[:data].read(params[:part_size])
+        unless part_data.size == 0
+          retry_attempts = 1
+          while true
+            begin
+              send_part_hash = generate_rest_request('PUT', params[:headers].merge({ :url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?partNumber=#{index}&uploadId=#{upload_id}", :data=>part_data } ))
+              send_part_resp = request_info(send_part_hash, S3HttpResponseHeadParser.new)
+              part_etags << {:part_num => index, :etag => send_part_resp['etag']}
+              index += 1
+              break # successful, can move to next part
+            rescue AwsError => e
+              if retry_attempts >= params[:retry_count]
+                raise e
+              else
+                #Hit an error attempting to transmit part, retry until retry_attemts have been exhausted
+                retry_attempts += 1
+              end
+            end
+          end
+        end
+      end
+
+      # assemble complete upload message
+      complete_body = "<CompleteMultipartUpload>"
+      part_etags.each do |part_hash|
+        complete_body << "<Part><PartNumber>#{part_hash[:part_num]}</PartNumber><ETag>#{part_hash[:etag]}</ETag></Part>"
+      end
+      complete_body << "</CompleteMultipartUpload>"
+      complete_req_hash = generate_rest_request('POST', {:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?uploadId=#{upload_id}", :data => complete_body})
+      return request_info(complete_req_hash, S3CompleteMultipartParser.new)
+    rescue
+      on_exception
+    end
+
+    class S3MultipartUploadInitiateResponseParser < RightAWSParser
+      def reset
+        @result = {}
+      end
+      def headers_to_string(headers)
+        result = {}
+        headers.each do |key, value|
+          value       = value.first if value.is_a?(Array) && value.size<2
+          result[key] = value
+        end
+        result
+      end
+      def tagend(name)
+        case name
+          when 'UploadId'          then @result[:upload_id] = @text
+        end
+      end
+    end
+
+    class S3CompleteMultipartParser < RightAWSParser  # :nodoc:
+      def reset
+        @result = {}
+      end
+      def tagend(name)
+        case name
+          when 'Location' then @result[:location] = @text
+          when 'Bucket'   then @result[:bucket]   = @text
+          when 'Key'      then @result[:key]      = @text
+          when 'ETag'     then @result[:e_tag]    = @text
+        end
+      end
+    end
+
       # Retrieves object data from Amazon. Returns a +hash+  or an exception.
       #
       #  s3.get('my_awesome_bucket', 'log/curent/1.log') #=>
@@ -655,6 +790,34 @@ module RightAws
       on_exception
     end
 
+    # Deletes multiple keys. Returns an array with errors, if any.
+    #
+    #  s3.delete_multiple('my_awesome_bucket', ['key1', 'key2', ...)
+    #    #=> [ { :key => 'key2', :code => 'AccessDenied', :message => "Access Denied" } ]
+    #
+    def delete_multiple(bucket, keys=[], headers={})
+      errors = []
+      keys = Array.new(keys)
+      while keys.length > 0
+        data = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+        data += "<Delete>\n<Quiet>true</Quiet>\n"
+        keys.take(MULTI_OBJECT_DELETE_MAX_KEYS).each do |key|
+          data += "<Object><Key>#{AwsUtils::xml_escape(key)}</Key></Object>\n"
+        end
+        data += "</Delete>"
+        req_hash = generate_rest_request('POST', headers.merge(
+          :url  => "#{bucket}?delete",
+          :data => data,
+          'content-md5' => AwsUtils::content_md5(data)
+        ))
+        errors += request_info(req_hash, S3DeleteMultipleParser.new)
+        keys = keys.drop(MULTI_OBJECT_DELETE_MAX_KEYS)
+      end
+      errors
+    rescue
+      on_exception
+    end
+
       # Copy an object. 
       #  directive: :copy    - copy meta-headers from source (default value)
       #             :replace - replace meta-headers by passed ones
@@ -1021,6 +1184,23 @@ module RightAws
       on_exception
     end
 
+    class S3DeleteMultipleParser < RightAWSParser # :nodoc:
+      def reset
+        @result = []
+      end
+      def tagstart(name, attributes)
+        @error = {} if name == 'Error'
+      end
+      def tagend(name)
+        case name
+          when 'Key'     then @error[:key]     = @text
+          when 'Code'    then @error[:code]    = @text
+          when 'Message' then @error[:message] = @text
+          when 'Error'   then @result << @error
+        end
+      end
+    end
+
     #-----------------------------------------------------------------
     #      PARSERS:
     #-----------------------------------------------------------------

data/right_aws.gemspec

@@ -43,6 +43,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('rake')
   spec.add_development_dependency('rcov')
 
+  spec.summary = 'The RightScale AWS gems have been designed to provide a robust, fast, and secure interface to Amazon EC2, EBS, S3, SQS, SDB, and CloudFront.'
   spec.description = <<-EOF
 == DESCRIPTION:
 

data/test/s3/test_right_s3.rb

@@ -13,6 +13,8 @@ class TestS3 < Test::Unit::TestCase
     @key1   = 'test/woohoo1/'
     @key2   = 'test1/key/woohoo2'
     @key3   = 'test2/A%B@C_D&E?F+G=H"I'
+    @key4   = 'test/large_multipart_file_string'
+    @key5   = 'test/large_multipart_file_stream'
     @key1_copy =     'test/woohoo1_2'
     @key1_new_name = 'test/woohoo1_3'
     @key2_new_name = 'test1/key/woohoo2_new'
@@ -41,6 +43,26 @@ class TestS3 < Test::Unit::TestCase
     assert @s3.put(@bucket, @key3, RIGHT_OBJECT_TEXT, 'x-amz-meta-family'=>'Woohoo3!'), 'Put bucket fail'
   end
 
+  def test_04_put_multipart_string
+    test_text = ""
+    for i in 1..100000
+      test_text << "Testing test text #{i}\n"
+    end
+    assert @s3.store_object_multipart({:bucket => @bucket, :key => @key4, :data => StringIO.new(test_text)}), 'Put bucket multipart fail'
+  end
+
+  def test_04b_store_object_multipart_stream
+    rd, wr = IO.pipe
+    producer = Thread.new(wr) do |out|
+      for i in 1..100000
+        out.write("Testing stream text #{i}\n")
+      end
+      out.close
+    end
+    assert @s3.store_object_multipart({:bucket => @bucket, :key => @key5, :data => rd , :part_size => (20*1024*1024)}), 'Put bucket multipart fail'
+    rd.close
+  end
+
   def test_05_get_and_get_object
     assert_raise(Rightscale::AwsError) { @s3.get(@bucket, 'undefined/key') }
     data1 = @s3.get(@bucket, @key1)
@@ -74,10 +96,12 @@ class TestS3 < Test::Unit::TestCase
 
   def test_08_keys
     keys = @s3.list_bucket(@bucket).map{|b| b[:key]}
-    assert_equal keys.size, 3, "There should be 3 keys"
+    assert_equal keys.size, 5, "There should be 5 keys"
     assert(keys.include?(@key1))
     assert(keys.include?(@key2))
     assert(keys.include?(@key3))
+    assert(keys.include?(@key4))
+    assert(keys.include?(@key5))
   end
 
   def test_09_copy_key
@@ -144,6 +168,7 @@ class TestS3 < Test::Unit::TestCase
 
 
 
+
   #---------------------------
   # Rightscale::S3 classes
   #---------------------------
@@ -408,7 +433,7 @@ class TestS3 < Test::Unit::TestCase
     sleep 10
 
     assert_equal({:enabled => true, :targetbucket => @bucket2, :targetprefix => "loggylogs/"}, bucket.logging_info)
-
+    
     assert bucket.disable_logging
 
       # check 'Drop' method
@@ -457,14 +482,63 @@ class TestS3 < Test::Unit::TestCase
     end
   end
 
+  def test_44_delete_multiple
+    bucket = RightAws::S3::Bucket.create(@s, @bucket, true)
+
+    key1 = Rightscale::S3::Key.create(bucket, @key1)
+    key2 = Rightscale::S3::Key.create(bucket, @key2)
+    key3 = Rightscale::S3::Key.create(bucket, @key3)
+
+    assert @s3.put(@bucket, @key1, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+    assert @s3.put(@bucket, @key2, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+    assert @s3.put(@bucket, @key3, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+
+    key1.refresh
+    key2.refresh
+    key3.refresh
+
+    assert key1.exists?
+    assert key2.exists?
+    assert key3.exists?
+
+    result = @s3.delete_multiple(@bucket, [@key1, @key2, @key3])
+    assert result.empty?
+
+    key1.refresh
+    key2.refresh
+    key3.refresh
+
+    assert !key1.exists?
+    assert !key2.exists?
+    assert !key3.exists?
+  end
+
+  def test_45_delete_multiple_more_than_1000_objects
+    n = 1200
+    keys = (1..n).map { |i| "key-#{i}"}
+
+    keys.each do |key|
+      assert @s3.put(@bucket, key, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+    end
+
+    result = @s3.delete_multiple(@bucket, keys)
+    assert result.empty?
+
+    keys_after = @s3.list_bucket(@bucket).map { |obj| obj[:key] }
+
+    keys.each do |key|
+      assert !keys_after.include?(key)
+    end
+  end
+
   private
 
   def request( uri )
     url = URI.parse( uri )
 
-    http = Net::HTTP.new(url.host, 443)
-    http.use_ssl = true
-    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+    http = Net::HTTP.new(url.host, 80)
+#    http.use_ssl = true
+#    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
     http.request(Net::HTTP::Get.new( url.request_uri ))
   end
 

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: right_aws
 version: !ruby/object:Gem::Version 
-  hash: 15
-  prerelease: 
+  prerelease: false
   segments: 
   - 3
   - 0
-  - 4
-  version: 3.0.4
+  - 5
+  version: 3.0.5
 platform: ruby
 authors: 
 - RightScale, Inc.
@@ -15,17 +14,16 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-04-10 00:00:00 Z
+date: 2013-03-05 00:00:00 -08:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: right_http_connection
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 21
         segments: 
         - 1
         - 2
@@ -37,11 +35,9 @@ dependencies:
   name: rake
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -51,11 +47,9 @@ dependencies:
   name: rcov
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -186,6 +180,7 @@ files:
 - test/sqs/test_right_sqs_gen2.rb
 - test/test_credentials.rb
 - test/ts_right_aws.rb
+has_rdoc: true
 homepage: 
 licenses: []
 
@@ -198,32 +193,28 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 57
       segments: 
       - 1
       - 8
       - 7
       version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: 
 - libxml-ruby >= 0.5.2.0 is encouraged
 rubyforge_project: rightaws
-rubygems_version: 1.8.17
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
-summary: Interface classes for the Amazon EC2, SQS, and S3 Web Services
+summary: The RightScale AWS gems have been designed to provide a robust, fast, and secure interface to Amazon EC2, EBS, S3, SQS, SDB, and CloudFront.
 test_files: 
 - test/acf/test_helper.rb
 - test/acf/test_right_acf.rb