right_aws 1.9.0 → 3.1.0

data/lib/s3/right_s3.rb

@@ -59,7 +59,6 @@ module RightAws
     #    {:server       => 's3.amazonaws.com'   # Amazon service host: 's3.amazonaws.com'(default)
     #     :port         => 443                  # Amazon service port: 80 or 443(default)
     #     :protocol     => 'https'              # Amazon service protocol: 'http' or 'https'(default)
-    #     :multi_thread => true|false           # Multi-threaded (connection per each thread): true or false(default)
     #     :logger       => Logger Object}       # Logger instance: logs to STDOUT if omitted }
     def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={})
       @interface = S3Interface.new(aws_access_key_id, aws_secret_access_key, params)
@@ -98,10 +97,24 @@ module RightAws
     #  (section: Canned Access Policies)
     #
     def bucket(name, create=false, perms=nil, headers={})
-      headers['x-amz-acl'] = perms if perms
-      @interface.create_bucket(name, headers) if create
-      buckets.each { |bucket| return bucket if bucket.name == name }
-      nil
+      result = nil
+      if create
+        headers['x-amz-acl'] = perms if perms
+        @interface.create_bucket(name, headers) 
+      end
+      begin
+        buckets.each do |bucket| 
+          if bucket.name == name 
+            result = bucket
+            break
+          end
+        end
+      rescue RightAws::AwsError => e
+        # With non root creds one can use bucket(s) but can't list them
+        raise e unless e.message['AccessDenied']
+        result = Bucket::new(self, name)
+      end
+      result
     end
     
 
@@ -217,25 +230,23 @@ module RightAws
         #
         #  keys, service = bucket.keys_and_service({'max-keys'=> 2, 'prefix' => 'logs'})
         #  p keys    #=> # 2 keys array
-        #  p service #=> {"max-keys"=>"2", "prefix"=>"logs", "name"=>"my_awesome_bucket", "marker"=>"", "is_truncated"=>true}
+        #  p service #=> {"max-keys"=>"2", "prefix"=>"logs", "name"=>"my_awesome_bucket", "marker"=>"", "is_truncated"=>true, :common_prefixes=>[]}
         #
       def keys_and_service(options={}, head=false)
         opt = {}; options.each{ |key, value| opt[key.to_s] = value }
-        service_data = {}
-        thislist = {}
-        list = []
-        @s3.interface.incrementally_list_bucket(@name, opt) do |thislist|
-          thislist[:contents].each do |entry|
+        service = {}
+        keys = []
+        @s3.interface.incrementally_list_bucket(@name, opt) do |_service|
+          service = _service
+          service[:contents].each do |entry|
             owner = Owner.new(entry[:owner_id], entry[:owner_display_name])
             key = Key.new(self, entry[:key], nil, {}, {}, entry[:last_modified], entry[:e_tag], entry[:size], entry[:storage_class], owner)
             key.head if head
-            list << key
+            keys << key
           end
         end
-        thislist.each_key do |key|
-          service_data[key] = thislist[key] unless (key == :contents || key == :common_prefixes)
-        end
-        [list, service_data]
+        service.delete(:contents)
+        [keys, service]
       end
 
         # Retrieve key information from Amazon. 
@@ -248,11 +259,12 @@ module RightAws
         #  key = RightAws::S3::Key.create(bucket, 'logs/today/1.log')
         #  key.head
         #
-      def key(key_name, head=false)
-        raise 'Key name can not be empty.' if key_name.blank?
+      def key(key_name, head=false, &blck)
+        raise 'Key name can not be empty.' if key_name.right_blank?
         key_instance = nil
           # if this key exists - find it ....
         keys({'prefix'=>key_name}, head).each do |key|
+          blck.call if block_given?
           if key.name == key_name.to_s
             key_instance = key
             break
@@ -271,17 +283,17 @@ module RightAws
         #
         #  bucket.put('logs/today/1.log', 'Olala!') #=> true
         #
-      def put(key, data=nil, meta_headers={}, perms=nil, headers={})
+      def put(key, data=nil, meta_headers={}, perms=nil, headers={}, &blck)
         key = Key.create(self, key.to_s, data, meta_headers) unless key.is_a?(Key) 
-        key.put(data, perms, headers)
+        key.put(data, perms, headers, &blck)
       end
 
-        # Retrieve object data from Amazon. 
+        # Retrieve data object from Amazon. 
         # The +key+ is a +String+ or Key. 
-        # Returns Key instance. 
+        # Returns String instance. 
         #
-        #  key = bucket.get('logs/today/1.log') #=> 
-        #  puts key.data #=> 'sasfasfasdf'
+        #  data = bucket.get('logs/today/1.log') #=> 
+        #  puts data #=> 'sasfasfasdf'
         #
       def get(key, headers={})
         key = Key.create(self, key.to_s) unless key.is_a?(Key)
@@ -346,9 +358,10 @@ module RightAws
         # If +force+ is set, clears and deletes the bucket. 
         # Returns +true+. 
         #
-        #  bucket.delete(true) #=> true
+        #  bucket.delete(:force => true) #=> true
         #
-      def delete(force=false)
+      def delete(options={})
+        force = options.is_a?(Hash) && options[:force]==true
         force ? @s3.interface.force_delete_bucket(@name) : @s3.interface.delete_bucket(@name)
       end
 
@@ -460,6 +473,30 @@ module RightAws
         get if !@data and exists?
         @data
       end
+        
+        # Getter for the 'content-type' metadata
+      def content_type
+        @headers['content-type'] if @headers
+      end
+      
+        # Helper to get and URI-decode a header metadata.
+        # Metadata have to be HTTP encoded (rfc2616) as we use the Amazon S3 REST api
+        # see http://docs.amazonwebservices.com/AmazonS3/latest/index.html?UsingMetadata.html
+      def decoded_meta_headers(key = nil)
+        if key
+          # Get one metadata value by its key
+          URI.decode(@meta_headers[key.to_s])
+        else
+          # Get a hash of all metadata with a decoded value
+          @decoded_meta_headers ||= begin
+            metadata = {}
+            @meta_headers.each do |key, value|
+              metadata[key.to_sym] = URI.decode(value)
+            end
+            metadata
+          end
+        end
+      end
       
         # Retrieve object data and attributes from Amazon. 
         # Returns a +String+. 
@@ -482,11 +519,33 @@ module RightAws
         #   ...
         #  key.put('Olala!')   #=> true
         #
-      def put(data=nil, perms=nil, headers={})
+      def put(data=nil, perms=nil, headers={}, &blck)
         headers['x-amz-acl'] = perms if perms
         @data = data || @data
         meta  = self.class.add_meta_prefix(@meta_headers)
-        @bucket.s3.interface.put(@bucket.name, @name, @data, meta.merge(headers))
+        @bucket.s3.interface.put(@bucket.name, @name, @data, meta.merge(headers), &blck)
+      end
+
+        # Store object data on S3 using the Multipart Upload API. This is useful if you do not know the file size
+        # upfront (for example reading from pipe or socket) or if you are transmitting data over an unreliable network.
+        #
+        # Parameter +data+ is an object which responds to :read or an object which can be converted to a String prior to upload.
+        # Parameter +part_size+ determines the size of each part sent (must be > 5MB per Amazon's API requirements)
+        #
+        # If data is a stream the caller is responsible for calling close() on the stream after this methods returns
+        #
+        # Returns +true+.
+        #
+        #  upload_data = StringIO.new('My sample data')
+        #  key = RightAws::S3::Key.create(bucket, 'logs/today/1.log')
+        #  key.data = upload_data
+        #  key.put_multipart(:part_size => 5*1024*1024)             #=> true
+        #
+      def put_multipart(data=nil, perms=nil, headers={}, part_size=nil)
+        headers['x-amz-acl'] = perms if perms
+        @data = data || @data
+        meta  = self.class.add_meta_prefix(@meta_headers)
+        @bucket.s3.interface.store_object_multipart({:bucket => @bucket.name, :key => @name, :data => @data, :headers => meta.merge(headers), :part_size => part_size})
       end
       
         # Rename an object. Returns new object name.
@@ -626,7 +685,7 @@ module RightAws
         #  key.delete #=> true
         #
       def delete
-        raise 'Key name must be specified.' if @name.blank?
+        raise 'Key name must be specified.' if @name.right_blank?
         @bucket.s3.interface.delete(@bucket, @name) 
       end
       
@@ -759,11 +818,11 @@ module RightAws
         @thing = thing
         @id    = id
         @name  = name
-        @perms = perms.to_a
+        @perms = Array(perms)
         case action
-          when :apply:             apply
-          when :refresh:           refresh
-          when :apply_and_refresh: apply; refresh
+          when :apply             then apply
+          when :refresh           then refresh
+          when :apply_and_refresh then apply; refresh
         end
       end
       
@@ -775,9 +834,13 @@ module RightAws
         false
       end
       
-        # Return Grantee type (+String+): "Group" or "CanonicalUser".
+        # Return Grantee type (+String+): "Group", "AmazonCustomerByEmail" or "CanonicalUser".
       def type
-        @id[/^http:/] ? "Group" : "CanonicalUser"
+        case @id
+        when /^http:/ then "Group"
+        when /@/      then "AmazonCustomerByEmail"
+        else               "CanonicalUser"
+        end
       end
  
         # Return a name or an id.
@@ -872,7 +935,11 @@ module RightAws
       end
 
       def to_xml   # :nodoc:
-        id_str = @id[/^http/] ? "<URI>#{@id}</URI>" : "<ID>#{@id}</ID>"
+        id_str = case @id
+                 when /^http/ then "<URI>#{@id}</URI>"
+                 when /@/     then "<EmailAddress>#{@id}</EmailAddress>"
+                 else              "<ID>#{@id}</ID>"
+                 end
         grants = ''
         @perms.each do |perm|
           grants << "<Grant>"    +
@@ -1012,8 +1079,8 @@ module RightAws
         #
         #  bucket.get('logs/today/1.log', 1.hour)
         #
-      def get(key, expires=nil, headers={})
-        @s3.interface.get_link(@name, key.to_s, expires, headers)
+      def get(key, expires=nil, headers={}, response_params={})
+        @s3.interface.get_link(@name, key.to_s, expires, headers, response_params)
       end
        
         # Generate link to delete bucket. 
@@ -1054,7 +1121,7 @@ module RightAws
         @bucket       = bucket
         @name         = name.to_s
         @meta_headers = meta_headers
-        raise 'Key name can not be empty.' if @name.blank?
+        raise 'Key name can not be empty.' if @name.right_blank?
       end
       
         # Generate link to PUT key data. 
@@ -1069,8 +1136,8 @@ module RightAws
         #
         #  bucket.get('logs/today/1.log', 1.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket/logs%2Ftoday%2F1.log?Signature=h...M%3D&Expires=1180820032&AWSAccessKeyId=1...2
         #
-      def get(expires=nil, headers={})
-        @bucket.s3.interface.get_link(@bucket.to_s, @name, expires, headers)
+      def get(expires=nil, headers={}, response_params={})
+        @bucket.s3.interface.get_link(@bucket.to_s, @name, expires, headers, response_params)
       end
        
         # Generate link to delete key. 

data/lib/s3/right_s3_interface.rb

@@ -26,18 +26,37 @@ module RightAws
   class S3Interface < RightAwsBase
     
     USE_100_CONTINUE_PUT_SIZE = 1_000_000
+    MINIMUM_PART_SIZE         = 5 * 1024 * 1024
+    DEFAULT_RETRY_COUNT       = 5
     
     include RightAwsBaseInterface
     
     DEFAULT_HOST           = 's3.amazonaws.com'
     DEFAULT_PORT           = 443
     DEFAULT_PROTOCOL       = 'https'
+    DEFAULT_SERVICE        = '/'
     REQUEST_TTL            = 30
     DEFAULT_EXPIRES_AFTER  =   1 * 24 * 60 * 60 # One day's worth of seconds
     ONE_YEAR_IN_SECONDS    = 365 * 24 * 60 * 60
     AMAZON_HEADER_PREFIX   = 'x-amz-'
     AMAZON_METADATA_PREFIX = 'x-amz-meta-'
+    S3_REQUEST_PARAMETERS = [ 'acl',
+        'location',
+        'logging', # this one is beta, no support for now
+        'partNumber',
+        'response-content-type',
+        'response-content-language',
+        'response-expires',
+        'response-cache-control',
+        'response-content-disposition',
+        'response-content-encoding',
+        'torrent',
+        'uploadId',
+        'uploads',
+        'delete'].sort
+    MULTI_OBJECT_DELETE_MAX_KEYS = 1000
 
+    
     @@bench = AwsBenchmarkingBlock.new
     def self.bench_xml
       @@bench.xml
@@ -46,23 +65,37 @@ module RightAws
       @@bench.service
     end
 
+      # Params supported:
+      #  :no_subdomains => true # do not use bucket as a part of domain name but as a part of path
+    @@params = {}
+    def self.params
+      @@params
+    end
+
+      # get custom option
+    def param(name)
+      # - check explicitly defined param (@params)
+      # - otherwise check implicitly defined one (@@params)
+      @params.has_key?(name) ? @params[name] : @@params[name]
+    end
 
       # Creates new RightS3 instance.
       #
-      #  s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX', {:multi_thread => true, :logger => Logger.new('/tmp/x.log')}) #=> #<RightAws::S3Interface:0xb7b3c27c>
+      #  s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX', {:logger => Logger.new('/tmp/x.log')}) #=> #<RightAws::S3Interface:0xb7b3c27c>
       #  
       # Params is a hash:
       #
-      #    {:server       => 's3.amazonaws.com'   # Amazon service host: 's3.amazonaws.com'(default)
-      #     :port         => 443                  # Amazon service port: 80 or 443(default)
-      #     :protocol     => 'https'              # Amazon service protocol: 'http' or 'https'(default)
-      #     :multi_thread => true|false           # Multi-threaded (connection per each thread): true or false(default)
-      #     :logger       => Logger Object}       # Logger instance: logs to STDOUT if omitted }
+      #    {:server        => 's3.amazonaws.com'   # Amazon service host: 's3.amazonaws.com'(default)
+      #     :port          => 443                  # Amazon service port: 80 or 443(default)
+      #     :protocol      => 'https'              # Amazon service protocol: 'http' or 'https'(default)
+      #     :logger        => Logger Object        # Logger instance: logs to STDOUT if omitted
+      #     :no_subdomains => true}                # Force placing bucket name into path instead of domain name
       #
     def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={})
       init({ :name             => 'S3', 
              :default_host     => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).host   : DEFAULT_HOST, 
-             :default_port     => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).port   : DEFAULT_PORT, 
+             :default_port     => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).port   : DEFAULT_PORT,
+             :default_service  => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).path   : DEFAULT_SERVICE,
              :default_protocol => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).scheme : DEFAULT_PROTOCOL }, 
            aws_access_key_id     || ENV['AWS_ACCESS_KEY_ID'], 
            aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'], 
@@ -78,7 +111,11 @@ module RightAws
       s3_headers = {}
       headers.each do |key, value|
         key = key.downcase
-        s3_headers[key] = value.to_s.strip if key[/^#{AMAZON_HEADER_PREFIX}|^content-md5$|^content-type$|^date$/o]
+        value = case
+                when value.is_a?(Array) then value.join('')
+                else                         value.to_s
+                end
+        s3_headers[key] = value.strip if key[/^#{AMAZON_HEADER_PREFIX}|^content-md5$|^content-type$|^date$/o]
       end
       s3_headers['content-type'] ||= ''
       s3_headers['content-md5']  ||= ''
@@ -89,13 +126,20 @@ module RightAws
       s3_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value|
         out_string << (key[/^#{AMAZON_HEADER_PREFIX}/o] ? "#{key}:#{value}\n" : "#{value}\n")
       end
-        # ignore everything after the question mark...
+      # ignore everything after the question mark by default...
       out_string << path.gsub(/\?.*$/, '')
-       # ...unless there is an acl or torrent parameter
-      out_string << '?acl'      if path[/[&?]acl($|&|=)/]
-      out_string << '?torrent'  if path[/[&?]torrent($|&|=)/]
-      out_string << '?location' if path[/[&?]location($|&|=)/]
-      out_string << '?logging'  if path[/[&?]logging($|&|=)/]  # this one is beta, no support for now
+      # ... unless there is a parameter that we care about.
+      S3_REQUEST_PARAMETERS.each do |parameter|
+        if path[/[&?]#{parameter}(=[^&]*)?($|&)/]
+          if $1
+            value = CGI::unescape($1)
+          else
+            value = ''
+          end
+          out_string << (out_string[/[?]/] ? "&#{parameter}#{value}" : "?#{parameter}#{value}")
+        end
+      end
+
       out_string
     end
 
@@ -108,37 +152,43 @@ module RightAws
       end
       true
     end
-    
-      # Generates request hash for REST API.
-      # Assumes that headers[:url] is URL encoded (use CGI::escape)
-    def generate_rest_request(method, headers)  # :nodoc:
+
+    def fetch_request_params(headers) #:nodoc:
       # default server to use
-      server = @params[:server]
-      # fix path
-      path_to_sign = headers[:url]
-      path_to_sign = "/#{path_to_sign}" unless path_to_sign[/^\//]
+      server  = @params[:server]
+      service = @params[:service].to_s
+      service.chop! if service[%r{/$}] # remove trailing '/' from service
       # extract bucket name and check it's dns compartibility
-      path_to_sign[%r{^/([a-z0-9._-]*)(/[^?]*)?(\?.+)?}i]
+      headers[:url].to_s[%r{^([a-z0-9._-]*)(/[^?]*)?(\?.+)?}i]
       bucket_name, key_path, params_list = $1, $2, $3
+      key_path = key_path.gsub( '%2F', '/' ) if key_path
       # select request model
-      if is_dns_bucket?(bucket_name)
-        # add backet to a server name
+      if !param(:no_subdomains) && is_dns_bucket?(bucket_name)
+        # fix a path
         server = "#{bucket_name}.#{server}"
-        # remove bucket from the path
-        path = "#{key_path || '/'}#{params_list}"
-        # refactor the path (add '/' before params_list if the key is empty)
-        path_to_sign = "/#{bucket_name}#{path}"
+        key_path ||= '/'
+        path = "#{service}#{key_path}#{params_list}"
       else
-        path = path_to_sign
+        path = "#{service}/#{bucket_name}#{key_path}#{params_list}"
       end
+      path_to_sign = "#{service}/#{bucket_name}#{key_path}#{params_list}"
+#      path_to_sign = "/#{bucket_name}#{key_path}#{params_list}"
+      [ server, path, path_to_sign ]
+    end
+
+      # Generates request hash for REST API.
+      # Assumes that headers[:url] is URL encoded (use CGI::escape)
+    def generate_rest_request(method, headers)  # :nodoc:
+        # calculate request data
+      server, path, path_to_sign = fetch_request_params(headers)
       data = headers[:data]
-        # remove unset(==optional) and symbolyc keys
-      headers.each{ |key, value| headers.delete(key) if (value.nil? || key.is_a?(Symbol)) }
+        # make sure headers are downcased strings
+      headers = AwsUtils::fix_headers(headers)
         #
       headers['content-type'] ||= ''
       headers['date']           = Time.now.httpdate
         # create request
-      request      = "Net::HTTP::#{method.capitalize}".constantize.new(path)
+      request      = "Net::HTTP::#{method.capitalize}".right_constantize.new(path)
       request.body = data if data
         # set request headers and meta headers
       headers.each      { |key, value| request[key.to_s] = value }
@@ -157,12 +207,9 @@ module RightAws
       # Sends request to Amazon and parses the response.
       # Raises AwsError if any banana happened.
     def request_info(request, parser, &block) # :nodoc:
-      thread = @params[:multi_thread] ? Thread.current : Thread.main
-      thread[:s3_connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError, :logger => @logger)
-      request_info_impl(thread[:s3_connection], @@bench, request, parser, &block)
+      request_info_impl(:s3_connection, @@bench, request, parser, &block)
     end
 
-
       # Returns an array of customer's buckets. Each item is a +hash+.
       #
       #  s3.list_all_my_buckets #=> 
@@ -187,8 +234,14 @@ module RightAws
       #
     def create_bucket(bucket, headers={})
       data = nil
-      unless headers[:location].blank?
-        data = "<CreateBucketConfiguration><LocationConstraint>#{headers[:location].to_s.upcase}</LocationConstraint></CreateBucketConfiguration>"
+      location = case headers[:location].to_s
+                 when 'us','US' then ''
+                 when 'eu'      then 'EU'
+                 else                headers[:location].to_s
+                 end
+      
+      unless location.right_blank?
+        data = "<CreateBucketConfiguration><LocationConstraint>#{location}</LocationConstraint></CreateBucketConfiguration>"
       end
       req_hash = generate_rest_request('PUT', headers.merge(:url=>bucket, :data => data))
       request_info(req_hash, RightHttp2xxParser.new)
@@ -238,7 +291,7 @@ module RightAws
       AwsUtils.allow_only([:bucket,:xmldoc, :headers], params)
       params[:headers] = {} unless params[:headers]
       req_hash = generate_rest_request('PUT', params[:headers].merge(:url=>"#{params[:bucket]}?logging", :data => params[:xmldoc]))
-      request_info(req_hash, S3TrueParser.new)
+      request_info(req_hash, RightHttp2xxParser.new)
     rescue
       on_exception
     end
@@ -273,7 +326,7 @@ module RightAws
       #                  'max-keys'     => "5"}, ..., {...}]
       #
     def list_bucket(bucket, options={}, headers={})
-      bucket  += '?'+options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless options.blank?
+      bucket  += '?'+options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless options.right_blank?
       req_hash = generate_rest_request('GET', headers.merge(:url=>bucket))
       request_info(req_hash, S3ListBucketParser.new(:logger => @logger))
     rescue
@@ -308,10 +361,10 @@ module RightAws
     #     ]
     #   }
     def incrementally_list_bucket(bucket, options={}, headers={}, &block)
-      internal_options = options.symbolize_keys
+      internal_options = options.right_symbolize_keys
       begin 
         internal_bucket = bucket.dup
-        internal_bucket  += '?'+internal_options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless internal_options.blank?
+        internal_bucket  += '?'+internal_options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless internal_options.right_blank?
         req_hash = generate_rest_request('GET', headers.merge(:url=>internal_bucket))
         response = request_info(req_hash, S3ImprovedListBucketParser.new(:logger => @logger))
         there_are_more_keys = response[:is_truncated]
@@ -382,7 +435,7 @@ module RightAws
       # mode.
       #
       
-    def put(bucket, key, data=nil, headers={})
+    def put(bucket, key, data=nil, headers={}, &blck)
       # On Windows, if someone opens a file in text mode, we must reset it so
       # to binary mode for streaming to work properly
       if(data.respond_to?(:binmode))
@@ -393,7 +446,7 @@ module RightAws
         headers['expect'] = '100-continue'
       end
       req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}", :data=>data))
-      request_info(req_hash, RightHttp2xxParser.new)
+      request_info(req_hash, RightHttp2xxParser.new, &blck)
     rescue
       on_exception
     end
@@ -478,6 +531,134 @@ module RightAws
       r[:verified_md5] ? (return r) : (raise AwsError.new("Uploaded object failed MD5 checksum verification: #{r.inspect}"))
     end
     
+    # New experimental API for uploading objects using the multipart upload API.
+    # store_object_multipart is similar in function to the store_object method, but breaks the input into parts and transmits each
+    # part separately.  The multipart upload API has the benefit of being be able to retransmit a part in isolation without needing to
+    # restart the entire upload.  This makes it ideal for uploading large files over unreliable networks.  It also does not
+    # require the file size to be known before starting the upload, making it useful for stream data as it is created (say via reading a pipe or socket).
+    # The hash of the response headers contains useful information like the location (the URI for the newly created object), bucket, key, and etag).
+    #
+    # The optional argument of :headers allows the caller to specify arbitrary request header values.
+    #
+    # s3.store_object_multipart(:bucket => "foobucket", :key => "foo", :data => "polemonium" )
+    #   => {:location=>"https://s3.amazonaws.com/right_s3_awesome_test_bucket_000B1_officedrop/test%2Flarge_multipart_file",
+    #       :e_tag=>"\"72b81ac08aed4d4d1055c11f56c2a258-1\"",
+    #       :key=>"test/large_multipart_file",
+    #       :bucket=>"right_s3_awesome_test_bucket_000B1_officedrop"}
+    #
+    # f = File.new("some_file", "r")
+    # s3.store_object_multipart(:bucket => "foobucket", :key => "foo", :data => f )
+    #   => {:location=>"https://s3.amazonaws.com/right_s3_awesome_test_bucket_000B1_officedrop/test%2Flarge_multipart_file",
+    #       :e_tag=>"\"72b81ac08aed4d4d1055c11f56c2a258-1\"",
+    #       :key=>"test/large_multipart_file",
+    #       :bucket=>"right_s3_awesome_test_bucket_000B1_officedrop"}
+    def store_object_multipart(params)
+      AwsUtils.allow_only([:bucket, :key, :data, :headers, :part_size, :retry_count], params)
+      AwsUtils.mandatory_arguments([:bucket, :key, :data], params)
+      params[:headers] = {} unless params[:headers]
+
+      params[:data].binmode if(params[:data].respond_to?(:binmode)) # On Windows, if someone opens a file in text mode, we must reset it to binary mode for streaming to work properly
+
+      # detect whether we are using straight read or converting to string first
+      unless(params[:data].respond_to?(:read))
+        params[:data] = StringIO.new(params[:data].to_s)
+      end
+
+      # make sure part size is > 5 MB minimum
+      params[:part_size] ||= MINIMUM_PART_SIZE
+      if params[:part_size] < MINIMUM_PART_SIZE
+        raise AwsError.new("Part size for a multipart upload must be greater than or equal to #{5 * 1024 * 1024} bytes.  #{params[:part_size]} bytes was provided.")
+      end
+
+      # make sure retry_count is positive
+      params[:retry_count] ||= DEFAULT_RETRY_COUNT
+      if params[:retry_count] < 0
+        raise AwsError.new("Retry count must be positive.  #{params[:retry_count]} bytes was provided.")
+      end
+
+      # Set 100-continue for large part sizes
+      if (params[:part_size] >= USE_100_CONTINUE_PUT_SIZE)
+        params[:headers]['expect'] = '100-continue'
+      end
+
+      # initiate upload
+      initiate_hash = generate_rest_request('POST', params[:headers].merge(:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?uploads"))
+      initiate_resp = request_info(initiate_hash, S3MultipartUploadInitiateResponseParser.new)
+      upload_id = initiate_resp[:upload_id]
+
+      # split into parts and upload each one, re-trying if necessary
+      #   upload occurs serially at this time.
+      part_etags = []
+      part_data = ""
+      index = 1
+      until params[:data].eof?
+        part_data = params[:data].read(params[:part_size])
+        unless part_data.size == 0
+          retry_attempts = 1
+          while true
+            begin
+              send_part_hash = generate_rest_request('PUT', params[:headers].merge({ :url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?partNumber=#{index}&uploadId=#{upload_id}", :data=>part_data } ))
+              send_part_resp = request_info(send_part_hash, S3HttpResponseHeadParser.new)
+              part_etags << {:part_num => index, :etag => send_part_resp['etag']}
+              index += 1
+              break # successful, can move to next part
+            rescue AwsError => e
+              if retry_attempts >= params[:retry_count]
+                raise e
+              else
+                #Hit an error attempting to transmit part, retry until retry_attemts have been exhausted
+                retry_attempts += 1
+              end
+            end
+          end
+        end
+      end
+
+      # assemble complete upload message
+      complete_body = "<CompleteMultipartUpload>"
+      part_etags.each do |part_hash|
+        complete_body << "<Part><PartNumber>#{part_hash[:part_num]}</PartNumber><ETag>#{part_hash[:etag]}</ETag></Part>"
+      end
+      complete_body << "</CompleteMultipartUpload>"
+      complete_req_hash = generate_rest_request('POST', {:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?uploadId=#{upload_id}", :data => complete_body})
+      return request_info(complete_req_hash, S3CompleteMultipartParser.new)
+    rescue
+      on_exception
+    end
+
+    class S3MultipartUploadInitiateResponseParser < RightAWSParser
+      def reset
+        @result = {}
+      end
+      def headers_to_string(headers)
+        result = {}
+        headers.each do |key, value|
+          value       = value.first if value.is_a?(Array) && value.size<2
+          result[key] = value
+        end
+        result
+      end
+      def tagend(name)
+        case name
+          when 'UploadId'          then @result[:upload_id] = @text
+        end
+      end
+    end
+
+    class S3CompleteMultipartParser < RightAWSParser  # :nodoc:
+      def reset
+        @result = {}
+      end
+      def tagend(name)
+        case name
+          when 'Location' then @result[:location] = @text
+          when 'Bucket'   then @result[:bucket]   = @text
+          when 'Key'      then @result[:key]      = @text
+          when 'ETag'     then @result[:e_tag]    = @text
+        end
+      end
+    end
+
       # Retrieves object data from Amazon. Returns a +hash+  or an exception.
       #
       #  s3.get('my_awesome_bucket', 'log/curent/1.log') #=>
@@ -609,6 +790,34 @@ module RightAws
       on_exception
     end
 
+    # Deletes multiple keys. Returns an array with errors, if any.
+    #
+    #  s3.delete_multiple('my_awesome_bucket', ['key1', 'key2', ...)
+    #    #=> [ { :key => 'key2', :code => 'AccessDenied', :message => "Access Denied" } ]
+    #
+    def delete_multiple(bucket, keys=[], headers={})
+      errors = []
+      keys = Array.new(keys)
+      while keys.length > 0
+        data = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+        data += "<Delete>\n<Quiet>true</Quiet>\n"
+        keys.take(MULTI_OBJECT_DELETE_MAX_KEYS).each do |key|
+          data += "<Object><Key>#{AwsUtils::xml_escape(key)}</Key></Object>\n"
+        end
+        data += "</Delete>"
+        req_hash = generate_rest_request('POST', headers.merge(
+          :url  => "#{bucket}?delete",
+          :data => data,
+          'content-md5' => AwsUtils::content_md5(data)
+        ))
+        errors += request_info(req_hash, S3DeleteMultipleParser.new)
+        keys = keys.drop(MULTI_OBJECT_DELETE_MAX_KEYS)
+      end
+      errors
+    rescue
+      on_exception
+    end
+
       # Copy an object. 
       #  directive: :copy    - copy meta-headers from source (default value)
       #             :replace - replace meta-headers by passed ones
@@ -674,7 +883,7 @@ module RightAws
       #                  <Permission>FULL_CONTROL</Permission></Grant></AccessControlList></AccessControlPolicy>" }
       #
     def get_acl(bucket, key='', headers={})
-      key = key.blank? ? '' : "/#{CGI::escape key}"
+      key = key.right_blank? ? '' : "/#{CGI::escape key}"
       req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}#{key}?acl"))
       request_info(req_hash, S3HttpResponseBodyParser.new) 
     rescue
@@ -704,7 +913,7 @@ module RightAws
       #       :display_name=>"root"}}
       #
     def get_acl_parse(bucket, key='', headers={})
-      key = key.blank? ? '' : "/#{CGI::escape key}"
+      key = key.right_blank? ? '' : "/#{CGI::escape key}"
       req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}#{key}?acl"))
       acl = request_info(req_hash, S3AclParser.new(:logger => @logger))
       result = {}
@@ -717,7 +926,7 @@ module RightAws
         else
           result[:grantees][key] = 
             { :display_name => grantee[:display_name] || grantee[:uri].to_s[/[^\/]*$/],
-              :permissions  => grantee[:permissions].to_a,
+              :permissions  => Array(grantee[:permissions]),
               :attributes   => grantee[:attributes] }
         end
       end
@@ -728,7 +937,7 @@ module RightAws
     
       # Sets the ACL on a bucket or object.
     def put_acl(bucket, key, acl_xml_doc, headers={})
-      key = key.blank? ? '' : "/#{CGI::escape key}"
+      key = key.right_blank? ? '' : "/#{CGI::escape key}"
       req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{bucket}#{key}?acl", :data=>acl_xml_doc))
       request_info(req_hash, S3HttpResponseBodyParser.new)
     rescue
@@ -806,36 +1015,25 @@ module RightAws
     #      Query API: Links
     #-----------------------------------------------------------------
 
+    def s3_link_escape(text)
+      #CGI::escape(text.to_s).gsub(/[+]/, '%20')
+      AwsUtils::amz_escape(text.to_s)
+    end
+    
       # Generates link for QUERY API
     def generate_link(method, headers={}, expires=nil) #:nodoc:
-      # default server to use
-      server = @params[:server]
-      # fix path
-      path_to_sign = headers[:url]
-      path_to_sign = "/#{path_to_sign}" unless path_to_sign[/^\//]
-      # extract bucket name and check it's dns compartibility
-      path_to_sign[%r{^/([a-z0-9._-]*)(/[^?]*)?(\?.+)?}i]
-      bucket_name, key_path, params_list = $1, $2, $3
-      # select request model
-      if is_dns_bucket?(bucket_name)
-        # add backet to a server name
-        server = "#{bucket_name}.#{server}"
-        # remove bucket from the path
-        path = "#{key_path || '/'}#{params_list}"
-        # refactor the path (add '/' before params_list if the key is empty)
-        path_to_sign = "/#{bucket_name}#{path}"
-      else
-        path = path_to_sign
-      end
-       # expiration time
+        # calculate request data
+      server, path, path_to_sign = fetch_request_params(headers)
+
+        # expiration time
       expires ||= DEFAULT_EXPIRES_AFTER
       expires   = Time.now.utc + expires if expires.is_a?(Fixnum) && (expires < ONE_YEAR_IN_SECONDS)
       expires   = expires.to_i
-        # remove unset(==optional) and symbolyc keys
-      headers.each{ |key, value| headers.delete(key) if (value.nil? || key.is_a?(Symbol)) }
+        # make sure headers are downcased strings
+      headers = AwsUtils::fix_headers(headers)
         #generate auth strings
       auth_string = canonical_string(method, path_to_sign, headers, expires)
-      signature   = CGI::escape(Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new("sha1"), @aws_secret_access_key, auth_string)).strip)
+      signature   = CGI::escape(AwsUtils::sign( @aws_secret_access_key, auth_string))
         # path building
       addon = "Signature=#{signature}&Expires=#{expires}&AWSAccessKeyId=#{@aws_access_key_id}"
       path += path[/\?/] ? "&#{addon}" : "?#{addon}"
@@ -879,7 +1077,7 @@ module RightAws
       #  s3.list_bucket_link('my_awesome_bucket') #=> url string
       #
     def list_bucket_link(bucket, options=nil, expires=nil, headers={})
-      bucket += '?' + options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless options.blank?
+      bucket += '?' + options.map{|k, v| "#{k.to_s}=#{s3_link_escape(v)}"}.join('&') unless options.right_blank?
       generate_link('GET', headers.merge(:url=>bucket), expires)
     rescue
       on_exception
@@ -890,7 +1088,7 @@ module RightAws
       #  s3.put_link('my_awesome_bucket',key, object) #=> url string
       #
     def put_link(bucket, key, data=nil, expires=nil, headers={})
-      generate_link('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}", :data=>data), expires)
+      generate_link('PUT', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}", :data=>data), expires)
     rescue
       on_exception
     end
@@ -907,8 +1105,20 @@ module RightAws
       #  s3.get_link('my_awesome_bucket',key) #=> https://s3.amazonaws.com:443/my_awesome_bucket/asia%2Fcustomers?Signature=QAO...
       #
       # see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/VirtualHosting.html
-    def get_link(bucket, key, expires=nil, headers={})
-      generate_link('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"), expires)
+      #
+      # To specify +response+-* parameters, define them in the response_params hash:
+      #
+      #  s3.get_link('my_awesome_bucket',key,nil,{},{ "response-content-disposition" => "attachment; filename=caf�.png", "response-content-type" => "image/png"})
+      #
+      #    #=> https://s3.amazonaws.com:443/my_awesome_bucket/asia%2Fcustomers?response-content-disposition=attachment%3B%20filename%3Dcaf%25C3%25A9.png&response-content-type=image%2Fpng&Signature=wio...
+      #
+    def get_link(bucket, key, expires=nil, headers={}, response_params={})
+      if response_params.size > 0
+        response_params = '?' + response_params.map { |k, v| "#{k}=#{s3_link_escape(v)}" }.join('&')
+      else
+        response_params = ''
+      end
+      generate_link('GET', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}#{response_params}"), expires)
     rescue
       on_exception
     end
@@ -918,7 +1128,7 @@ module RightAws
       #  s3.head_link('my_awesome_bucket',key) #=> url string
       #
     def head_link(bucket, key, expires=nil,  headers={})
-      generate_link('HEAD', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"), expires)
+      generate_link('HEAD', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}"), expires)
     rescue
       on_exception
     end
@@ -928,7 +1138,7 @@ module RightAws
       #  s3.delete_link('my_awesome_bucket',key) #=> url string
       #
     def delete_link(bucket, key, expires=nil, headers={})
-      generate_link('DELETE', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"), expires)
+      generate_link('DELETE', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}"), expires)
     rescue
       on_exception
     end
@@ -939,7 +1149,7 @@ module RightAws
       #  s3.get_acl_link('my_awesome_bucket',key) #=> url string
       #
     def get_acl_link(bucket, key='', headers={})
-      return generate_link('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}?acl"))
+      return generate_link('GET', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}?acl"))
     rescue
       on_exception
     end
@@ -949,7 +1159,7 @@ module RightAws
       #  s3.put_acl_link('my_awesome_bucket',key) #=> url string
       #
     def put_acl_link(bucket, key='', headers={})
-      return generate_link('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}?acl"))
+      return generate_link('PUT', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}?acl"))
     rescue
       on_exception
     end
@@ -974,6 +1184,23 @@ module RightAws
       on_exception
     end
 
+    class S3DeleteMultipleParser < RightAWSParser # :nodoc:
+      def reset
+        @result = []
+      end
+      def tagstart(name, attributes)
+        @error = {} if name == 'Error'
+      end
+      def tagend(name)
+        case name
+          when 'Key'     then @error[:key]     = @text
+          when 'Code'    then @error[:code]    = @text
+          when 'Message' then @error[:message] = @text
+          when 'Error'   then @result << @error
+        end
+      end
+    end
+
     #-----------------------------------------------------------------
     #      PARSERS:
     #-----------------------------------------------------------------
@@ -988,11 +1215,11 @@ module RightAws
       end
       def tagend(name)
         case name
-          when 'ID'          ; @owner[:owner_id]               = @text
-          when 'DisplayName' ; @owner[:owner_display_name]     = @text
-          when 'Name'        ; @current_bucket[:name]          = @text
-          when 'CreationDate'; @current_bucket[:creation_date] = @text
-          when 'Bucket'      ; @result << @current_bucket.merge(@owner)
+          when 'ID'          then @owner[:owner_id]               = @text
+          when 'DisplayName' then @owner[:owner_display_name]     = @text
+          when 'Name'        then @current_bucket[:name]          = @text
+          when 'CreationDate'then @current_bucket[:creation_date] = @text
+          when 'Bucket'      then @result << @current_bucket.merge(@owner)
         end
       end
     end
@@ -1009,21 +1236,23 @@ module RightAws
       def tagend(name)
         case name
             # service info
-          when 'Name'        ; @service['name']         = @text
-          when 'Prefix'      ; @service['prefix']       = @text
-          when 'Marker'      ; @service['marker']       = @text
-          when 'MaxKeys'     ; @service['max-keys']     = @text
-          when 'Delimiter'   ; @service['delimiter']    = @text
-          when 'IsTruncated' ; @service['is_truncated'] = (@text =~ /false/ ? false : true)
+          when 'Name'        then @service['name']         = @text
+          when 'Prefix'      then @service['prefix']       = @text
+          when 'Marker'      then @service['marker']       = @text
+          when 'MaxKeys'     then @service['max-keys']     = @text
+          when 'Delimiter'   then @service['delimiter']    = @text
+          when 'IsTruncated' then @service['is_truncated'] = (@text =~ /false/ ? false : true)
             # key data
-          when 'Key'         ; @current_key[:key]                = @text
-          when 'LastModified'; @current_key[:last_modified]      = @text
-          when 'ETag'        ; @current_key[:e_tag]              = @text
-          when 'Size'        ; @current_key[:size]               = @text.to_i
-          when 'StorageClass'; @current_key[:storage_class]      = @text
-          when 'ID'          ; @current_key[:owner_id]           = @text
-          when 'DisplayName' ; @current_key[:owner_display_name] = @text
-          when 'Contents'    ; @current_key[:service]            = @service;  @result << @current_key
+          when 'Key'         then @current_key[:key]                = @text
+          when 'LastModified'then @current_key[:last_modified]      = @text
+          when 'ETag'        then @current_key[:e_tag]              = @text
+          when 'Size'        then @current_key[:size]               = @text.to_i
+          when 'StorageClass'then @current_key[:storage_class]      = @text
+          when 'ID'          then @current_key[:owner_id]           = @text
+          when 'DisplayName' then @current_key[:owner_display_name] = @text
+          when 'Contents'
+            @current_key[:service] = @service
+            @result << @current_key
         end
       end
     end
@@ -1045,27 +1274,29 @@ module RightAws
       def tagend(name)
         case name
             # service info
-          when 'Name'        ; @result[:name]         = @text
+          when 'Name'        then @result[:name]         = @text
           # Amazon uses the same tag for the search prefix and for the entries
             # in common prefix...so use our simple flag to see which element
             # we are parsing
-          when 'Prefix'      ; @in_common_prefixes ? @common_prefixes << @text : @result[:prefix] = @text
-          when 'Marker'      ; @result[:marker]       = @text
-          when 'MaxKeys'     ; @result[:max_keys]     = @text
-          when 'Delimiter'   ; @result[:delimiter]    = @text
-          when 'IsTruncated' ; @result[:is_truncated] = (@text =~ /false/ ? false : true)
-          when 'NextMarker'  ; @result[:next_marker]  = @text
+          when 'Prefix'      then @in_common_prefixes ? @common_prefixes << @text : @result[:prefix] = @text
+          when 'Marker'      then @result[:marker]       = @text
+          when 'MaxKeys'     then @result[:max_keys]     = @text
+          when 'Delimiter'   then @result[:delimiter]    = @text
+          when 'IsTruncated' then @result[:is_truncated] = (@text =~ /false/ ? false : true)
+          when 'NextMarker'  then @result[:next_marker]  = @text
             # key data
-          when 'Key'         ; @current_key[:key]                = @text
-          when 'LastModified'; @current_key[:last_modified]      = @text
-          when 'ETag'        ; @current_key[:e_tag]              = @text
-          when 'Size'        ; @current_key[:size]               = @text.to_i
-          when 'StorageClass'; @current_key[:storage_class]      = @text
-          when 'ID'          ; @current_key[:owner_id]           = @text
-          when 'DisplayName' ; @current_key[:owner_display_name] = @text
-          when 'Contents'    ; @result[:contents] << @current_key
+          when 'Key'         then @current_key[:key]                = @text
+          when 'LastModified'then @current_key[:last_modified]      = @text
+          when 'ETag'        then @current_key[:e_tag]              = @text
+          when 'Size'        then @current_key[:size]               = @text.to_i
+          when 'StorageClass'then @current_key[:storage_class]      = @text
+          when 'ID'          then @current_key[:owner_id]           = @text
+          when 'DisplayName' then @current_key[:owner_display_name] = @text
+          when 'Contents'    then @result[:contents] << @current_key
             # Common Prefix stuff
-          when 'CommonPrefixes' ; @result[:common_prefixes] = @common_prefixes; @in_common_prefixes = false
+          when 'CommonPrefixes' 
+            @result[:common_prefixes] = @common_prefixes
+            @in_common_prefixes = false
         end
       end
     end
@@ -1140,8 +1371,8 @@ module RightAws
       end
       def tagend(name)
         case name
-        when 'LastModified' : @result[:last_modified] = @text
-        when 'ETag'         : @result[:e_tag]         = @text
+        when 'LastModified' then @result[:last_modified] = @text
+        when 'ETag'         then @result[:e_tag]         = @text
         end
       end
     end
@@ -1158,7 +1389,7 @@ module RightAws
       def headers_to_string(headers)
         result = {}
         headers.each do |key, value|
-          value       = value.to_s if value.is_a?(Array) && value.size<2
+          value       = value.first if value.is_a?(Array) && value.size<2
           result[key] = value
         end
         result