right_aws 1.4.3 → 1.5.0

data/lib/right_aws.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2007 RightScale Inc
+# Copyright (c) 2007-2008 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -43,13 +43,14 @@ require 's3/right_s3_interface'
 require 's3/right_s3'
 require 'sqs/right_sqs_interface'
 require 'sqs/right_sqs'
+require 'sdb/right_sdb_interface'
 
 
 module RightAws #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 1
-    MINOR = 4
-    TINY  = 3
+    MINOR = 5
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/s3/right_s3.rb

@@ -34,7 +34,19 @@ module RightAws
   #
   # Error handling: all operations raise an RightAws::AwsError in case
   # of problems. Note that transient errors are automatically retried.
-    
+  #
+  # It is a good way to use domain naming style getting a name for the buckets. 
+  # See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingBucket.html
+  # about the naming convention for the buckets. This case they can be accessed using a virtual domains.
+  # 
+  # Let assume you have 3 buckets: 'awesome-bucket', 'awesome_bucket' and 'AWEsomE-bucket'.
+  # The first ones objects can be accessed as: http:// awesome-bucket.s3.amazonaws.com/key/object
+  # 
+  # But the rest have to be accessed as:
+  # http:// s3.amazonaws.com/awesome_bucket/key/object and  http:// s3.amazonaws.com/AWEsomE-bucket/key/object
+  #  
+  # See: http://docs.amazonwebservices.com/AmazonS3/2006-03-01/VirtualHosting.html for better explanation.
+  #
   class S3
     attr_reader :interface
     
@@ -60,22 +72,24 @@ module RightAws
     
     # Retrieve an individual bucket.
     # If the bucket does not exist and +create+ is set, a new bucket
-    # is created on S3.  The +create+ parameter has no effect if
-    # the bucket alrady exists. 
+    # is created on S3. Launching this method with +create+=+true+ may
+    # affect on the bucket's ACL if the bucket already exists.
     # Returns a RightAws::S3::Bucket instance or +nil+ if the bucket does not exist 
     # and +create+ is not set.
     #
     #  s3 = RightAws::S3.new(aws_access_key_id, aws_secret_access_key)
-    #  bucket1 = s3.bucket('my_awesome_bucket')
+    #  bucket1 = s3.bucket('my_awesome_bucket_1')
     #  bucket1.keys  #=> exception here if the bucket does not exists
     #   ...
-    #  bucket2 = s3.bucket('my_awesome_bucket', true)
+    #  bucket2 = s3.bucket('my_awesome_bucket_2', true)
     #  bucket2.keys  #=> list of keys
-    #
+    #  # create a bucket at the European location with public read access
+    #  bucket3 = s3.bucket('my-awesome-bucket-3', true, 'public-read', :location => :eu)
+    #  
     #  see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAccessPolicy.html
     #  (section: Canned Access Policies)
     #
-    def bucket(name, create=true, perms=nil, headers={})
+    def bucket(name, create=false, perms=nil, headers={})
       headers['x-amz-acl'] = perms if perms
       @interface.create_bucket(name, headers) if create
       buckets.each { |bucket| return bucket if bucket.name == name }
@@ -86,28 +100,33 @@ module RightAws
     class Bucket
       attr_reader :s3, :name, :owner, :creation_date
       
-        # Create a Bucket instance.
+      # Create a Bucket instance.
       # If the bucket does not exist and +create+ is set, a new bucket
-      # is created on S3.  The +create+ parameter has no effect if
-      # the bucket alrady exists. 
+      # is created on S3. Launching this method with +create+=+true+ may
+      # affect on the bucket's ACL if the bucket already exists.
       # Returns Bucket instance or +nil+ if the bucket does not exist 
       # and +create+ is not set.
-        #
-        #  s3 = RightAws::S3.new(aws_access_key_id, aws_secret_access_key)
-        #   ...
-        #  bucket1 = RightAws::S3::Bucket.create(s3, 'my_awesome_bucket')
-        #  bucket1.keys  #=> exception here if the bucket does not exists
-        #   ...
-        #  bucket2 = RightAws::S3::Bucket.create(s3, 'my_awesome_bucket', true)
-        #  bucket2.keys  #=> list of keys
-        #
-      def self.create(s3, name, create=true, perms=nil, headers={}) 
+      #
+      #  s3 = RightAws::S3.new(aws_access_key_id, aws_secret_access_key)
+      #   ...
+      #  bucket1 = RightAws::S3::Bucket.create(s3, 'my_awesome_bucket_1')
+      #  bucket1.keys  #=> exception here if the bucket does not exists
+      #   ...
+      #  bucket2 = RightAws::S3::Bucket.create(s3, 'my_awesome_bucket_2', true)
+      #  bucket2.keys  #=> list of keys
+      #  # create a bucket at the European location with public read access
+      #  bucket3 = RightAws::S3::Bucket.create(s3,'my-awesome-bucket-3', true, 'public-read', :location => :eu)
+      #  
+      #  see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAccessPolicy.html
+      #  (section: Canned Access Policies)
+      #
+      def self.create(s3, name, create=false, perms=nil, headers={}) 
         s3.bucket(name, create, perms, headers)
       end
 
 
         # Create a bucket instance. In normal use this method should
-	# not be called directly.
+        # not be called directly.
         # Use RightAws::S3::Bucket.create or RightAws::S3.bucket instead. 
       def initialize(s3, name, creation_date=nil, owner=nil)
         @s3    = s3
@@ -137,6 +156,11 @@ module RightAws
         params = @s3.interface.params
         "#{params[:protocol]}://#{params[:server]}:#{params[:port]}/#{full_name}"
       end
+      
+        # Returns the bucket location
+      def location
+        @location ||= @s3.interface.bucket_location(@name)
+      end
 
         # Retrieve a group of keys from Amazon. 
         # +options+ is a hash: { 'prefix'=>'', 'marker'=>'', 'max-keys'=>5, 'delimiter'=>'' }). 
@@ -569,13 +593,15 @@ module RightAws
         # Create a new Grantee instance. 
         # Grantee +id+ must exist on S3. If +action+ == :refresh, then retrieve
         # permissions from S3 and update @perms. If +action+ == :apply, then apply
-        # perms to +thing+ at S3. The default action is :refresh.
+        # perms to +thing+ at S3. If +action+ == :apply_and_refresh then it performs.
+        # both the actions. This is used for the new grantees that had no perms to 
+        # this thing before. The default action is :refresh.
         #
         #  bucket = s3.bucket('my_awesome_bucket', true, 'public-read')
         #  grantee1 = RightAws::S3::Grantee.new(bucket, 'a123b...223c', FULL_CONTROL)
         #    ...
         #  grantee2 = RightAws::S3::Grantee.new(bucket, 'abcde...asdf', [FULL_CONTROL, READ], :apply)
-        #
+        #  grantee3 = RightAws::S3::Grantee.new(bucket, 'aaaaa...aaaa', 'READ', :apply_and_refresh)  
         #
       def initialize(thing, id, perms=[], action=:refresh, name=nil)
         @thing = thing
@@ -583,11 +609,20 @@ module RightAws
         @name  = name
         @perms = perms.to_a
         case action
-          when :apply;   apply
-          when :refresh; refresh
+          when :apply:             apply
+          when :refresh:           refresh
+          when :apply_and_refresh: apply; refresh
         end
       end
       
+        # Return +true+ if the grantee has any permissions to the thing.
+      def exists?
+        self.class.grantees(@thing).each do |grantee|
+          return true if @id == grantee.id
+        end
+        false
+      end
+      
         # Return Grantee type (+String+): "Group" or "CanonicalUser".
       def type
         @id[/^http:/] ? "Group" : "CanonicalUser"
@@ -603,11 +638,16 @@ module RightAws
         # See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingPermissions.html .
         # Returns +true+. 
         #
-        #  grantee.grant('FULL_CONTROL') #=> true
-        #
-      def grant(permission)
-        return true if @perms.include?(permission)
-        @perms += permission.to_a
+        #  grantee.grant('FULL_CONTROL')                  #=> true
+        #  grantee.grant('FULL_CONTROL','WRITE','READ')   #=> true
+        #  grantee.grant(['WRITE_ACP','READ','READ_ACP']) #=> true
+        #  
+      def grant(*permissions)
+        permissions.flatten!
+        old_perms = @perms.dup
+        @perms   += permissions
+        @perms.uniq!
+        return true if @perms == old_perms
         apply
       end
       
@@ -617,11 +657,16 @@ module RightAws
         # Default value is 'FULL_CONTROL'. 
         # Returns +true+.
         #
-        #  grantee.revoke #=> true
+        #  grantee.revoke('READ')                   #=> true
+        #  grantee.revoke('FULL_CONTROL','WRITE')   #=> true
+        #  grantee.revoke(['READ_ACP','WRITE_ACP']) #=> true
         #
-      def revoke(permission)
-        return true unless @perms.include?(permission)
-        @perms -= permission.to_a
+      def revoke(*permissions)
+        permissions.flatten!
+        old_perms = @perms.dup
+        @perms   -= permissions
+        @perms.uniq!
+        return true if @perms == old_perms
         apply
       end
      
@@ -664,10 +709,13 @@ module RightAws
         #  grantee.apply #=> true
         #
       def apply
+        @perms.uniq!
         owner, grantees = self.class.owner_and_grantees(@thing)
-        grantees.map! do |grantee|
-          grantee.id == @id ? self : grantee
-        end
+        # walk through all the grantees and replace the data for the current one and ...
+        grantees.map! { |grantee| grantee.id == @id ? self : grantee }
+        # ... if this grantee is not known - add this bad boy to a list
+        grantees << self unless grantees.include?(self)
+        # set permissions
         self.class.put_acl(@thing, owner, grantees)
       end
 

data/lib/s3/right_s3_interface.rb

@@ -24,6 +24,9 @@
 module RightAws
 
   class S3Interface < RightAwsBase
+    
+    USE_100_CONTINUE_PUT_SIZE = 1_000_000
+    
     include RightAwsBaseInterface
     
     DEFAULT_HOST           = 's3.amazonaws.com'
@@ -88,15 +91,44 @@ module RightAws
         # ignore everything after the question mark...
       out_string << path.gsub(/\?.*$/, '')
        # ...unless there is an acl or torrent parameter
-      out_string << '?acl'    if path[/[&?]acl($|&|=)/]
-      out_string << '?torrent'if path[/[&?]torrent($|&|=)/]
+      out_string << '?acl'      if path[/[&?]acl($|&|=)/]
+      out_string << '?torrent'  if path[/[&?]torrent($|&|=)/]
+      out_string << '?location' if path[/[&?]location($|&|=)/]
+#      out_string << '?logging'  if path[/[&?]logging($|&|=)/]  # this one is beta, no support for now
       out_string
     end
 
+    def is_dns_bucket?(bucket_name)
+      bucket_name = bucket_name.to_s
+      return nil unless (3..63) === bucket_name.size
+      bucket_name.split('.').each do |component|
+        return nil unless component[/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/]
+      end
+      true
+    end
+    
       # Generates request hash for REST API.
+      # Assumes that headers[:url] is URL encoded (use CGI::escape)
     def generate_rest_request(method, headers)  # :nodoc:
-      path = headers[:url]
-      path = "/#{path}" unless path[/^\//]
+      # default server to use
+      server = @params[:server]
+      # fix path
+      path_to_sign = headers[:url]
+      path_to_sign = "/#{path_to_sign}" unless path_to_sign[/^\//]
+      # extract bucket name and check it's dns compartibility
+      path_to_sign[%r{^/([a-z0-9._-]*)(/[^?]*)?(\?.+)?}i]
+      bucket_name, key_path, params_list = $1, $2, $3
+      # select request model
+      if is_dns_bucket?(bucket_name)
+        # add backet to a server name
+        server = "#{bucket_name}.#{server}"
+        # remove bucket from the path
+        path = "#{key_path || '/'}#{params_list}"
+        # refactor the path (add '/' before params_list if the key is empty)
+        path_to_sign = "/#{bucket_name}#{path}"
+      else
+        path = path_to_sign
+      end
       data = headers[:data]
         # remove unset(==optional) and symbolyc keys
       headers.each{ |key, value| headers.delete(key) if (value.nil? || key.is_a?(Symbol)) }
@@ -104,22 +136,22 @@ module RightAws
       headers['content-type'] ||= ''
       headers['date']           = Time.now.httpdate
         # create request
-      request      = "Net::HTTP::#{method.capitalize}".constantize.new(URI::escape(CGI::unescape(path)))
+      request      = "Net::HTTP::#{method.capitalize}".constantize.new(path)
       request.body = data if data
         # set request headers and meta headers
       headers.each      { |key, value| request[key.to_s] = value }
         #generate auth strings
-      auth_string = canonical_string(request.method, request.path, request.to_hash)
+      auth_string = canonical_string(request.method, path_to_sign, request.to_hash)
       signature   = AwsUtils::sign(@aws_secret_access_key, auth_string)
         # set other headers
       request['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}"
         # prepare output hash
       { :request  => request, 
-        :server   => @params[:server],
+        :server   => server,
         :port     => @params[:port],
         :protocol => @params[:protocol] }
-    end
-
+      end
+      
       # Sends request to Amazon and parses the response.
       # Raises AwsError if any banana happened.
     def request_info(request, parser, &block) # :nodoc:
@@ -146,11 +178,34 @@ module RightAws
     
       # Creates new bucket. Returns +true+ or an exception.
       #
-      #  s3.create_bucket('my_awesome_bucket') #=> true
+      #  # create a bucket at American server
+      #  s3.create_bucket('my-awesome-bucket-us') #=> true
+      #  # create a bucket at European server
+      #  s3.create_bucket('my-awesome-bucket-eu', :location => :eu) #=> true
       #
     def create_bucket(bucket, headers={})
-      req_hash = generate_rest_request('PUT', headers.merge(:url=>bucket))
+      data = nil
+      unless headers[:location].blank?
+        data = "<CreateBucketConfiguration><LocationConstraint>#{headers[:location].to_s.upcase}</LocationConstraint></CreateBucketConfiguration>"
+      end
+      req_hash = generate_rest_request('PUT', headers.merge(:url=>bucket, :data => data))
       request_info(req_hash, S3TrueParser.new)
+    rescue Exception => e
+        # if the bucket exists AWS returns an error for the location constraint interface. Drop it
+      e.is_a?(RightAws::AwsError) && e.message.include?('BucketAlreadyOwnedByYou') ? true  : on_exception
+    end
+    
+      # Retrieve bucket location
+      # 
+      #  s3.create_bucket('my-awesome-bucket-us')        #=> true
+      #  puts s3.bucket_location('my-awesome-bucket-us') #=> '' (Amazon's default value assumed)
+      #
+      #  s3.create_bucket('my-awesome-bucket-eu', :location => :eu) #=> true
+      #  puts s3.bucket_location('my-awesome-bucket-eu')            #=> 'EU'
+      #
+    def bucket_location(bucket, headers={})
+      req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}?location"))
+      request_info(req_hash, S3BucketLocationParser.new)
     rescue
       on_exception
     end
@@ -220,7 +275,7 @@ module RightAws
     #     ]
     #   }
     def incrementally_list_bucket(bucket, options={}, headers={}, &block)
-      internal_options = options.dup
+      internal_options = options.symbolize_keys
       begin 
         internal_bucket = bucket.dup
         internal_bucket  += '?'+internal_options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless internal_options.blank?
@@ -229,11 +284,11 @@ module RightAws
         there_are_more_keys = response[:is_truncated]
         if(there_are_more_keys)
           if(response[:next_marker])
-            internal_options['marker'] = response[:next_marker]
+            internal_options[:marker] = response[:next_marker]
           else 
-            internal_options['marker'] = response[:contents].last[:key]
+            internal_options[:marker] = response[:contents].last[:key]
           end
-          internal_options['max-keys'] ? (internal_options['max-keys'] -= response[:contents].length) : nil 
+          internal_options[:'max-keys'] ? (internal_options[:'max-keys'] -= response[:contents].length) : nil 
         end
         yield response
       end while there_are_more_keys 
@@ -242,12 +297,49 @@ module RightAws
       on_exception
     end
     
-      # Saves object to Amazon. Returns +true+  or an exception. 
-      # Any header starting with AMAZON_METADATA_PREFIX is considered user metadata. It will be stored with the object and returned when you retrieve the object. The total size of the HTTP request, not including the body, must be less than 4 KB.
+      # Saves object to Amazon. Returns +true+  or an exception.
+      # Any header starting with AMAZON_METADATA_PREFIX is considered
+      # user metadata. It will be stored with the object and returned
+      # when you retrieve the object. The total size of the HTTP
+      # request, not including the body, must be less than 4 KB.
+      #
+      #  s3.put('my_awesome_bucket', 'log/current/1.log', 'Ola-la!', 'x-amz-meta-family'=>'Woho556!') #=> true
+      #
+      # This method is capable of 'streaming' uploads; that is, it can upload
+      # data from a file or other IO object without first reading all the data
+      # into memory.  This is most useful for large PUTs - it is difficult to read
+      # a 2 GB file entirely into memory before sending it to S3.
+      # To stream an upload, pass an object that responds to 'read' (like the read
+      # method of IO) and to either 'lstat' or 'size'.  For files, this means
+      # streaming is enabled by simply making the call:
+      #
+      #  s3.put(bucket_name, 'S3keyname.forthisfile',  File.open('localfilename.dat'))
+      #
+      # If the IO object you wish to stream from responds to the read method but
+      # doesn't implement lstat or size, you can extend the object dynamically
+      # to implement these methods, or define your own class which defines these
+      # methods.  Be sure that your class returns 'nil' from read() after having
+      # read 'size' bytes. Otherwise S3 will drop the socket after
+      # 'Content-Length' bytes have been uploaded, and HttpConnection will
+      # interpret this as an error. 
       #    
-      #  s3.put('my_awesome_bucket', 'log/curent/1.log', 'Ola-la!', 'x-amz-meta-family'=>'Woho556!') #=> true
+      # This method now supports very large PUTs, where very large
+      # is > 2 GB. 
+      # 
+      # For Win32 users: Files and IO objects should be opened in binary mode.  If
+      # a text mode IO object is passed to PUT, it will be converted to binary
+      # mode.
       #
     def put(bucket, key, data=nil, headers={})
+      # On Windows, if someone opens a file in text mode, we must reset it so
+      # to binary mode for streaming to work properly
+      if(data.respond_to?(:binmode))
+        data.binmode
+      end
+      if (data.respond_to?(:lstat) && data.lstat.size >= USE_100_CONTINUE_PUT_SIZE) ||
+         (data.respond_to?(:size)  && data.size       >= USE_100_CONTINUE_PUT_SIZE)
+        headers['expect'] = '100-continue'
+      end
       req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}", :data=>data))
       request_info(req_hash, S3TrueParser.new)
     rescue
@@ -470,8 +562,25 @@ module RightAws
 
       # Generates link for QUERY API
     def generate_link(method, headers={}, expires=nil) #:nodoc:
-      path = headers[:url]
-      path = "/#{path}" unless path[/^\//]
+      # default server to use
+      server = @params[:server]
+      # fix path
+      path_to_sign = headers[:url]
+      path_to_sign = "/#{path_to_sign}" unless path_to_sign[/^\//]
+      # extract bucket name and check it's dns compartibility
+      path_to_sign[%r{^/([a-z0-9._-]*)(/[^?]*)?(\?.+)?}i]
+      bucket_name, key_path, params_list = $1, $2, $3
+      # select request model
+      if is_dns_bucket?(bucket_name)
+        # add backet to a server name
+        server = "#{bucket_name}.#{server}"
+        # remove bucket from the path
+        path = "#{key_path || '/'}#{params_list}"
+        # refactor the path (add '/' before params_list if the key is empty)
+        path_to_sign = "/#{bucket_name}#{path}"
+      else
+        path = path_to_sign
+      end
        # expiration time
       expires ||= DEFAULT_EXPIRES_AFTER
       expires   = Time.now.utc.since(expires) if expires.is_a?(Fixnum) && (expires<1.year)
@@ -479,12 +588,12 @@ module RightAws
         # remove unset(==optional) and symbolyc keys
       headers.each{ |key, value| headers.delete(key) if (value.nil? || key.is_a?(Symbol)) }
         #generate auth strings
-      auth_string = canonical_string(method, path, headers, expires)
+      auth_string = canonical_string(method, path_to_sign, headers, expires)
       signature   = CGI::escape(Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new("sha1"), @aws_secret_access_key, auth_string)).strip)
         # path building
       addon = "Signature=#{signature}&Expires=#{expires}&AWSAccessKeyId=#{@aws_access_key_id}"
       path += path[/\?/] ? "&#{addon}" : "?#{addon}"
-      "#{@params[:protocol]}://#{@params[:server]}:#{@params[:port]}#{path}"
+      "#{@params[:protocol]}://#{server}:#{@params[:port]}#{path}"
     rescue
       on_exception
     end
@@ -707,6 +816,48 @@ module RightAws
       end
     end
 
+    class S3BucketLocationParser < RightAWSParser # :nodoc:
+      def reset
+        @result = ''
+      end
+      def tagend(name)
+        @result = @text if name == 'LocationConstraint'
+      end
+    end
+
+    class S3AclParser < RightAWSParser  # :nodoc:
+      def reset
+        @result          = {:grantees=>[], :owner=>{}}
+        @current_grantee = {}
+      end
+      def tagstart(name, attributes)
+        @current_grantee = { :attributes => attributes } if name=='Grantee'
+      end
+      def tagend(name)
+        case name
+            # service info
+          when 'ID'
+            if @xmlpath == 'AccessControlPolicy/Owner'
+              @result[:owner][:id] = @text
+            else
+              @current_grantee[:id] = @text
+            end
+          when 'DisplayName'
+            if @xmlpath == 'AccessControlPolicy/Owner'
+              @result[:owner][:display_name] = @text
+            else
+              @current_grantee[:display_name] = @text
+            end
+          when 'URI'
+            @current_grantee[:uri] = @text
+          when 'Permission'
+            @current_grantee[:permissions] = @text
+          when 'Grant'
+            @result[:grantees] << @current_grantee
+        end
+      end
+    end
+    
     #-----------------------------------------------------------------
     #      PARSERS: Non XML
     #-----------------------------------------------------------------
@@ -746,40 +897,6 @@ module RightAws
         @result = headers_to_string(response.to_hash)
       end
     end
-
-
-    class S3AclParser < RightAWSParser  # :nodoc:
-      def reset
-        @result          = {:grantees=>[], :owner=>{}}
-        @current_grantee = {}
-      end
-      def tagstart(name, attributes)
-        @current_grantee = { :attributes => attributes } if name=='Grantee'
-      end
-      def tagend(name)
-        case name
-            # service info
-          when 'ID'
-            if @xmlpath == 'AccessControlPolicy/Owner'
-              @result[:owner][:id] = @text
-            else
-              @current_grantee[:id] = @text
-            end
-          when 'DisplayName'
-            if @xmlpath == 'AccessControlPolicy/Owner'
-              @result[:owner][:display_name] = @text
-            else
-              @current_grantee[:display_name] = @text
-            end
-          when 'URI'
-            @current_grantee[:uri] = @text
-          when 'Permission'
-            @current_grantee[:permissions] = @text
-          when 'Grant'
-            @result[:grantees] << @current_grantee
-        end
-      end
-    end
     
   end