croaker-aws-s3 0.5.2.20090127001

data/lib/aws/s3/authentication.rb

@@ -0,0 +1,222 @@
+module AWS
+  module S3    
+    # All authentication is taken care of for you by the AWS::S3 library. None the less, some details of the two types
+    # of authentication and when they are used may be of interest to some.
+    #
+    # === Header based authentication
+    #
+    # Header based authentication is achieved by setting a special <tt>Authorization</tt> header whose value 
+    # is formatted like so:
+    #
+    #   "AWS #{access_key_id}:#{encoded_canonical}"
+    #
+    # The <tt>access_key_id</tt> is the public key that is assigned by Amazon for a given account which you use when
+    # establishing your initial connection. The <tt>encoded_canonical</tt> is computed according to rules layed out 
+    # by Amazon which we will describe presently.
+    #
+    # ==== Generating the encoded canonical string
+    #
+    # The "canonical string", generated by the CanonicalString class, is computed by collecting the current request method, 
+    # a set of significant headers of the current request, and the current request path into a string. 
+    # That canonical string is then encrypted with the <tt>secret_access_key</tt> assigned by Amazon. The resulting encrypted canonical 
+    # string is then base 64 encoded.
+    #
+    # === Query string based authentication
+    #
+    # When accessing a restricted object from the browser, you can authenticate via the query string, by setting the following parameters:
+    #
+    #   "AWSAccessKeyId=#{access_key_id}&Expires=#{expires}&Signature=#{encoded_canonical}"
+    # 
+    # The QueryString class is responsible for generating the appropriate parameters for authentication via the
+    # query string.
+    #
+    # The <tt>access_key_id</tt> and <tt>encoded_canonical</tt> are the same as described in the Header based authentication section. 
+    # The <tt>expires</tt> value dictates for how long the current url is valid (by default, it will expire in 5 minutes). Expiration can be specified
+    # either by an absolute time (expressed in seconds since the epoch), or in relative time (in number of seconds from now).
+    # Details of how to customize the expiration of the url are provided in the documentation for the QueryString class.
+    #
+    # All requests made by this library use header authentication. When a query string authenticated url is needed, 
+    # the S3Object#url method will include the appropriate query string parameters.
+    #
+    # === Full authentication specification
+    #
+    # The full specification of the authentication protocol can be found at
+    # http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html    
+    class Authentication
+      constant :AMAZON_HEADER_PREFIX, 'x-amz-'
+      
+      # Signature is the abstract super class for the Header and QueryString authentication methods. It does the job
+      # of computing the canonical_string using the CanonicalString class as well as encoding the canonical string. The subclasses
+      # parameterize these computations and arrange them in a string form appropriate to how they are used, in one case a http request
+      # header value, and in the other case key/value query string parameter pairs.
+      class Signature < String #:nodoc:
+        attr_reader :request, :access_key_id, :secret_access_key, :options
+  
+        def initialize(request, access_key_id, secret_access_key, options = {})
+          super()
+          @request, @access_key_id, @secret_access_key = request, access_key_id, secret_access_key
+          @options = options
+        end
+  
+        private
+    
+          def canonical_string            
+            options = {}
+            options[:expires] = expires if expires?
+            options[:bucket_address] = @options[:bucket_address] if @options[:bucket_address]
+            CanonicalString.new(request, options)
+          end
+          memoized :canonical_string
+    
+          def encoded_canonical
+            digest   = OpenSSL::Digest::Digest.new('sha1')
+            b64_hmac = Base64.encode64(OpenSSL::HMAC.digest(digest, secret_access_key, canonical_string)).strip
+            url_encode? ? CGI.escape(b64_hmac) : b64_hmac
+          end
+          
+          def url_encode?
+            !@options[:url_encode].nil?
+          end
+          
+          def expires?
+            is_a? QueryString
+          end
+          
+          def date
+            request['date'].to_s.strip.empty? ? Time.now : Time.parse(request['date'])
+          end
+      end
+      
+      # Provides header authentication by computing the value of the Authorization header. More details about the
+      # various authentication schemes can be found in the docs for its containing module, Authentication.
+      class Header < Signature #:nodoc:
+        def initialize(*args)
+          super
+          self << "AWS #{access_key_id}:#{encoded_canonical}"
+        end
+      end
+      
+      # Provides query string authentication by computing the three authorization parameters: AWSAccessKeyId, Expires and Signature.
+      # More details about the various authentication schemes can be found in the docs for its containing module, Authentication.
+      class QueryString < Signature #:nodoc:
+        constant :DEFAULT_EXPIRY, 300 # 5 minutes
+        def initialize(*args)
+          super
+          options[:url_encode] = true
+          self << build
+        end
+        
+        private
+          
+          # Will return one of three values, in the following order of precedence:
+          #
+          #   1) Seconds since the epoch explicitly passed in the +:expires+ option
+          #   2) The current time in seconds since the epoch plus the number of seconds passed in
+          #      the +:expires_in+ option
+          #   3) The current time in seconds since the epoch plus the default number of seconds (60 seconds)
+          def expires
+            return options[:expires] if options[:expires]
+            date.to_i + expires_in
+          end
+          
+          def expires_in
+            options.has_key?(:expires_in) ? Integer(options[:expires_in]) : DEFAULT_EXPIRY
+          end
+          
+          # Keep in alphabetical order
+          def build
+            "AWSAccessKeyId=#{access_key_id}&Expires=#{expires}&Signature=#{encoded_canonical}"
+          end
+      end
+      
+      # The CanonicalString is used to generate an encrypted signature, signed with your secrect access key. It is composed of 
+      # data related to the given request for which it provides authentication. This data includes the request method, request headers,
+      # and the request path. Both Header and QueryString use it to generate their signature.
+      class CanonicalString < String #:nodoc:
+        class << self
+          def default_headers
+            %w(content-type content-md5)
+          end
+
+          def interesting_headers
+            ['content-md5', 'content-type', 'date', amazon_header_prefix]
+          end
+          
+          def amazon_header_prefix
+            /^#{AMAZON_HEADER_PREFIX}/io
+          end
+        end
+        
+        attr_reader :request, :headers
+        
+        def initialize(request, options = {})
+          super()
+          @request = request
+          @headers = {}
+          @options = options
+          
+          # CHANGED: Reflect the bucket adressing changes
+          request['Host'] = options[:bucket_address] ? "#{options[:bucket_address]}.#{DEFAULT_HOST}" : DEFAULT_HOST
+          build
+        end
+    
+        private
+          def build
+            self << "#{request.method}\n"
+            ensure_date_is_valid
+            
+            initialize_headers
+            set_expiry!
+        
+            headers.sort_by {|k, _| k}.each do |key, value|
+              value = value.to_s.strip
+              self << (key =~ self.class.amazon_header_prefix ? "#{key}:#{value}" : value)
+              self << "\n"
+            end
+            self << path
+          end
+      
+          def initialize_headers
+            identify_interesting_headers
+            set_default_headers
+          end
+          
+          def set_expiry!
+            self.headers['date'] = @options[:expires] if @options[:expires]
+          end
+          
+          def ensure_date_is_valid
+            request['Date'] ||= Time.now.httpdate
+          end
+
+          def identify_interesting_headers
+            request.each do |key, value|
+              key = key.downcase # Can't modify frozen string so no bang
+              if self.class.interesting_headers.any? {|header| header === key}
+                self.headers[key] = value.to_s.strip
+              end
+            end
+          end
+
+          def set_default_headers
+            self.class.default_headers.each do |header|
+              self.headers[header] ||= ''
+            end
+          end
+
+          def path
+            path_with_bucket = "/#{@options[:bucket_address]}#{only_path}" if @options[:bucket_address]
+            [path_with_bucket || only_path, extract_significant_parameter].compact.join('?')
+          end
+          
+          def extract_significant_parameter
+            request.path[/[&?](acl|torrent|logging)(?:&|=|$)/, 1]
+          end
+          
+          def only_path
+            request.path[/^[^?]*/]
+          end
+      end
+    end
+  end
+end

data/lib/aws/s3/base.rb

@@ -0,0 +1,257 @@
+module AWS #:nodoc:
+  # AWS::S3 is a Ruby library for Amazon's Simple Storage Service's REST API (http://aws.amazon.com/s3).
+  # Full documentation of the currently supported API can be found at http://docs.amazonwebservices.com/AmazonS3/2006-03-01.
+  # 
+  # == Getting started
+  # 
+  # To get started you need to require 'aws/s3':
+  # 
+  #   % irb -rubygems
+  #   irb(main):001:0> require 'aws/s3'
+  #   # => true
+  # 
+  # The AWS::S3 library ships with an interactive shell called <tt>s3sh</tt>. From within it, you have access to all the operations the library exposes from the command line.
+  # 
+  #   % s3sh
+  #   >> Version
+  # 
+  # Before you can do anything, you must establish a connection using Base.establish_connection!.  A basic connection would look something like this:
+  # 
+  #   AWS::S3::Base.establish_connection!(
+  #     :access_key_id     => 'abc', 
+  #     :secret_access_key => '123'
+  #   )
+  # 
+  # The minimum connection options that you must specify are your access key id and your secret access key.
+  # 
+  # (If you don't already have your access keys, all you need to sign up for the S3 service is an account at Amazon. You can sign up for S3 and get access keys by visiting http://aws.amazon.com/s3.)
+  # 
+  # For convenience, if you set two special environment variables with the value of your access keys, the console will automatically create a default connection for you. For example:
+  # 
+  #   % cat .amazon_keys
+  #   export AMAZON_ACCESS_KEY_ID='abcdefghijklmnop'
+  #   export AMAZON_SECRET_ACCESS_KEY='1234567891012345'
+  # 
+  # Then load it in your shell's rc file.
+  # 
+  #   % cat .zshrc
+  #   if [[ -f "$HOME/.amazon_keys" ]]; then
+  #     source "$HOME/.amazon_keys";
+  #   fi
+  # 
+  # See more connection details at AWS::S3::Connection::Management::ClassMethods.
+  module S3
+    constant :DEFAULT_HOST, 's3.amazonaws.com'
+    
+    # AWS::S3::Base is the abstract super class of all classes who make requests against S3, such as the built in
+    # Service, Bucket and S3Object classes. It provides methods for making requests, inferring or setting response classes,
+    # processing request options, and accessing attributes from S3's response data.
+    #
+    # Establishing a connection with the Base class is the entry point to using the library:
+    #
+    #   AWS::S3::Base.establish_connection!(:access_key_id => '...', :secret_access_key => '...')
+    #
+    # The <tt>:access_key_id</tt> and <tt>:secret_access_key</tt> are the two required connection options. More 
+    # details can be found in the docs for Connection::Management::ClassMethods.
+    #
+    # Extensive examples can be found in the README[link:files/README.html].
+    class Base        
+      class << self
+        
+        # Wraps the current connection's request method and picks the appropriate response class to wrap the response in.
+        # If the response is an error, it will raise that error as an exception. All such exceptions can be caught by rescuing
+        # their superclass, the ResponseError exception class.
+        #
+        # It is unlikely that you would call this method directly. Subclasses of Base have convenience methods for each http request verb
+        # that wrap calls to request.
+        def request(verb, path, options = {}, body = nil, attempts = 0, &block)
+          Service.response = nil
+          process_options!(options, verb)
+                    
+          # This has been changed so that buckets are addressed via the subdomain rather than the path. It is now possible to connect to european buckets as well.
+          # One drawback is that there are no persistent connections anymore since we have to reconnect everytime a new bucket is adressed.
+          bucket_name = AWS::S3.remove_or_extract_bucket_from_path(path, true)
+            
+          if bucket_name
+            establish_connection!(:server => "#{bucket_name}.#{DEFAULT_HOST}") unless connection.is_for_bucket?(bucket_name)
+            path = AWS::S3.remove_or_extract_bucket_from_path(path)   
+          else
+            establish_connection!(:server => DEFAULT_HOST) if connection.has_subdomain?
+          end 
+                
+          response = response_class.new(connection.request(verb, path, options, body, attempts, &block))
+          Service.response = response
+
+          Error::Response.new(response.response).error.raise if response.error?
+          response
+        # Once in a while, a request to S3 returns an internal error. A glitch in the matrix I presume. Since these 
+        # errors are few and far between the request method will rescue InternalErrors the first three times they encouter them
+        # and will retry the request again. Most of the time the second attempt will work.
+        rescue *retry_exceptions
+          attempts == 3 ? raise : (attempts += 1; retry)
+        end
+
+        [:get, :post, :put, :delete, :head].each do |verb|
+          class_eval(<<-EVAL, __FILE__, __LINE__)
+            def #{verb}(path, headers = {}, body = nil, &block)
+              request(:#{verb}, path, headers, body, &block)
+            end
+          EVAL
+        end
+        
+        # Called when a method which requires a bucket name is called without that bucket name specified. It will try to
+        # infer the current bucket by looking for it as the subdomain of the current connection's address. If no subdomain
+        # is found, CurrentBucketNotSpecified will be raised.
+        #
+        #   MusicBucket.establish_connection! :server => 'jukeboxzero.s3.amazonaws.com'
+        #   MusicBucket.connection.server
+        #   => 'jukeboxzero.s3.amazonaws.com'
+        #   MusicBucket.current_bucket
+        #   => 'jukeboxzero'
+        #
+        # Rather than infering the current bucket from the subdomain, the current class' bucket can be explicitly set with
+        # set_current_bucket_to.
+        def current_bucket
+          connection.subdomain or raise CurrentBucketNotSpecified.new(connection.http.address)
+        end
+        
+        # If you plan on always using a specific bucket for certain files, you can skip always having to specify the bucket by creating 
+        # a subclass of Bucket or S3Object and telling it what bucket to use:
+        # 
+        #   class JukeBoxSong < AWS::S3::S3Object
+        #     set_current_bucket_to 'jukebox'
+        #   end
+        # 
+        # For all methods that take a bucket name as an argument, the current bucket will be used if the bucket name argument is omitted.
+        #
+        #   other_song = 'baby-please-come-home.mp3'
+        #   JukeBoxSong.store(other_song, open(other_song))
+        # 
+        # This time we didn't have to explicitly pass in the bucket name, as the JukeBoxSong class knows that it will
+        # always use the 'jukebox' bucket. 
+        # 
+        # "Astute readers", as they say, may have noticed that we used the third parameter to pass in the content type,
+        # rather than the fourth parameter as we had the last time we created an object. If the bucket can be inferred, or
+        # is explicitly set, as we've done in the JukeBoxSong class, then the third argument can be used to pass in
+        # options.
+        # 
+        # Now all operations that would have required a bucket name no longer do.
+        # 
+        #   other_song = JukeBoxSong.find('baby-please-come-home.mp3')
+        def set_current_bucket_to(name)
+          raise ArgumentError, "`#{__method__}' must be called on a subclass of #{self.name}" if self == AWS::S3::Base
+          instance_eval(<<-EVAL)
+            def current_bucket
+              '#{name}'
+            end
+          EVAL
+        end
+        alias_method :current_bucket=, :set_current_bucket_to
+          
+        private
+          
+          def response_class
+            FindResponseClass.for(self)
+          end
+          
+          def process_options!(options, verb)
+            options.replace(RequestOptions.process(options, verb))
+          end
+          
+          # Using the conventions layed out in the <tt>response_class</tt> works for more than 80% of the time.
+          # There are a few edge cases though where we want a given class to wrap its responses in different
+          # response classes depending on which method is being called.
+          def respond_with(klass)
+            eval(<<-EVAL, binding, __FILE__, __LINE__)
+              def new_response_class
+                #{klass}
+              end
+
+              class << self
+                alias_method :old_response_class, :response_class
+                alias_method :response_class, :new_response_class
+              end
+            EVAL
+
+            yield
+          ensure
+            # Restore the original version
+            eval(<<-EVAL, binding, __FILE__, __LINE__)
+              class << self
+                alias_method :response_class, :old_response_class
+              end
+            EVAL
+          end
+          
+          def bucket_name(name)
+            name || current_bucket
+          end
+
+          def retry_exceptions
+            [InternalError, RequestTimeout]
+          end
+          
+          class RequestOptions < Hash #:nodoc:
+            attr_reader :options, :verb
+            
+            class << self
+              def process(*args, &block)
+                new(*args, &block).process!
+              end
+            end
+            
+            def initialize(options, verb = :get)
+              @options = options.to_normalized_options
+              @verb    = verb
+              super()
+            end
+            
+            def process!
+              set_access_controls! if verb == :put
+              replace(options)
+            end
+            
+            private 
+              def set_access_controls!
+                ACL::OptionProcessor.process!(options)
+              end
+          end
+      end
+      
+      def initialize(attributes = {}) #:nodoc:
+        @attributes = attributes
+      end
+      
+      private
+        attr_reader :attributes
+        
+        def connection
+          self.class.connection
+        end
+        
+        def http
+          connection.http
+        end
+        
+        def request(*args, &block)
+          self.class.request(*args, &block)
+        end
+        
+        def method_missing(method, *args, &block)
+          case
+          when attributes.has_key?(method.to_s): attributes[method.to_s]
+          when attributes.has_key?(method):      attributes[method]
+          else super
+          end
+        end
+    end
+    
+    protected
+    
+      def self.remove_or_extract_bucket_from_path(path, extract=false)
+        path.match(/^\/([a-z0-9_-]+)[\/]{0,1}(.*)$/)
+      
+        return extract ? $1 : "/#{$2}"
+      end
+  end
+end

data/lib/aws/s3/bittorrent.rb

@@ -0,0 +1,58 @@
+module AWS
+  module S3
+    # Objects on S3 can be distributed via the BitTorrent file sharing protocol. 
+    #
+    # You can get a torrent file for an object by calling <tt>torrent_for</tt>:
+    #
+    #   S3Object.torrent_for 'kiss.jpg', 'marcel'
+    #
+    # Or just call the <tt>torrent</tt> method if you already have the object:
+    #
+    #   song = S3Object.find 'kiss.jpg', 'marcel'
+    #   song.torrent
+    #
+    # Calling <tt>grant_torrent_access_to</tt> on a object will allow anyone to anonymously
+    # fetch the torrent file for that object:
+    #
+    #   S3Object.grant_torrent_access_to 'kiss.jpg', 'marcel'
+    #
+    # Anonymous requests to
+    #
+    #   http://s3.amazonaws.com/marcel/kiss.jpg?torrent
+    #
+    # will serve up the torrent file for that object.
+    module BitTorrent
+      def self.included(klass) #:nodoc:
+        klass.extend ClassMethods
+      end
+      
+      # Adds methods to S3Object for accessing the torrent of a given object.
+      module ClassMethods
+        # Returns the torrent file for the object with the given <tt>key</tt>.
+        def torrent_for(key, bucket = nil)
+          get(path!(bucket, key) << '?torrent').body
+        end
+        alias_method :torrent, :torrent_for
+        
+        # Grants access to the object with the given <tt>key</tt> to be accessible as a torrent.
+        def grant_torrent_access_to(key, bucket = nil)
+          policy = acl(key, bucket)
+          return true if policy.grants.include?(:public_read)
+          policy.grants << ACL::Grant.grant(:public_read)
+          acl(key, bucket, policy)
+        end
+        alias_method :grant_torrent_access, :grant_torrent_access_to
+      end
+      
+      # Returns the torrent file for the object.
+      def torrent
+        self.class.torrent_for(key, bucket.name)
+      end
+      
+      # Grants torrent access publicly to anyone who requests it on this object.
+      def grant_torrent_access
+        self.class.grant_torrent_access_to(key, bucket.name)
+      end
+    end
+  end
+end