aws-s3-multi-region 0.6.3

data/lib/aws/s3/bittorrent.rb

@@ -0,0 +1,58 @@
+module AWS
+  module S3
+    # Objects on S3 can be distributed via the BitTorrent file sharing protocol. 
+    #
+    # You can get a torrent file for an object by calling <tt>torrent_for</tt>:
+    #
+    #   S3Object.torrent_for 'kiss.jpg', 'marcel'
+    #
+    # Or just call the <tt>torrent</tt> method if you already have the object:
+    #
+    #   song = S3Object.find 'kiss.jpg', 'marcel'
+    #   song.torrent
+    #
+    # Calling <tt>grant_torrent_access_to</tt> on a object will allow anyone to anonymously
+    # fetch the torrent file for that object:
+    #
+    #   S3Object.grant_torrent_access_to 'kiss.jpg', 'marcel'
+    #
+    # Anonymous requests to
+    #
+    #   http://s3.amazonaws.com/marcel/kiss.jpg?torrent
+    #
+    # will serve up the torrent file for that object.
+    module BitTorrent
+      def self.included(klass) #:nodoc:
+        klass.extend ClassMethods
+      end
+      
+      # Adds methods to S3Object for accessing the torrent of a given object.
+      module ClassMethods
+        # Returns the torrent file for the object with the given <tt>key</tt>.
+        def torrent_for(key, bucket = nil)
+          get(path!(bucket, key) << '?torrent').body
+        end
+        alias_method :torrent, :torrent_for
+        
+        # Grants access to the object with the given <tt>key</tt> to be accessible as a torrent.
+        def grant_torrent_access_to(key, bucket = nil)
+          policy = acl(key, bucket)
+          return true if policy.grants.include?(:public_read)
+          policy.grants << ACL::Grant.grant(:public_read)
+          acl(key, bucket, policy)
+        end
+        alias_method :grant_torrent_access, :grant_torrent_access_to
+      end
+      
+      # Returns the torrent file for the object.
+      def torrent
+        self.class.torrent_for(key, bucket.name)
+      end
+      
+      # Grants torrent access publicly to anyone who requests it on this object.
+      def grant_torrent_access
+        self.class.grant_torrent_access_to(key, bucket.name)
+      end
+    end
+  end
+end

data/lib/aws/s3/bucket.rb

@@ -0,0 +1,338 @@
+module AWS
+  module S3
+    # Buckets are containers for objects (the files you store on S3). To create a new bucket you just specify its name.
+    #
+    #   # Pick a unique name, or else you'll get an error
+    #   # if the name is already taken.
+    #   Bucket.create('jukebox')
+    #
+    # Bucket names must be unique across the entire S3 system, sort of like domain names across the internet. If you try
+    # to create a bucket with a name that is already taken, you will get an error.
+    #
+    # Assuming the name you chose isn't already taken, your new bucket will now appear in the bucket list:
+    #
+    #   Service.buckets
+    #   # => [#<AWS::S3::Bucket @attributes={"name"=>"jukebox"}>]
+    #
+    # Once you have succesfully created a bucket you can you can fetch it by name using Bucket.find.
+    #
+    #   music_bucket = Bucket.find('jukebox')
+    #
+    # The bucket that is returned will contain a listing of all the objects in the bucket.
+    #
+    #   music_bucket.objects.size
+    #   # => 0
+    #
+    # If all you are interested in is the objects of the bucket, you can get to them directly using Bucket.objects.
+    #
+    #   Bucket.objects('jukebox').size
+    #   # => 0
+    #
+    # By default all objects will be returned, though there are several options you can use to limit what is returned, such as
+    # specifying that only objects whose name is after a certain place in the alphabet be returned, and etc. Details about these options can
+    # be found in the documentation for Bucket.find.
+    #
+    # To add an object to a bucket you specify the name of the object, its value, and the bucket to put it in.
+    #
+    #   file = 'black-flowers.mp3'
+    #   S3Object.store(file, open(file), 'jukebox')
+    #
+    # You'll see your file has been added to it:
+    #
+    #   music_bucket.objects
+    #   # => [#<AWS::S3::S3Object '/jukebox/black-flowers.mp3'>]
+    #
+    # You can treat your bucket like a hash and access objects by name:
+    #
+    #   jukebox['black-flowers.mp3']
+    #   # => #<AWS::S3::S3Object '/jukebox/black-flowers.mp3'>
+    #
+    # In the event that you want to delete a bucket, you can use Bucket.delete.
+    #
+    #   Bucket.delete('jukebox')
+    #
+    # Keep in mind, like unix directories, you can not delete a bucket unless it is empty. Trying to delete a bucket
+    # that contains objects will raise a BucketNotEmpty exception.
+    #
+    # Passing the :force => true option to delete will take care of deleting all the bucket's objects for you.
+    #
+    #   Bucket.delete('photos', :force => true)
+    #   # => true
+    class Bucket < Base
+      class << self
+        # Creates a bucket named <tt>name</tt>.
+        #
+        #   Bucket.create('jukebox')
+        #
+        # Your bucket name must be unique across all of S3. If the name
+        # you request has already been taken, you will get a 409 Conflict response, and a BucketAlreadyExists exception
+        # will be raised.
+        #
+        # By default new buckets have their access level set to private. You can override this using the <tt>:access</tt> option.
+        #
+        #   Bucket.create('internet_drop_box', :access => :public_read_write)
+        #
+        # The full list of access levels that you can set on Bucket and S3Object creation are listed in the README[link:files/README.html]
+        # in the section called 'Setting access levels'.
+        def create(name, options = {})
+          self.current_host = name
+          validate_name!(name)
+          put("/", options, BucketConfiguration.new(options[:location]).to_s).success?
+        end
+
+        class BucketConfiguration < XmlGenerator
+          attr_reader :location
+
+          def initialize(location)
+            @location = location
+            super()
+          end
+
+          def build
+            return nil unless location == :eu
+            xml.tag!('CreateBucketConfiguration') do
+              xml.LocationConstraint 'EU'
+            end
+          end
+        end
+
+        # Fetches the bucket named <tt>name</tt>.
+        #
+        #   Bucket.find('jukebox')
+        #
+        # If a default bucket is inferable from the current connection's subdomain, or if set explicitly with Base.set_current_bucket,
+        # it will be used if no bucket is specified.
+        #
+        #   MusicBucket.current_bucket
+        #   => 'jukebox'
+        #   MusicBucket.find.name
+        #   => 'jukebox'
+        #
+        # By default all objects contained in the bucket will be returned (sans their data) along with the bucket.
+        # You can access your objects using the Bucket#objects method.
+        #
+        #   Bucket.find('jukebox').objects
+        #
+        # There are several options which allow you to limit which objects are retrieved. The list of object filtering options
+        # are listed in the documentation for Bucket.objects.
+        def find(name = nil, options = {})
+          new(get(path(name, options)).bucket)
+        end
+
+        # Return just the objects in the bucket named <tt>name</tt>.
+        #
+        # By default all objects of the named bucket will be returned. There are options, though, for filtering
+        # which objects are returned.
+        #
+        # === Object filtering options
+        #
+        # * <tt>:max_keys</tt> - The maximum number of keys you'd like to see in the response body.
+        #   The server may return fewer than this many keys, but will not return more.
+        #
+        #     Bucket.objects('jukebox').size
+        #     # => 3
+        #     Bucket.objects('jukebox', :max_keys => 1).size
+        #     # => 1
+        #
+        # * <tt>:prefix</tt> - Restricts the response to only contain results that begin with the specified prefix.
+        #
+        #     Bucket.objects('jukebox')
+        #     # => [<AWS::S3::S3Object '/jazz/miles.mp3'>, <AWS::S3::S3Object '/jazz/dolphy.mp3'>, <AWS::S3::S3Object '/classical/malher.mp3'>]
+        #     Bucket.objects('jukebox', :prefix => 'classical')
+        #     # => [<AWS::S3::S3Object '/classical/malher.mp3'>]
+        #
+        # * <tt>:marker</tt> - Marker specifies where in the result set to resume listing. It restricts the response
+        #   to only contain results that occur alphabetically _after_ the value of marker. To retrieve the next set of results,
+        #   use the last key from the current page of results as the marker in your next request.
+        #
+        #     # Skip 'mahler'
+        #     Bucket.objects('jukebox', :marker => 'mb')
+        #     # => [<AWS::S3::S3Object '/jazz/miles.mp3'>]
+        #
+        # === Examples
+        #
+        #   # Return no more than 2 objects whose key's are listed alphabetically after the letter 'm'.
+        #   Bucket.objects('jukebox', :marker => 'm', :max_keys => 2)
+        #   # => [<AWS::S3::S3Object '/jazz/miles.mp3'>, <AWS::S3::S3Object '/classical/malher.mp3'>]
+        #
+        #   # Return no more than 2 objects whose key's are listed alphabetically after the letter 'm' and have the 'jazz' prefix.
+        #   Bucket.objects('jukebox', :marker => 'm', :max_keys => 2, :prefix => 'jazz')
+        #   # => [<AWS::S3::S3Object '/jazz/miles.mp3'>]
+        def objects(name = nil, options = {})
+          find(name, options).object_cache
+        end
+
+        # Deletes the bucket named <tt>name</tt>.
+        #
+        # All objects in the bucket must be deleted before the bucket can be deleted. If the bucket is not empty,
+        # BucketNotEmpty will be raised.
+        #
+        # You can side step this issue by passing the :force => true option to delete which will take care of
+        # emptying the bucket before deleting it.
+        #
+        #   Bucket.delete('photos', :force => true)
+        #
+        # Only the owner of a bucket can delete a bucket, regardless of the bucket's access control policy.
+        def delete(name = nil, options = {})
+          find(name).delete_all if options[:force]
+
+          name = path(name)
+          Base.delete(name).success?
+        end
+
+        # List all your buckets. This is a convenient wrapper around AWS::S3::Service.buckets.
+        def list(reload = false)
+          self.current_host = nil
+          Service.buckets(reload)
+        end
+
+        private
+          def validate_name!(name)
+            raise InvalidBucketName.new(name) unless name =~ /^[-\w.]{3,255}$/
+          end
+
+          def path(name, options = {})
+            if name.is_a?(Hash)
+              options = name
+              name    = nil
+            end
+            self.current_host = bucket_name(name)
+            "/#{RequestOptions.process(options).to_query_string}"
+          end
+      end
+
+      attr_reader :object_cache #:nodoc:
+
+      include Enumerable
+
+      def initialize(attributes = {}) #:nodoc:
+        super
+        @object_cache = []
+        build_contents!
+      end
+
+      # Fetches the object named <tt>object_key</tt>, or nil if the bucket does not contain an object with the
+      # specified key.
+      #
+      #   bucket.objects
+      #   => [#<AWS::S3::S3Object '/marcel_molina/beluga_baby.jpg'>,
+      #       #<AWS::S3::S3Object '/marcel_molina/tongue_overload.jpg'>]
+      #   bucket['beluga_baby.jpg']
+      #   => #<AWS::S3::S3Object '/marcel_molina/beluga_baby.jpg'>
+      def [](object_key)
+        detect {|file| file.key == object_key.to_s}
+      end
+
+      # Initializes a new S3Object belonging to the current bucket.
+      #
+      #   object = bucket.new_object
+      #   object.value = data
+      #   object.key   = 'classical/mahler.mp3'
+      #   object.store
+      #   bucket.objects.include?(object)
+      #   => true
+      def new_object(attributes = {})
+        object = S3Object.new(attributes)
+        register(object)
+        object
+      end
+
+      # List S3Object's of the bucket.
+      #
+      # Once fetched the objects will be cached. You can reload the objects by passing <tt>:reload</tt>.
+      #
+      #   bucket.objects(:reload)
+      #
+      # You can also filter the objects using the same options listed in Bucket.objects.
+      #
+      #   bucket.objects(:prefix => 'jazz')
+      #
+      # Using these filtering options will implictly reload the objects.
+      #
+      # To reclaim all the objects for the bucket you can pass in :reload again.
+      def objects(options = {})
+        if options.is_a?(Hash)
+          reload = !options.empty?
+        else
+          reload  = options
+          options = {}
+        end
+
+        reload!(options) if reload || object_cache.empty?
+        object_cache
+      end
+
+      # Iterates over the objects in the bucket.
+      #
+      #   bucket.each do |object|
+      #     # Do something with the object ...
+      #   end
+      def each(&block)
+        # Dup the collection since we might be destructively modifying the object_cache during the iteration.
+        objects.dup.each(&block)
+      end
+
+      # Returns true if there are no objects in the bucket.
+      def empty?
+        objects.empty?
+      end
+
+      # Returns the number of objects in the bucket.
+      def size
+        objects.size
+      end
+
+      # Deletes the bucket. See its class method counter part Bucket.delete for caveats about bucket deletion and how to ensure
+      # a bucket is deleted no matter what.
+      def delete(options = {})
+        self.class.delete(name, options)
+      end
+
+      # Delete all files in the bucket. Use with caution. Can not be undone.
+      def delete_all
+        each do |object|
+          object.delete
+        end
+        self
+      end
+      alias_method :clear, :delete_all
+
+      # Buckets observe their objects and have this method called when one of their objects
+      # is either stored or deleted.
+      def update(action, object) #:nodoc:
+        case action
+        when :stored  then add object unless objects.include?(object)
+        when :deleted then object_cache.delete(object)
+        end
+      end
+
+      private
+        def build_contents!
+          return unless has_contents?
+          attributes.delete('contents').each do |content|
+            add new_object(content)
+          end
+        end
+
+        def has_contents?
+          attributes.has_key?('contents')
+        end
+
+        def add(object)
+          register(object)
+          object_cache << object
+        end
+
+        def register(object)
+          object.bucket = self
+        end
+
+        def reload!(options = {})
+          object_cache.clear
+          self.class.objects(name, options).each do |object|
+            add object
+          end
+        end
+    end
+  end
+end

data/lib/aws/s3/connection.rb

@@ -0,0 +1,288 @@
+module AWS
+  module S3
+    class Connection #:nodoc:
+      class << self
+        def connect(options = {})
+          new(options)
+        end
+
+        def prepare_path(path)
+          path = path.remove_extended unless path.valid_utf8?
+          URI.escape(path)
+        end
+      end
+
+      attr_reader :access_key_id, :secret_access_key, :http, :options
+
+      # Creates a new connection. Connections make the actual requests to S3, though these requests are usually
+      # called from subclasses of Base.
+      #
+      # For details on establishing connections, check the Connection::Management::ClassMethods.
+      def initialize(options = {})
+        @options = Options.new(options)
+        connect
+      end
+
+      def request(verb, path, headers = {}, body = nil, attempts = 0, current_host = nil, &block)
+        body.rewind if body.respond_to?(:rewind) unless attempts.zero?
+
+        requester = Proc.new do
+          path    = self.class.prepare_path(path) if attempts.zero? # Only escape the path once
+          request = request_method(verb).new(path, headers)
+          ensure_content_type!(request)
+          add_user_agent!(request)
+          set_host!(request, current_host)
+          authenticate!(request, current_host)
+          if body
+            if body.respond_to?(:read)
+              request.body_stream = body
+            else
+              request.body = body
+            end
+            request.content_length = body.respond_to?(:lstat) ? body.stat.size : body.size
+          else
+            request.content_length = 0
+          end
+          http.request(request, &block)
+        end
+
+        if persistent?
+          http.start unless http.started?
+          requester.call
+        else
+          http.start(&requester)
+        end
+      rescue Errno::EPIPE, Timeout::Error, Errno::EINVAL, EOFError
+        @http = create_connection
+        attempts == 3 ? raise : (attempts += 1; retry)
+      end
+
+      def url_for(path, current_host, options = {})
+        authenticate = options.delete(:authenticated)
+        # Default to true unless explicitly false
+        authenticate = true if authenticate.nil?
+        path         = self.class.prepare_path(path)
+        request      = request_method(:get).new(path, {})
+        query_string = query_string_authentication(request, current_host, options)
+        returning "#{protocol(options)}#{get_host(current_host)}#{port_string}#{path}" do |url|
+          url << "?#{query_string}" if authenticate
+        end
+      end
+
+      def subdomain
+        subdomain = http.address.gsub("#{DEFAULT_HOST}", "").gsub(/.$/,'')
+        http.address[/^([^.]+).#{DEFAULT_HOST}$/, 1]
+      end
+
+      def persistent?
+        options[:persistent]
+      end
+
+      def protocol(options = {})
+        # This always trumps http.use_ssl?
+        if options[:use_ssl] == false
+          'http://'
+        elsif options[:use_ssl] || http.use_ssl?
+          'https://'
+        else
+          'http://'
+        end
+      end
+
+      private
+        def extract_keys!
+          missing_keys = []
+          extract_key = Proc.new {|key| options[key] || (missing_keys.push(key); nil)}
+          @access_key_id     = extract_key[:access_key_id]
+          @secret_access_key = extract_key[:secret_access_key]
+          raise MissingAccessKey.new(missing_keys) unless missing_keys.empty?
+        end
+
+        def create_connection
+          http             = http_class.new(options[:server], options[:port])
+          http.use_ssl     = !options[:use_ssl].nil? || options[:port] == 443
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http
+        end
+
+        def http_class
+          if options.connecting_through_proxy?
+            Net::HTTP::Proxy(*options.proxy_settings)
+          else
+            Net::HTTP
+          end
+        end
+
+        def connect
+          extract_keys!
+          @http = create_connection
+        end
+
+        def port_string
+          default_port = options[:use_ssl] ? 443 : 80
+          http.port == default_port ? '' : ":#{http.port}"
+        end
+
+        def ensure_content_type!(request)
+          request['Content-Type'] ||= 'binary/octet-stream'
+        end
+
+        # Just do Header authentication for now
+        def authenticate!(request, current_host)
+          request['Authorization'] = Authentication::Header.new(request, access_key_id, secret_access_key, current_host)
+        end
+
+        def add_user_agent!(request)
+          request['User-Agent'] ||= "AWS::S3/#{Version}"
+        end
+
+        def set_host!(request, host)
+          request['Host'] = get_host(host)
+        end
+
+        def get_host(host)
+          (host && http.address.match(host)) ? http.address : host.nil? ? http.address : host.match(/amazonaws.com/) ? host : "#{host}.#{http.address}"
+        end
+
+        def query_string_authentication(request, options = {})
+          Authentication::QueryString.new(request, access_key_id, secret_access_key, options)
+        end
+
+        def request_method(verb)
+          Net::HTTP.const_get(verb.to_s.capitalize)
+        end
+
+        def method_missing(method, *args, &block)
+          options[method] || super
+        end
+
+      module Management #:nodoc:
+        def self.included(base)
+          base.cattr_accessor :connections
+          base.connections = {}
+          base.extend ClassMethods
+        end
+
+        # Manage the creation and destruction of connections for AWS::S3::Base and its subclasses. Connections are
+        # created with establish_connection!.
+        module ClassMethods
+          # Creates a new connection with which to make requests to the S3 servers for the calling class.
+          #
+          #   AWS::S3::Base.establish_connection!(:access_key_id => '...', :secret_access_key => '...')
+          #
+          # You can set connections for every subclass of AWS::S3::Base. Once the initial connection is made on
+          # Base, all subsequent connections will inherit whatever values you don't specify explictly. This allows you to
+          # customize details of the connection, such as what server the requests are made to, by just specifying one
+          # option.
+          #
+          #   AWS::S3::Bucket.established_connection!(:use_ssl => true)
+          #
+          # The Bucket connection would inherit the <tt>:access_key_id</tt> and the <tt>:secret_access_key</tt> from
+          # Base's connection. Unlike the Base connection, all Bucket requests would be made over SSL.
+          #
+          # == Required arguments
+          #
+          # * <tt>:access_key_id</tt> - The access key id for your S3 account. Provided by Amazon.
+          # * <tt>:secret_access_key</tt> - The secret access key for your S3 account. Provided by Amazon.
+          #
+          # If any of these required arguments is missing, a MissingAccessKey exception will be raised.
+          #
+          # == Optional arguments
+          #
+          # * <tt>:server</tt> - The server to make requests to. You can use this to specify your bucket in the subdomain,
+          # or your own domain's cname if you are using virtual hosted buckets. Defaults to <tt>s3.amazonaws.com</tt>.
+          # * <tt>:port</tt> - The port to the requests should be made on. Defaults to 80 or 443 if the <tt>:use_ssl</tt>
+          # argument is set.
+          # * <tt>:use_ssl</tt> - Whether requests should be made over SSL. If set to true, the <tt>:port</tt> argument
+          # will be implicitly set to 443, unless specified otherwise. Defaults to false.
+          # * <tt>:persistent</tt> - Whether to use a persistent connection to the server. Having this on provides around a two fold
+          # performance increase but for long running processes some firewalls may find the long lived connection suspicious and close the connection.
+          # If you run into connection errors, try setting <tt>:persistent</tt> to false. Defaults to false.
+          # * <tt>:proxy</tt> - If you need to connect through a proxy, you can specify your proxy settings by specifying a <tt>:host</tt>, <tt>:port</tt>, <tt>:user</tt>, and <tt>:password</tt>
+          # with the <tt>:proxy</tt> option.
+          # The <tt>:host</tt> setting is required if specifying a <tt>:proxy</tt>.
+          #
+          #   AWS::S3::Bucket.established_connection!(:proxy => {
+          #     :host => '...', :port => 8080, :user => 'marcel', :password => 'secret'
+          #   })
+          def establish_connection!(options = {})
+            # After you've already established the default connection, just specify
+            # the difference for subsequent connections
+            options = default_connection.options.merge(options) if connected?
+            connections[connection_name] = Connection.connect(options)
+          end
+
+          # Returns the connection for the current class, or Base's default connection if the current class does not
+          # have its own connection.
+          #
+          # If not connection has been established yet, NoConnectionEstablished will be raised.
+          def connection
+            if connected?
+              connections[connection_name] || default_connection
+            else
+              raise NoConnectionEstablished
+            end
+          end
+
+          # Returns true if a connection has been made yet.
+          def connected?
+            !connections.empty?
+          end
+
+          # Removes the connection for the current class. If there is no connection for the current class, the default
+          # connection will be removed.
+          def disconnect(name = connection_name)
+            name       = default_connection unless connections.has_key?(name)
+            connection = connections[name]
+            connection.http.finish if connection.persistent?
+            connections.delete(name)
+          end
+
+          # Clears *all* connections, from all classes, with prejudice.
+          def disconnect!
+            connections.each_key {|connection| disconnect(connection)}
+          end
+
+          private
+            def connection_name
+              name
+            end
+
+            def default_connection_name
+              'AWS::S3::Base'
+            end
+
+            def default_connection
+              connections[default_connection_name]
+            end
+        end
+      end
+
+      class Options < Hash #:nodoc:
+        VALID_OPTIONS = [:access_key_id, :secret_access_key, :server, :port, :use_ssl, :persistent, :proxy].freeze
+
+        def initialize(options = {})
+          super()
+          validate(options)
+          replace(:server => DEFAULT_HOST, :port => (options[:use_ssl] ? 443 : 80))
+          merge!(options)
+        end
+
+        def connecting_through_proxy?
+          !self[:proxy].nil?
+        end
+
+        def proxy_settings
+          self[:proxy].values_at(:host, :port, :user, :password)
+        end
+
+        private
+          def validate(options)
+            invalid_options = options.keys - VALID_OPTIONS
+            raise InvalidConnectionOption.new(invalid_options) unless invalid_options.empty?
+            raise ArgumentError, "Missing proxy settings. Must specify at least :host." if options[:proxy] && !options[:proxy][:host]
+          end
+      end
+    end
+  end
+end