s3fsr 1.4

data/lib/aws-matt/s3/bucket.rb

@@ -0,0 +1,348 @@
+module AWS
+  module S3
+    # Buckets are containers for objects (the files you store on S3). To create a new bucket you just specify its name.
+    # 
+    #   # Pick a unique name, or else you'll get an error 
+    #   # if the name is already taken.
+    #   Bucket.create('jukebox')
+    # 
+    # Bucket names must be unique across the entire S3 system, sort of like domain names across the internet. If you try
+    # to create a bucket with a name that is already taken, you will get an error.
+    #
+    # Assuming the name you chose isn't already taken, your new bucket will now appear in the bucket list:
+    # 
+    #   Service.buckets
+    #   # => [#<AWS::S3::Bucket @attributes={"name"=>"jukebox"}>]
+    # 
+    # Once you have succesfully created a bucket you can you can fetch it by name using Bucket.find.
+    #
+    #   music_bucket = Bucket.find('jukebox')
+    #
+    # The bucket that is returned will contain a listing of all the objects in the bucket.
+    #
+    #   music_bucket.objects.size
+    #   # => 0
+    #
+    # If all you are interested in is the objects of the bucket, you can get to them directly using Bucket.objects.
+    #
+    #   Bucket.objects('jukebox').size
+    #   # => 0
+    #
+    # By default all objects will be returned, though there are several options you can use to limit what is returned, such as
+    # specifying that only objects whose name is after a certain place in the alphabet be returned, and etc. Details about these options can
+    # be found in the documentation for Bucket.find.
+    #
+    # To add an object to a bucket you specify the name of the object, its value, and the bucket to put it in.
+    # 
+    #   file = 'black-flowers.mp3'
+    #   S3Object.store(file, open(file), 'jukebox')
+    #
+    # You'll see your file has been added to it:
+    # 
+    #   music_bucket.objects
+    #   # => [#<AWS::S3::S3Object '/jukebox/black-flowers.mp3'>]
+    # 
+    # You can treat your bucket like a hash and access objects by name:
+    # 
+    #   jukebox['black-flowers.mp3']
+    #   # => #<AWS::S3::S3Object '/jukebox/black-flowers.mp3'>
+    # 
+    # In the event that you want to delete a bucket, you can use Bucket.delete.
+    #
+    #   Bucket.delete('jukebox')
+    #
+    # Keep in mind, like unix directories, you can not delete a bucket unless it is empty. Trying to delete a bucket
+    # that contains objects will raise a BucketNotEmpty exception.
+    #
+    # Passing the :force => true option to delete will take care of deleting all the bucket's objects for you.
+    #
+    #   Bucket.delete('photos', :force => true)
+    #   # => true
+    class Bucket < Base
+      class << self
+        # Creates a bucket named <tt>name</tt>. 
+        #
+        #   Bucket.create('jukebox')
+        #
+        # Your bucket name must be unique across all of S3. If the name
+        # you request has already been taken, you will get a 409 Conflict response, and a BucketAlreadyExists exception
+        # will be raised.
+        #
+        # By default new buckets have their access level set to private. You can override this using the <tt>:access</tt> option.
+        #
+        #   Bucket.create('internet_drop_box', :access => :public_read_write)
+        #
+        # The full list of access levels that you can set on Bucket and S3Object creation are listed in the README[link:files/README.html] 
+        # in the section called 'Setting access levels'.
+        def create(name, options = {})
+          validate_name!(name)
+          put("/#{name}", options).success?
+        end
+        
+        # Fetches the bucket named <tt>name</tt>. 
+        #
+        #   Bucket.find('jukebox')
+        #
+        # If a default bucket is inferable from the current connection's subdomain, or if set explicitly with Base.set_current_bucket, 
+        # it will be used if no bucket is specified.
+        #
+        #   MusicBucket.current_bucket
+        #   => 'jukebox'
+        #   MusicBucket.find.name
+        #   => 'jukebox'
+        #
+        # By default all objects contained in the bucket will be returned (sans their data) along with the bucket.
+        # You can access your objects using the Bucket#objects method.
+        #
+        #   Bucket.find('jukebox').objects
+        # 
+        # There are several options which allow you to limit which objects are retrieved. The list of object filtering options
+        # are listed in the documentation for Bucket.objects.
+        def find(name = nil, options = {})
+          new(get(path(name, options)).bucket)
+        end
+        
+        # Return just the objects in the bucket named <tt>name</tt>.
+        #
+        # By default all objects of the named bucket will be returned. There are options, though, for filtering
+        # which objects are returned.
+        #
+        # === Object filtering options
+        # 
+        # * <tt>:max_keys</tt> - The maximum number of keys you'd like to see in the response body. 
+        #   The server may return fewer than this many keys, but will not return more.
+        #
+        #     Bucket.objects('jukebox').size
+        #     # => 3
+        #     Bucket.objects('jukebox', :max_keys => 1).size
+        #     # => 1
+        #
+        # * <tt>:prefix</tt> - Restricts the response to only contain results that begin with the specified prefix.
+        #
+        #     Bucket.objects('jukebox')
+        #     # => [<AWS::S3::S3Object '/jazz/miles.mp3'>, <AWS::S3::S3Object '/jazz/dolphy.mp3'>, <AWS::S3::S3Object '/classical/malher.mp3'>]
+        #     Bucket.objects('jukebox', :prefix => 'classical')
+        #     # => [<AWS::S3::S3Object '/classical/malher.mp3'>]
+        #
+        # * <tt>:marker</tt> - Marker specifies where in the result set to resume listing. It restricts the response 
+        #   to only contain results that occur alphabetically _after_ the value of marker. To retrieve the next set of results, 
+        #   use the last key from the current page of results as the marker in your next request.
+        # 
+        #     # Skip 'mahler'
+        #     Bucket.objects('jukebox', :marker => 'mb')
+        #     # => [<AWS::S3::S3Object '/jazz/miles.mp3'>]
+        #
+        # === Examples
+        #
+        #   # Return no more than 2 objects whose key's are listed alphabetically after the letter 'm'.
+        #   Bucket.objects('jukebox', :marker => 'm', :max_keys => 2)
+        #   # => [<AWS::S3::S3Object '/jazz/miles.mp3'>, <AWS::S3::S3Object '/classical/malher.mp3'>]
+        #
+        #   # Return no more than 2 objects whose key's are listed alphabetically after the letter 'm' and have the 'jazz' prefix.
+        #   Bucket.objects('jukebox', :marker => 'm', :max_keys => 2, :prefix => 'jazz')
+        #   # => [<AWS::S3::S3Object '/jazz/miles.mp3'>]
+        def objects(name = nil, options = {})
+          find(name, options).object_cache
+        end
+        
+        def common_prefixes(name= nil, options = {})
+          find(name, options).common_prefix_cache
+        end
+        
+        # Deletes the bucket named <tt>name</tt>.
+        #
+        # All objects in the bucket must be deleted before the bucket can be deleted. If the bucket is not empty, 
+        # BucketNotEmpty will be raised.
+        #
+        # You can side step this issue by passing the :force => true option to delete which will take care of
+        # emptying the bucket before deleting it.
+        #
+        #   Bucket.delete('photos', :force => true)
+        #
+        # Only the owner of a bucket can delete a bucket, regardless of the bucket's access control policy.
+        def delete(name = nil, options = {})
+          find(name).delete_all if options[:force]
+          
+          name = path(name)
+          Base.delete(name).success?
+        end
+        
+        # List all your buckets. This is a convenient wrapper around AWS::S3::Service.buckets.
+        def list(reload = false)
+          Service.buckets(reload)
+        end
+        
+        private
+          def validate_name!(name)
+            raise InvalidBucketName.new(name) unless name =~ /^[-\w.]{3,255}$/
+          end
+          
+          def path(name, options = {})
+            if name.is_a?(Hash)
+              options = name
+              name    = nil
+            end
+            "/#{bucket_name(name)}#{RequestOptions.process(options).to_query_string}"
+          end
+      end
+      
+      attr_reader :object_cache, :common_prefix_cache #:nodoc:
+      
+      include Enumerable
+      
+      def initialize(attributes = {}) #:nodoc:
+        super
+        @object_cache = []
+        @common_prefix_cache = []
+        build_contents!
+      end
+      
+      # Fetches the object named <tt>object_key</tt>, or nil if the bucket does not contain an object with the 
+      # specified key.
+      #
+      #   bucket.objects
+      #   => [#<AWS::S3::S3Object '/marcel_molina/beluga_baby.jpg'>,
+      #       #<AWS::S3::S3Object '/marcel_molina/tongue_overload.jpg'>]
+      #   bucket['beluga_baby.jpg']
+      #   => #<AWS::S3::S3Object '/marcel_molina/beluga_baby.jpg'>
+      def [](object_key)
+        detect {|file| file.key == object_key.to_s}
+      end
+      
+      # Initializes a new S3Object belonging to the current bucket.
+      #
+      #   object = bucket.new_object
+      #   object.value = data
+      #   object.key   = 'classical/mahler.mp3'
+      #   object.store
+      #   bucket.objects.include?(object)
+      #   => true
+      def new_object(attributes = {})
+        object = S3Object.new(attributes)
+        register(object)
+        object
+      end
+      
+      # List S3Object's of the bucket.
+      #
+      # Once fetched the objects will be cached. You can reload the objects by passing <tt>:reload</tt>.
+      #
+      #   bucket.objects(:reload)
+      #
+      # You can also filter the objects using the same options listed in Bucket.objects.
+      #
+      #   bucket.objects(:prefix => 'jazz')
+      #
+      # Using these filtering options will implictly reload the objects.
+      #
+      # To reclaim all the objects for the bucket you can pass in :reload again.
+      def objects(options = {})
+        if options.is_a?(Hash)
+          reload = !options.empty?
+        else
+          reload  = options
+          options = {}
+        end
+        
+        reload!(options) if reload || object_cache.empty?
+        object_cache
+      end
+      
+      def common_prefixes(options = {})
+        if options.is_a?(Hash)
+          reload = !options.empty?
+        else
+          reload = options
+          options = {}
+        end
+        
+        reload!(options) if reload || common_prefix_cache.empty?
+        common_prefix_cache
+      end
+      
+      # Iterates over the objects in the bucket.
+      #
+      #   bucket.each do |object|
+      #     # Do something with the object ...
+      #   end
+      def each(&block)
+        # Dup the collection since we might be destructively modifying the object_cache during the iteration.
+        objects.dup.each(&block)
+      end
+      
+      # Returns true if there are no objects in the bucket.
+      def empty?
+        objects.empty? && common_prefixes.empty?
+      end
+      
+      # Returns the number of objects in the bucket.
+      def size
+        objects.size
+      end
+      
+      # Deletes the bucket. See its class method counter part Bucket.delete for caveats about bucket deletion and how to ensure
+      # a bucket is deleted no matter what.
+      def delete(options = {})
+        self.class.delete(name, options)
+      end
+      
+      # Delete all files in the bucket. Use with caution. Can not be undone.
+      def delete_all
+        each do |object|
+          object.delete
+        end
+        self
+      end
+      alias_method :clear, :delete_all
+      
+      # Buckets observe their objects and have this method called when one of their objects
+      # is either stored or deleted.
+      def update(action, object) #:nodoc:        
+        case action
+        when :stored  then add object unless objects.include?(object)
+        when :deleted then object_cache.delete(object)
+        end
+      end
+      
+      private        
+        def build_contents!
+          if has_contents? then
+            attributes.delete('contents').each do |content|
+              add new_object(content)
+            end
+          end
+          
+          if attributes['common_prefixes']
+            attributes.delete('common_prefixes').each do |common_prefix|
+              common_prefix_cache << common_prefix['prefix']
+            end
+          end
+        end
+        
+        def has_contents?
+          attributes.has_key?('contents')
+        end
+        
+        def add(object)
+          register(object)
+          object_cache << object
+        end
+        
+        def register(object)
+          object.bucket = self
+        end
+        
+        def reload!(options = {})
+          object_cache.clear
+          self.class.objects(name, options).each do |object| 
+            add object
+          end
+          
+          common_prefix_cache.clear
+          self.class.common_prefixes(name, options).each do |common_prefix|
+            common_prefix_cache << common_prefix
+          end
+        end         
+    end
+  end
+end

data/lib/aws-matt/s3/connection.rb

@@ -0,0 +1,323 @@
+module AWS
+  module S3
+    class Connection #:nodoc:
+      class << self
+        def connect(options = {})
+          new(options)
+        end
+        
+        def prepare_path(path)
+          path = path.remove_extended unless path.utf8?
+          URI.escape(path)
+        end
+      end
+      
+      attr_reader :access_key_id, :secret_access_key, :http, :options
+      
+      # Creates a new connection. Connections make the actual requests to S3, though these requests are usually 
+      # called from subclasses of Base.
+      # 
+      # For details on establishing connections, check the Connection::Management::ClassMethods.
+      def initialize(options = {})
+        @options = Options.new(options)
+        connect
+      end
+          
+      def request(verb, path, headers = {}, body = nil, attempts = 0, &block)
+        body.rewind if body.respond_to?(:rewind) unless attempts.zero?      
+        
+        requester = Proc.new do 
+          path    = self.class.prepare_path(path) if attempts.zero? # Only escape the path once
+          request = request_method(verb).new(path, headers)
+          ensure_content_type!(request)
+          add_user_agent!(request)
+          authenticate!(request)
+          if body
+            if body.respond_to?(:read)                                                                
+              request.body_stream = body                                                           
+            else                                                                                      
+              request.body = body                                                                     
+            end
+            request.content_length = body.respond_to?(:lstat) ? body.stat.size : body.size         
+          else
+            request.content_length = 0                                                                                       
+          end
+          http.request(request, &block)
+        end
+        
+        if persistent?
+          http.start unless http.started?
+          requester.call
+        else
+          http.start(&requester)
+        end
+      rescue Errno::EPIPE, Timeout::Error, Errno::EINVAL, EOFError
+        @http = create_connection
+        attempts == 3 ? raise : (attempts += 1; retry)
+      end
+      
+      def url_for(path, options = {})
+        authenticate = options.delete(:authenticated)
+        # Default to true unless explicitly false
+        authenticate = true if authenticate.nil? 
+        path         = self.class.prepare_path(path)
+        request      = request_method(:get).new(path, {})
+        query_string = query_string_authentication(request, options)
+        returning "#{protocol(options)}#{http.address}#{port_string}#{path}" do |url|
+          url << "?#{query_string}" if authenticate
+        end
+      end
+      
+      def subdomain
+        http.address[/^([^.]+).#{DEFAULT_HOST}$/, 1]
+      end
+      
+      def persistent?
+        options[:persistent]
+      end
+      
+      def protocol(options = {})
+        # This always trumps http.use_ssl?
+        if options[:use_ssl] == false 
+          'http://'
+        elsif options[:use_ssl] || http.use_ssl?
+          'https://'
+        else
+          'http://'
+        end
+      end
+      
+      private
+        def extract_keys!
+          missing_keys = []
+          extract_key = Proc.new {|key| options[key] || (missing_keys.push(key); nil)}
+          @access_key_id     = extract_key[:access_key_id]
+          @secret_access_key = extract_key[:secret_access_key]
+          raise MissingAccessKey.new(missing_keys) unless missing_keys.empty?
+        end
+        
+        def create_connection
+          http             = http_class.new(options[:server], options[:port])
+          http.use_ssl     = !options[:use_ssl].nil? || options[:port] == 443
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http
+        end
+        
+        def http_class
+          if options.connecting_through_proxy?
+            Net::HTTP::Proxy(*options.proxy_settings)
+          else
+            Net::HTTP
+          end
+        end
+        
+        def connect
+          extract_keys!
+          @http = create_connection
+        end
+
+        def port_string
+          default_port = options[:use_ssl] ? 443 : 80
+          http.port == default_port ? '' : ":#{http.port}"
+        end
+
+        def ensure_content_type!(request)
+          request['Content-Type'] ||= 'binary/octet-stream'
+        end
+        
+        # Just do Header authentication for now
+        def authenticate!(request)
+          request['Authorization'] = Authentication::Header.new(request, access_key_id, secret_access_key)
+        end
+        
+        def add_user_agent!(request)
+          request['User-Agent'] ||= "AWS::S3/#{Version}"
+        end
+        
+        def query_string_authentication(request, options = {})
+          Authentication::QueryString.new(request, access_key_id, secret_access_key, options)
+        end
+
+        def request_method(verb)
+          Net::HTTP.const_get(verb.to_s.capitalize)
+        end
+        
+        def method_missing(method, *args, &block)
+          options[method] || super
+        end
+        
+      module Management #:nodoc:
+        def self.included(base)
+          base.cattr_accessor :connections
+          base.connections = {}
+          base.extend ClassMethods
+        end
+        
+        # Manage the creation and destruction of connections for AWS::S3::Base and its subclasses. Connections are
+        # created with establish_connection!.
+        module ClassMethods
+          # Creates a new connection with which to make requests to the S3 servers for the calling class.
+          #   
+          #   AWS::S3::Base.establish_connection!(:access_key_id => '...', :secret_access_key => '...')
+          #
+          # You can set connections for every subclass of AWS::S3::Base. Once the initial connection is made on
+          # Base, all subsequent connections will inherit whatever values you don't specify explictly. This allows you to
+          # customize details of the connection, such as what server the requests are made to, by just specifying one 
+          # option. 
+          #
+          #   AWS::S3::Bucket.established_connection!(:use_ssl => true)
+          #
+          # The Bucket connection would inherit the <tt>:access_key_id</tt> and the <tt>:secret_access_key</tt> from
+          # Base's connection. Unlike the Base connection, all Bucket requests would be made over SSL.
+          #
+          # == Required arguments
+          #
+          # * <tt>:access_key_id</tt> - The access key id for your S3 account. Provided by Amazon.
+          # * <tt>:secret_access_key</tt> - The secret access key for your S3 account. Provided by Amazon.
+          #
+          # If any of these required arguments is missing, a MissingAccessKey exception will be raised.
+          #
+          # == Optional arguments
+          #
+          # * <tt>:server</tt> - The server to make requests to. You can use this to specify your bucket in the subdomain,
+          # or your own domain's cname if you are using virtual hosted buckets. Defaults to <tt>s3.amazonaws.com</tt>.
+          # * <tt>:port</tt> - The port to the requests should be made on. Defaults to 80 or 443 if the <tt>:use_ssl</tt>
+          # argument is set.
+          # * <tt>:use_ssl</tt> - Whether requests should be made over SSL. If set to true, the <tt>:port</tt> argument
+          # will be implicitly set to 443, unless specified otherwise. Defaults to false.
+          # * <tt>:persistent</tt> - Whether to use a persistent connection to the server. Having this on provides around a two fold 
+          # performance increase but for long running processes some firewalls may find the long lived connection suspicious and close the connection.
+          # If you run into connection errors, try setting <tt>:persistent</tt> to false. Defaults to false.
+          # * <tt>:proxy</tt> - If you need to connect through a proxy, you can specify your proxy settings by specifying a <tt>:host</tt>, <tt>:port</tt>, <tt>:user</tt>, and <tt>:password</tt>
+          # with the <tt>:proxy</tt> option.
+          # The <tt>:host</tt> setting is required if specifying a <tt>:proxy</tt>. 
+          #   
+          #   AWS::S3::Bucket.established_connection!(:proxy => {
+          #     :host => '...', :port => 8080, :user => 'marcel', :password => 'secret'
+          #   })
+          def establish_connection!(options = {})
+            # After you've already established the default connection, just specify 
+            # the difference for subsequent connections
+            options = default_connection.options.merge(options) if connected?
+            connections[connection_name] = Connection.connect(options)
+          end
+          
+          # Returns the connection for the current class, or Base's default connection if the current class does not
+          # have its own connection.
+          #
+          # If not connection has been established yet, NoConnectionEstablished will be raised.
+          def connection
+            if connected?
+              connections[connection_name] || default_connection
+            else
+              raise NoConnectionEstablished
+            end
+          end
+          
+          # Returns true if a connection has been made yet.
+          def connected?
+            !connections.empty?
+          end
+          
+          # Removes the connection for the current class. If there is no connection for the current class, the default
+          # connection will be removed.
+          def disconnect(name = connection_name)
+            name       = default_connection unless connections.has_key?(name)
+            connection = connections[name]
+            connection.http.finish if connection.persistent?
+            connections.delete(name)
+          end
+          
+          # Clears *all* connections, from all classes, with prejudice. 
+          def disconnect!
+            connections.each_key {|connection| disconnect(connection)}
+          end
+
+          private
+            def connection_name
+              name
+            end
+
+            def default_connection_name
+              'AWS::S3::Base'
+            end
+
+            def default_connection
+              connections[default_connection_name]
+            end
+        end
+      end
+        
+      class Options < Hash #:nodoc:
+        class << self
+          def valid_options
+            [:access_key_id, :secret_access_key, :server, :port, :use_ssl, :persistent, :proxy]
+          end
+        end
+        
+        attr_reader :options
+        def initialize(options = {})
+          super()
+          @options = options
+          validate!
+          extract_proxy_settings!
+          extract_persistent!
+          extract_server!
+          extract_port!
+          extract_remainder!
+        end
+        
+        def connecting_through_proxy?
+          !self[:proxy].nil?
+        end
+        
+        def proxy_settings
+          proxy_setting_keys.map do |proxy_key| 
+            self[:proxy][proxy_key]
+          end
+        end
+        
+        private
+          def proxy_setting_keys
+            [:host, :port, :user, :password]
+          end
+          
+          def missing_proxy_settings?
+            !self[:proxy].keys.include?(:host)
+          end
+          
+          def extract_persistent!
+            self[:persistent] = options.has_key?(:persitent) ? options[:persitent] : false
+          end
+          
+          def extract_proxy_settings!
+            self[:proxy] = options.delete(:proxy) if options.include?(:proxy)
+            validate_proxy_settings!
+          end
+          
+          def extract_server!
+            self[:server] = options.delete(:server) || DEFAULT_HOST
+          end
+
+          def extract_port!
+            self[:port] = options.delete(:port) || (options[:use_ssl] ? 443 : 80)
+          end
+          
+          def extract_remainder!
+            update(options)
+          end
+          
+          def validate!
+            invalid_options = options.keys.select {|key| !self.class.valid_options.include?(key)}
+            raise InvalidConnectionOption.new(invalid_options) unless invalid_options.empty?
+          end
+          
+          def validate_proxy_settings!
+            if connecting_through_proxy? && missing_proxy_settings?
+              raise ArgumentError, "Missing proxy settings. Must specify at least :host."
+            end
+          end
+      end
+    end
+  end
+end