s33r 0.2 → 0.3

data/lib/s33r/bucket_listing.rb

@@ -1,35 +1,49 @@
-# use prefix to limit keys to some subset of all available keys;
-# use delimiter to group keys
+# TODO: use prefix to limit keys to some subset of all available keys;
+# TODO: use delimiter to group keys
 
 require 'date'
 require 'xml/libxml'
 
-module S3
+module S33r
+  # Object representation of the content of a bucket.
   class BucketListing
-    attr_reader :listing_xml
-
-    # properties of the bucket are publically read-only;
-    # can only be reset by resetting the listing_xml via listing_xml=
-    # (which parses the bucket listing XML)
-    attr_reader :name, :delimiter, :prefix, :marker, :max_keys, :is_truncated, :contents
-
-    def initialize(bucket_listing_xml)
+    attr_reader :delimiter, :prefix, :marker, :max_keys, :is_truncated, :common_prefixes
+      
+    # Name of the bucket this listing is for.
+    attr_reader :name
+    # Hash of objects in this bucket, keyed by their S3 keys.
+    attr_reader :contents
+    # A NamedBucket instance associated with this listing.
+    attr_accessor :named_bucket
+
+    # Create a new object representing a ListBucketResult.
+    # 
+    # +bucket_listing_xml+ is a ListBucketResult document, as returned from a GET on a bucket
+    # (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/).
+    # 
+    # +named_bucket+ can be set to an existing NamedBucket instance, so that any objects
+    # inside this listing can be associated with that instance. This enables objects to be easily deleted
+    # without having to create a new Client instance.
+    def initialize(bucket_listing_xml, named_bucket=nil)
       @contents = {}
       @common_prefixes = {}
+      # the NamedBucket instance associated with this listing (if any)
+      @named_bucket = named_bucket
       set_listing_xml(bucket_listing_xml)
     end
 
+    # Convert a ListBucketResult XML document into an object representation.
     def set_listing_xml(bucket_listing_xml)
       # remove the namespace declaration: libxml doesn't like it
       bucket_listing_xml.gsub!(/ xmlns="http:\/\/s3.amazonaws.com\/doc\/2006-03-01\/"/, '')
-      @listing_xml = bucket_listing_xml
       parse_listing(bucket_listing_xml)
     rescue
       message = "Cannot create bucket listing from supplied XML"
       message += " (was nil)" if bucket_listing_xml.nil?
-      raise S3Exception::InvalidBucketListing, message
+      raise S33rException::InvalidBucketListing, message
     end
 
+    # Parse raw XML ListBucketResponse from S3 into object instances.
     def parse_listing(bucket_listing_xml)
       doc = XML.get_xml_doc(bucket_listing_xml)
 
@@ -48,43 +62,30 @@ module S3
 
       # contents
       doc.find('//Contents').to_a.each do |node|
-        obj = S3Object.new
-        obj.set_from_node(node)
+        obj = S3Object.new(node)
+        # Add to the content listing for the bucket
         @contents[obj.key] = obj
       end
     end
     
-    # return an object in this bucket
+    # Return an object in this bucket by key.
     def [](key)
       @contents[key]
     end
+    
+    # Pretty listing of keys in alphabetical order.
+    def pretty
+      @contents.keys.sort.each { |k| puts k }
+    end
 
-    # setters
+    # Setters which perform some type casts and normalisation.
     private
-      def name=(val)
-        @name = string_prop_normalise(val)
-      end
-
-      def prefix=(val)
-        @prefix = string_prop_normalise(val)
-      end
-
-      def delimiter=(val)
-        @delimiter = string_prop_normalise(val)
-      end
-
-      def marker=(val)
-        @marker = string_prop_normalise(val)
-      end
-
-      def max_keys=(val)
-        @max_keys = val.to_i
-      end
-
-      def is_truncated=(val)
-        @is_truncated = false
-        @is_truncated = true if ('true' == val || true == val || 'True' == val)
-      end
+      def name=(val); @name = string_prop_normalise(val); end
+      def prefix=(val); @prefix = string_prop_normalise(val); end
+      def delimiter=(val); @delimiter = string_prop_normalise(val); end
+      def marker=(val); @marker = string_prop_normalise(val); end
+      def max_keys=(val); @max_keys = val.to_i; end
+      def is_truncated=(val); @is_truncated = ('true' == val || true == val || 'True' == val); end
 
       # normalise string properties:
       # if value for XML element is nil, set property to empty string
@@ -95,44 +96,52 @@ module S3
 
   end
 
+  # Representation of an object stored in a bucket.
   class S3Object
-    attr_reader :bucket_name
-    attr_accessor :key, :last_modified, :etag, :size, :owner, :storage_class
-
-    # bucket: NamedBucket which is the parent of this object
-    def initialize(named_bucket=nil)
+    attr_reader :named_bucket, :key, :last_modified, :etag, :size, :owner, :storage_class
+    attr_writer :named_bucket
+    
+    # Create from a node.
+    def initialize(node=nil, named_bucket=nil)
       @named_bucket = named_bucket
+      self.set_from_node(node) unless node.nil?
+    end
+    
+    # Remove this object from associated NamedBucket.
+    def delete
+      @named_bucket.delete_key(@key) unless @named_bucket.nil?
     end
 
-    # set properties from an XML string
-    # if using this, should be an XML document containing a <Contents> element as root
+    # Set properties of the object from an XML string.
+    # 
+    # +xml_str+ should be a string representing a full XML document,
+    # containing a <Contents> element as its root element.
     def set_from_xml_string(xml_str)
-      doc = XML.get_xml_doc(xml_str)
-      set_from_node(doc)
+      set_from_node(XML.get_xml_doc(xml_str))
     end
 
-    # set properties from an XML document
-    # XML::Document doc: XML document to parse
-    # options:
-    # :full => true: set all properties (default is to only set those in bucket listing)
-    def set_from_node(doc, options={})
+    # Set properties of the object from an XML document.
+    # 
+    # +doc+: XML::Document instance to parse to get properties for this object.
+    def set_from_node(doc)
       @key = doc.xget('Key')
       @last_modified = DateTime.parse(doc.xget('LastModified'))
       @etag = doc.xget('ETag').gsub("\"", "")
       @size = doc.xget('Size').to_i
       @owner = S3User.new(doc.find('Owner').to_a.first)
       
-      if options[:full]
-        # TODO: if setting from a full object listing (GET on a resource key),
-        # do additional field setting here (e.g. x-amz-meta- headers)
-      end
+      # TODO: if setting from a full object listing (GET on a resource key),
+      # do additional field setting here (e.g. x-amz-meta- headers)
+      # and assign the response body to some data field; detect whether
+      # these fields exist before attempting to set properties
     end
   end
 
   class S3User
     attr_accessor :id, :display_name
 
-    # XML::Document owner_xml_doc: <Owner> node from inside ListBucketResult <Contents> element
+    # +owner_xml_doc+: XML::Document instance, representing an <Owner> node from 
+    # inside a ListBucketResult <Contents> element (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/).
     def initialize(owner_xml_doc)
       @id = owner_xml_doc.xget('//ID')
       @display_name = owner_xml_doc.xget('//DisplayName')

data/lib/s33r/client.rb

@@ -1,25 +1,36 @@
 require 'net/https'
 require 'cgi'
 
-module S3
+module S33r
   include Net
 
-  # the client actually performs operations over the network,
+  # The client performs operations over the network,
   # using the core to build request headers and content;
   # only client-specific headers are managed here: other headers
-  # can be handled by the core
-  # TODO: need to wrap XML returned into object representation
-  # TODO: use customisable thread pool for requests
+  # can be handled by the core.
+  #--
+  # TODO: need to wrap XML returned into object representation. 
+  # TODO: use customisable thread pool for requests.
+  #--
   class Client
-    include S3
+    include S33r
 
-    attr_accessor :chunk_size, :default_headers
-
-    # options: hash of optional client config.
-    # :ssl => false : only use plain HTTP
-    # :dump_requests => true: dump each request's initial line and headers
+    attr_accessor :aws_access_key, :aws_secret_access_key
+    
+    # Size of data chunk to be sent per request when putting data.
+    attr_accessor :chunk_size
+    
+    # Headers which should be sent with every request by default (unless overridden).
+    attr_accessor :client_headers
+
+    # Configure either an SSL-enabled or plain HTTP client.
+    # (If using SSL, no verification of server certificate is performed.)
+    # 
+    # +options+: hash of optional client config.:
+    # * <tt>:use_ssl => false</tt>: only use plain HTTP for connections
+    # * <tt>:dump_requests => true</tt>: dump each request's initial line and headers to STDOUT
     def initialize(aws_access_key, aws_secret_access_key, options={})
-      if false == options[:ssl]
+      if false == options[:use_ssl]
         @client = HTTP.new(HOST, NON_SSL_PORT)
         @client.use_ssl = false
       else
@@ -31,8 +42,8 @@ module S3
       
       @dump_requests = (true == options[:dump_requests])
 
-      # set default chunk size for streaming request body (1 Mb)
-      @chunk_size = 1048576
+      # set default chunk size for streaming request body
+      @chunk_size = DEFAULT_CHUNK_SIZE
 
       # Amazon S3 developer keys
       @aws_access_key = aws_access_key
@@ -40,16 +51,51 @@ module S3
 
       # headers sent with every request made by this client
       @client_headers = {}
+    end
+    
+    # Initialise client from YAML configuration file
+    # (see load_config method for details of acceptable format).
+    def Client.init(config_file)
+      aws_access_key, aws_secret_access_key, options, _ = load_config(config_file)
+      Client.new(aws_access_key, aws_secret_access_key, options)
+    end
+    
+    # Load YAML config. file for a client. The config. file looks like this:
+    # 
+    #   :include: test/files/namedbucket_config.yml
+    # 
+    # The +options+ section contains settings specific to Client and NamedClient instances; +custom+ 
+    # contains extra settings specific to your application.
+    # +options+ and +custom+ sections can be omitted, but settings for AWS keys are required.
+    # 
+    # Returns an array <tt>[aws_access_key, aws_secret_access_key, options, custom]</tt>, where +options+
+    # and +custom+ are hashes.
+    def Client.load_config(config_file)
+      require 'yaml'
+      config = YAML::load_file(config_file)
+      aws_access_key = config['aws_access_key']
+      aws_secret_access_key = config['aws_secret_access_key']
       
-      yield self if block_given?
+      options = {}
+      options = S33r.keys_to_symbols(config['options']) if config['options']
+      
+      custom = {}
+      custom = S33r.keys_to_symbols(config['custom']) if config['custom']
+      
+      [aws_access_key, aws_secret_access_key, options, custom]
     end
 
-    # wrapper round embedded client use_ssl accessor
+    # Wrapper round embedded client +use_ssl+ accessor.
     def use_ssl?
       @client.use_ssl
     end
 
-    # send a request over the wire
+    # Send a request over the wire.
+    # 
+    # This method streams +data+ if it responds to the +stat+ method
+    # (as files do).
+    # 
+    #-- TODO: set timeout on requests in case S3 is unresponsive
     def do_request(method, path, data=nil, headers={})
       req = get_requester(method, path)
       req.chunk_size = @chunk_size
@@ -89,92 +135,97 @@ module S3
 
     end
 
-    # get
-    def do_get(path='/', headers={})
-      do_request('GET', path, headers)
-    end
-
-    # head
-    def do_head(path='/', headers={})
-      do_request('HEAD', path, nil, headers)
-    end
-
-    # post
-    def do_post(path='/', data=nil, headers={})
-      do_request('POST', path, data, headers)
-    end
-
-    # put
-    def do_put(path='/', data=nil, headers={})
-      do_request('PUT', path, data, headers)
-    end
-
-    # delete
-    def do_delete(path, headers={})
-      do_request('DELETE', path, nil, headers)
-    end
-
-    # return an instance of an appropriate request class
+    # Return an instance of an appropriate request class.
     def get_requester(method, path)
-      raise S3Exception::UnsupportedHTTPMethod, "The #{method} HTTP method is not supported" if !(METHOD_VERBS.include?(method))
+      raise S33rException::UnsupportedHTTPMethod, "The #{method} HTTP method is not supported" if !(METHOD_VERBS.include?(method))
       eval("HTTP::" + method[0,1].upcase + method[1..-1].downcase + ".new('#{path}')")
     end
 
-    # list all buckets
-    def list_all_buckets
+    # List all buckets.
+    def list_buckets
       do_get('/')
     end
 
-    # list entries in a bucket
-    # query_params: hash of options on the bucket listing, passed as querystring parameters to S3
-    # :prefix => 'some_string' : restrict results to keys beginning with 'some_string'
-    # :marker => 'some_string' : restict results to keys occurring lexicographically after 'some_string'
-    # :max_keys => 1000 : return at most this number of keys (maximum possible value is 1000)
-    # :delimiter => 'some_string' :
+    # List entries in a bucket.
+    # 
+    # +query_params+: hash of options on the bucket listing request, passed as querystring parameters to S3 
+    # (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/).
+    # * <tt>:prefix => 'some_string'</tt>: restrict results to keys beginning with 'some_string'
+    # * <tt>:marker => 'some_string'</tt>: restict results to keys occurring lexicographically after 'some_string'
+    # * <tt>:max_keys => 1000</tt>: return at most this number of keys (maximum possible value is 1000)
+    # * <tt>:delimiter => 'some_string'</tt>: keys containing the same string between prefix and the delimiter
+    #   are rolled up into a CommonPrefixes element inside the response
     def list_bucket(bucket_name, query_params={})
       if query_params[:max_keys]
         max_keys = query_params[:max_keys].to_i
-        raise S3Exception::BucketListingMaxKeysError, "max_keys option to list bucket cannot be > #{BUCKET_LIST_MAX_MAX_KEYS}" \
+        raise S33rException::BucketListingMaxKeysError, "max_keys option to list bucket cannot be > #{BUCKET_LIST_MAX_MAX_KEYS}" \
         if max_keys > BUCKET_LIST_MAX_MAX_KEYS
 
         # take out the max_keys parameter and move it to max-keys
         query_params['max-keys'] = query_params.delete(:max_keys)
       end
-      do_get("/#{bucket_name}" + generate_querystring(query_params))
+      
+      resp = do_get("/#{bucket_name}" + generate_querystring(query_params))
+      bucket_listing = BucketListing.new(resp.body)
+      
+      [resp, bucket_listing]
     end
 
-    # create a bucket
+    # Create a bucket.
     def create_bucket(bucket_name, headers={})
       do_put("/#{bucket_name}", nil, headers)
     end
 
-    # delete a bucket
-    # TODO: enable deletion of keys inside the bucket
-    # TODO: maybe delete keys matching a partial path
-    def delete_bucket(bucket_name, headers={})
+    # Delete a bucket.
+    # 
+    # +options+ hash can contain the following:
+    # * <tt>:force => true</tt>: delete all keys within the bucket then delete the bucket itself
+    #-- TODO: maybe delete keys matching a partial path
+    def delete_bucket(bucket_name, headers={}, options={})
+      if true == options[:force]
+        _, bucket_listing = list_bucket(bucket_name)
+        bucket_listing.contents.each_value do |obj|
+          delete_resource(bucket_name, obj.key)
+        end
+      end
+      
       do_delete("/#{bucket_name}", headers)
     end
 
-    # return true if bucket exists
+    # Returns true if bucket exists.
     def bucket_exists?(bucket_name)
-      list_bucket(bucket_name).ok?
+      do_head("/#{bucket_name}").ok?
+    end
+    
+    # Fetch a resource and return a fleshed-out S3Object instance.
+    def get_resource(bucket_name, resource_key, headers={})
+      do_get("/#{bucket_name}/#{resource_key}", headers)
     end
 
-    # put some generic resource onto S3
+    # Put some generic resource onto S3.
     def put_resource(bucket_name, resource_key, data, headers={})
       do_put("/#{bucket_name}/" + "#{CGI::escape(resource_key)}", data, headers)
     end
 
-    # put a string onto S3
+    # Put a string onto S3.
     def put_text(string, bucket_name, resource_key, headers={})
       headers["Content-Type"] = "text/plain"
       put_resource(bucket_name, resource_key, string, headers)
     end
 
-    # put a file onto S3
-    # options: to simplify setting of some headers with specific meaning to S3
-    # :render_as_attachment => true: set the Content-Disposition for this file to "attachment" and set 
-    # the default filename for saving the file (when accessed by a web browser) to _filename_
+    # Put a file onto S3.
+    # 
+    # If +resource_key+ is nil, the filename is used as the key instead.
+    # 
+    # +headers+ sets some headers with the request; useful if you have an odd file type
+    # not recognised by the mimetypes library, and want to explicitly set the Content-Type header.
+    # 
+    # +options+ hash simplifies setting some headers with specific meaning to S3:
+    # * <tt>:render_as_attachment => true</tt>: set the Content-Disposition for this file to "attachment" and set 
+    #   the default filename for saving the file (when accessed by a web browser) to +filename+; this
+    #   turns the file into a download when opened in a browser, rather than trying to render it inline.
+    #   
+    # Note that this method uses a handle to the file, so it can be streamed in chunks to S3.
     def put_file(filename, bucket_name, resource_key=nil, headers={}, options={})
       # default to the file path as the resource key if none explicitly set
       if resource_key.nil?
@@ -186,7 +237,7 @@ module S3
         headers['Content-Disposition'] = "attachment; filename=#{File.basename(filename)}"
       end
 
-      # content type is explicitly set in the headers
+      # content type is explicitly set in the headers, so apply to request
       if headers[:content_type]
         # use the first MIME type corresponding to this content type string
         # (MIME::Types returns an array of possible MIME types)
@@ -205,14 +256,40 @@ module S3
       end
     end
     
-    # TODO: delete resource by bucket and key
-    def delete_resource(bucket_name, resource_key)
+    # Delete a resource from S3.
+    def delete_resource(bucket_name, resource_key, headers={})
+      do_delete("/#{bucket_name}/#{resource_key}", headers)
     end
 
-    # add any default headers which should be sent with every request from the client;
-    # any headers passed into this method override the defaults in @client_headers
+    # Add any default headers which should be sent with every request from the client.
+    # 
+    # +headers+ is a hash of headers already set up. Any headers passed in here 
+    # override the defaults in +client_headers+.
+    # 
+    # Returns +headers+ with the content of +client_headers+ merged in.
     def add_client_headers(headers)
-      headers.merge!(@client_headers) { |key, arg, default| arg }
+      headers.merge!(client_headers) { |key, arg, default| arg }
+    end
+    
+    protected
+    def do_get(path='/', headers={})
+      do_request('GET', path, headers)
+    end
+
+    def do_head(path='/', headers={})
+      do_request('HEAD', path, nil, headers)
+    end
+
+    def do_post(path='/', data=nil, headers={})
+      do_request('POST', path, data, headers)
+    end
+
+    def do_put(path='/', data=nil, headers={})
+      do_request('PUT', path, data, headers)
+    end
+
+    def do_delete(path, headers={})
+      do_request('DELETE', path, nil, headers)
     end
 
   end