s33r 0.3.1 → 0.4

data/lib/s33r/builder.rb

@@ -0,0 +1,20 @@
+require 'builder'
+
+module S33r
+  # Variant of XmlMarkup which explicitly orders attributes.
+  class OrderlyXmlMarkup < Builder::XmlMarkup
+  
+    # Override Builder _insert_attributes so attributes are ordered.
+    def _insert_attributes(attrs, order=[])
+      return if attrs.nil?
+      order.each do |k|
+        v = attrs[k]
+        @target << %{ #{k}="#{_attr_value(v)}"} if v
+      end
+      attrs_sorted = attrs.sort { |x,y| x.to_s <=> y.to_s }
+      attrs_sorted.each do |k, v|
+        @target << %{ #{k}="#{_attr_value(v)}"} unless order.member?(k)
+      end
+    end
+  end
+end

data/lib/s33r/client.rb

@@ -1,5 +1,9 @@
 require 'net/https'
 require 'cgi'
+require 'erb'
+require 'yaml'
+require File.join(File.dirname(__FILE__), 's3_acl')
+require File.join(File.dirname(__FILE__), 's33r_exception')
 
 module S33r
   include Net
@@ -15,6 +19,7 @@ module S33r
   class Client
     include S33r
 
+    # S3 keys.
     attr_accessor :aws_access_key, :aws_secret_access_key
     
     # Size of data chunk to be sent per request when putting data.
@@ -22,6 +27,15 @@ module S33r
     
     # Headers which should be sent with every request by default (unless overridden).
     attr_accessor :client_headers
+    
+    # Whether client should use SSL.
+    attr_accessor :use_ssl
+    
+    # Whether client dumps headers from requests.
+    attr_accessor :dump_requests
+    
+    # Default log bucket location.
+    attr_accessor :log_bucket
 
     # Configure either an SSL-enabled or plain HTTP client.
     # (If using SSL, no verification of server certificate is performed.)
@@ -30,20 +44,15 @@ module S33r
     # * <tt>:use_ssl => false</tt>: only use plain HTTP for connections
     # * <tt>:dump_requests => true</tt>: dump each request's initial line and headers to STDOUT
     def initialize(aws_access_key, aws_secret_access_key, options={})
-      if false == options[:use_ssl]
-        @client = HTTP.new(HOST, NON_SSL_PORT)
-        @client.use_ssl = false
-      else
-        @client = HTTP.new(HOST, PORT)
-        # turn off SSL certificate verification
-        @client.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        @client.use_ssl = true
-      end
+      @use_ssl = true
+      @use_ssl = false if (false == options[:use_ssl])
       
       @dump_requests = (true == options[:dump_requests])
 
       # set default chunk size for streaming request body
       @chunk_size = DEFAULT_CHUNK_SIZE
+      
+      @log_bucket = options[:log_bucket]
 
       # Amazon S3 developer keys
       @aws_access_key = aws_access_key
@@ -53,10 +62,29 @@ module S33r
       @client_headers = {}
     end
     
+    # Get an HTTP client instance.
+    # 
+    # NB this has been moved here so that client instances are
+    # only instantiated when needed (so Client can be used
+    # as an empty shell when list_buckets is called).
+    def get_client
+      if @use_ssl
+        client = HTTP.new(HOST, PORT)
+        # turn off SSL certificate verification
+        client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        client.use_ssl = true
+      else
+        client = HTTP.new(HOST, NON_SSL_PORT)
+        client.use_ssl = false
+      end
+      
+      client
+    end
+    
     # Initialise client from YAML configuration file
     # (see load_config method for details of acceptable format).
     def Client.init(config_file)
-      aws_access_key, aws_secret_access_key, options, _ = load_config(config_file)
+      aws_access_key, aws_secret_access_key, options = load_config(config_file)
       Client.new(aws_access_key, aws_secret_access_key, options)
     end
     
@@ -64,56 +92,56 @@ module S33r
     # 
     #   :include: test/files/namedbucket_config.yml
     # 
+    # Note that the loader also runs the config. file through ERB, so you can
+    # add dynamic blocks of ERB (Ruby) code into your files.
+    # 
     # The +options+ section contains settings specific to Client and NamedClient instances; +custom+ 
     # contains extra settings specific to your application.
     # +options+ and +custom+ sections can be omitted, but settings for AWS keys are required.
     # 
-    # Returns an array <tt>[aws_access_key, aws_secret_access_key, options, custom]</tt>, where +options+
-    # and +custom+ are hashes.
+    # Returns an array <tt>[aws_access_key, aws_secret_access_key, options]</tt>, where +options+
+    # is a hash.
     def Client.load_config(config_file)
-      require 'yaml'
-      config = YAML::load_file(config_file)
+      config = YAML::load(ERB.new(IO.read(config_file)).result)
       aws_access_key = config['aws_access_key']
       aws_secret_access_key = config['aws_secret_access_key']
       
       options = {}
       options = S33r.keys_to_symbols(config['options']) if config['options']
       
-      custom = {}
-      custom = S33r.keys_to_symbols(config['custom']) if config['custom']
-      
-      [aws_access_key, aws_secret_access_key, options, custom]
-    end
-
-    # Wrapper round embedded client +use_ssl+ accessor.
-    def use_ssl?
-      @client.use_ssl
+      [aws_access_key, aws_secret_access_key, options]
     end
 
     # Send a request over the wire.
     # 
     # This method streams +data+ if it responds to the +stat+ method
     # (as files do).
+    # 
+    # Returns a Net::HTTPResponse instance.
     def do_request(method, path, data=nil, headers={})
       req = get_requester(method, path)
       req.chunk_size = @chunk_size
 
-      # add the S3 headers which are always required
+      # Add the S3 headers which are always required.
       headers = add_default_headers(headers)
 
-      # add any client-specific default headers
+      # Add any client-specific default headers.
       headers = add_client_headers(headers)
 
+      # Generate the S3 authorization header.
       headers['Authorization'] = generate_auth_header_value(method, path, headers,
         @aws_access_key, @aws_secret_access_key)
 
+      # Insert the headers into the request object.
       headers.each do |key, value|
         req[key] = value
       end
 
+      # Add data to the request as a stream.
       if req.request_body_permitted?
-        # for streaming files; NB Content-Length will be set by Net::HTTP
-        # for character-based body content
+        # For streaming files; NB Content-Length will be set by Net::HTTP
+        # for character-based data: this section of code is only used
+        # when reading directly from a file.
         if data.respond_to?(:stat)
           req.body_stream = data
           req['Content-Length'] = data.stat.size.to_s
@@ -127,9 +155,14 @@ module S33r
         puts req.to_s
       end
 
-      @client.start do
-        response = @client.request(req, data)
+      # Run the request.
+      client = get_client
+      client.start do
+        response = client.request(req, data)
+        
+        # Check the response to see whether S3 is down.
         response.check_s3_availability
+        
         response
       end
     end
@@ -141,8 +174,26 @@ module S33r
     end
 
     # List all buckets.
+    # 
+    # Returns an Array of NamedBucket instances.
     def list_buckets
-      do_get('/')
+      bucket_list_xml = do_get('/').body
+      doc = XML.get_xml_doc(S33r.remove_namespace(bucket_list_xml))
+      
+      named_buckets = []
+      
+      doc.find("//Bucket").to_a.each do |node|
+        bucket_name = node.xget('Name')
+        named_buckets << NamedBucket.new(@aws_access_key, @aws_secret_access_key, 
+        {:default_bucket => bucket_name, :dump_request => self.dump_requests})
+      end
+      
+      named_buckets
+    end
+    
+    # List just bucket names.
+    def list
+      list_buckets.map {|bucket| bucket.name}
     end
 
     # List entries in a bucket.
@@ -154,25 +205,42 @@ module S33r
     # * <tt>:max_keys => 1000</tt>: return at most this number of keys (maximum possible value is 1000)
     # * <tt>:delimiter => 'some_string'</tt>: keys containing the same string between prefix and the delimiter
     #   are rolled up into a CommonPrefixes element inside the response
+    # 
+    # NB if you pass a :marker, this takes up one of your :max_keys; so if you are fetching page
+    # two from a bucket, and you want 10 items, you need to set :max_keys to 11.
+    # 
+    # To page through a bucket 10 keys at a time, you can do:
+    #   
+    #   resp, listing = list_bucket('mybucket', :max_keys => 10)
+    #   resp, listing = list_bucket('mybucket', :max_keys => 11, :marker => listing.last_key)
+    #   resp, listing = list_bucket('mybucket', :max_keys => 11, :marker => listing.last_key)
+    #   etc.
+    #   
+    # Note in the example code, +listing+ is a BucketListing instance; call its contents method
+    # to get a hash of the keys in the bucket, along with associated objects.
+    # 
+    # Returns [raw_response, BucketListing instance].
     def list_bucket(bucket_name, query_params={})
       if query_params[:max_keys]
         max_keys = query_params[:max_keys].to_i
         raise S33rException::BucketListingMaxKeysError, "max_keys option to list bucket cannot be > #{BUCKET_LIST_MAX_MAX_KEYS}" \
         if max_keys > BUCKET_LIST_MAX_MAX_KEYS
 
-        # take out the max_keys parameter and move it to max-keys
+        # convert max_keys parameter to :max-keys parameter
         query_params['max-keys'] = query_params.delete(:max_keys)
       end
       
       resp = do_get("/#{bucket_name}" + generate_querystring(query_params))
-      bucket_listing = BucketListing.new(resp.body)
       
-      [resp, bucket_listing]
+      [resp, BucketListing.new(resp.body)]
     end
 
     # Create a bucket.
+    # 
+    # Returns true if response returned a 200 code; false otherwise.
     def create_bucket(bucket_name, headers={})
-      do_put("/#{bucket_name}", nil, headers)
+      resp = do_put("/#{bucket_name}", nil, headers)
+      resp.ok?
     end
 
     # Delete a bucket.
@@ -180,6 +248,7 @@ module S33r
     # +options+ hash can contain the following:
     # * <tt>:force => true</tt>: delete all keys within the bucket then delete the bucket itself
     #-- TODO: maybe delete keys matching a partial path
+    #-- TODO: if multiple pages of keys in buckets, need to get them by page.
     def delete_bucket(bucket_name, headers={}, options={})
       if true == options[:force]
         _, bucket_listing = list_bucket(bucket_name)
@@ -191,14 +260,11 @@ module S33r
       do_delete("/#{bucket_name}", headers)
     end
 
+    # Check whether a bucket exists or not.
+    # 
     # Returns true if bucket exists.
     def bucket_exists?(bucket_name)
-      do_head("/#{bucket_name}").ok?
-    end
-    
-    # Fetch head info for a key in a bucket.
-    def head_resource(bucket_name, resource_key, headers={})
-      do_head("/#{bucket_name}/#{resource_key}", headers)
+      resource_exists?(bucket_name)
     end
     
     # Fetch a resource.
@@ -206,10 +272,139 @@ module S33r
       do_get("/#{bucket_name}/#{resource_key}", headers)
     end
     
+    # Check whether a bucket contains a key.
+    # 
+    # Returns true if resource_key exists inside bucket_name exists.
+    def resource_exists?(bucket_name, resource_key=nil)
+      path = "/#{bucket_name}"
+      path += "/#{resource_key}" unless resource_key.nil?
+      do_head(path).ok?
+    end
+    
+    # Fetch an object.
+    # 
     # TODO: return S3Object
     def get_object(bucket_name, resource_key, headers)
       response = get_resource(bucket_name, resource_key, headers)
     end
+    
+    # Fetch the ACL document for a resource.
+    # 
+    # Returns nil if there is a problem with the resource
+    # (e.g. it doesn't exist).
+    def get_acl(bucket_name, resource_key='')
+      path = s3_acl_path(bucket_name, resource_key)
+      response = do_get(path)
+      if response.ok?
+        S3ACL::ACLDoc.from_xml(response.body)
+      else
+        raise S33rException::MissingResource, "Tried to get an ACL from a non-existent resource [#{path}]"
+      end
+    end
+    
+    # Put the ACL document for a resource.
+    # 
+    # +acl_doc+ is an S33r::S3ACL::ACLDoc instance.
+    # 
+    # Returns true if response had a 200 code, false otherwise.
+    # If you get a 400 Bad Request back, it means a CanonicalUser
+    # could not be identified from the email address.
+    def set_acl(acl_doc, bucket_name, resource_key='')
+      path = s3_acl_path(bucket_name, resource_key)
+      response = do_put(path, acl_doc.to_xml)
+      response.ok?
+    end
+    
+    # Set up logging for a bucket and resource key.
+    # 
+    # +logging_resource+ = a LoggingResource instance.
+    # +bucket_name+ = a bucket to log.
+    # +resource_key+ = a resource to log (if empty, logging
+    # gets added to the bucket).
+    def set_logging(logging_resource, bucket_name, resource_key='')
+      path = s3_logging_path(bucket_name, resource_key)
+      response = do_put(path, logging_resource.to_xml)
+    end
+    
+    # Make a resource public (i.e. grant READ permissions
+    # to the AllUsers group type). NB separate method is used
+    # on buckets, to make all of their content public too.
+    # 
+    # Returns nil if resource does not exist.
+    def make_public(bucket_name, resource_key='')
+      acl = get_acl(bucket_name, resource_key)
+      if !acl.nil? and acl.add_public_read_grants
+        set_acl(acl, bucket_name, resource_key)
+      end
+    end
+    
+    # TODO
+    def make_private
+    end
+    
+    # Make a bucket capable of being a target for access logging.
+    # 
+    # Returns true if the bucket is now a possible log target;
+    # false otherwise.
+    # 
+    #-- TODO: tests
+    def enable_log_target(bucket_name)
+      acl = get_acl(bucket_name)
+      if acl.add_log_target_grants
+        set_acl(acl, bucket_name)
+      end
+      acl.log_targetable?
+    end
+    
+    # Disable permissions for access logging into a bucket.
+    # 
+    # Returns true if the bucket is no longer log targetable;
+    # false if it remains a log target.
+    # 
+    #-- TODO: tests
+    def disable_log_target(bucket_name)
+      acl = get_acl(bucket_name)
+      acl.remove_log_target
+      set_acl(acl, bucket_name)
+      !acl.log_targetable?
+    end
+    
+    # Enable logging for a resource (bucket or key).
+    # 
+    # +log_prefix+ is the prefix for the logs.
+    # +bucket_name+ is the bucket to log.
+    # +log_bucket+ is the bucket to put logs into.
+    # 
+    # options:
+    #   +:for_key => 'key'+ is the (optional) resource to log in the bucket 
+    #   (NB this is not currently supported by S3).
+    #   +:log_prefix => 'prefix'+ is the (optional) log file prefix
+    #   (defaults to bucket_name + '-')
+    # 
+    def enable_logging(bucket_name, log_bucket=nil, options={})
+      log_bucket ||= @log_bucket
+    
+      resource_key = options[:for_key]
+      resource_key ||= ''
+      
+      log_prefix = options[:prefix]
+      log_prefix ||= bucket_name + '-'
+      
+      log_bucket_acl = get_acl(log_bucket)
+      if !(log_bucket_acl.log_targetable?)
+        raise BucketNotLogTargetable, "The bucket #{log_bucket} cannot be specified as a log target"
+      end
+      logging_resource = LoggingResource.new(log_bucket, log_prefix)
+      set_logging(logging_resource, bucket_name, resource_key)
+    end
+    
+    # TODO
+    def disable_logging
+    end
+    
+    # TODO
+    def get_logging
+    end
 
     # Put some generic resource onto S3.
     def put_resource(bucket_name, resource_key, data, headers={})
@@ -266,6 +461,10 @@ module S33r
     end
     
     # Delete a resource from S3.
+    # 
+    # Note that S3 returns the same response code () regardless
+    # of whether the resource was successfully deleted, or didn't exist
+    # in the first place.
     def delete_resource(bucket_name, resource_key, headers={})
       do_delete("/#{bucket_name}/#{resource_key}", headers)
     end
@@ -280,9 +479,8 @@ module S33r
       headers.merge!(client_headers) { |key, arg, default| arg }
     end
     
-    protected
     def do_get(path='/', headers={})
-      do_request('GET', path, headers)
+      do_request('GET', path, nil, headers)
     end
 
     def do_head(path='/', headers={})