s33r 0.2 → 0.3

data/lib/s33r/core.rb

@@ -1,3 +1,14 @@
+require 'base64'
+require 'time'
+require 'net/http'
+require 'net/https'
+require 'openssl'
+require 'xml/libxml'
+
+# Module to handle S3 operations which don't require an internet connection,
+# i.e. data validation and request-building operations;
+# also holds all the constants relating to S3.
+# 
 # Parts of this code are heavily based on Amazon's code. Here's their license:
 #
 #  This software code is made available "AS IS" without warranties of any
@@ -8,35 +19,28 @@
 #  Digital Services, Inc. or its affiliates with respect to your use of
 #  this software code. (c) 2006 Amazon Digital Services, Inc. or its
 #  affiliates.
-
-require 'base64'
-require 'time'
-require 'net/http'
-require 'net/https'
-require 'openssl'
-require 'xml/libxml'
-
-# this module handles operations which don't require an internet connection,
-# i.e. data validation and request building operations;
-# it also holds all the constants relating to S3
-module S3
+module S33r
   HOST = 's3.amazonaws.com'
   PORT = 443
   NON_SSL_PORT = 80
   METADATA_PREFIX = 'x-amz-meta-'
+  # Size of each chunk (in bytes) to be sent per request when putting files.
+  DEFAULT_CHUNK_SIZE = 1048576
   AWS_HEADER_PREFIX = 'x-amz-'
   AWS_AUTH_HEADER_VALUE = "AWS %s:%s"
   INTERESTING_HEADERS = ['content-md5', 'content-type', 'date']
+  # Headers which must be included with every request to S3.
   REQUIRED_HEADERS = ['Content-Type', 'Date']
   CANNED_ACLS = ['private', 'public-read', 'public-read-write', 'authenticated-read']
+  # HTTP methods which S3 will respond to.
   METHOD_VERBS = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE']
-  # maximum number which can be passed in max-keys parameter when GETting bucket list
+  # Maximum number which can be passed in max-keys parameter when GETting bucket list.
   BUCKET_LIST_MAX_MAX_KEYS = 1000
-  # default number of seconds an authenticated URL will last for (15 minutes)
+  # Default number of seconds an authenticated URL will last for (15 minutes).
   DEFAULT_EXPIRY_SECS = 60 * 15
 
-  # builds the canonical string for signing;
-  # modified (slightly) from the Amazon sample code
+  # Build canonical string for signing;
+  # modified (slightly) from the Amazon sample code.
   def generate_canonical_string(method, path, headers={}, expires=nil)
     interesting_headers = {}
     headers.each do |key, value|
@@ -77,14 +81,14 @@ module S3
     return buf
   end
 
-  # get the header value for AWS authentication
+  # Get the value for the AWS authentication header.
   def generate_auth_header_value(method, path, headers, aws_access_key, aws_secret_access_key)
-    raise S3Exception::MethodNotAvailable, "Method %s not available" % method if !METHOD_VERBS.include?(method)
+    raise S33rException::MethodNotAvailable, "Method %s not available" % method if !METHOD_VERBS.include?(method)
 
     # check the headers needed for authentication have been set
     missing_headers = REQUIRED_HEADERS - headers.keys
     if !(missing_headers.empty?)
-      raise S3Exception::MissingRequiredHeaders,
+      raise S33rException::MissingRequiredHeaders,
         "Headers required for AWS auth value are missing: " + missing_headers.join(', ')
     end
 
@@ -94,24 +98,22 @@ module S3
     AWS_AUTH_HEADER_VALUE % [aws_access_key, signature]
   end
 
-  # encode the given string with the aws_secret_access_key, by taking the
-  # hmac sha1 sum, and then base64 encoding it
+  # Encode the given string with the aws_secret_access_key, by taking the
+  # hmac sha1 sum, and then base64 encoding it.
   def generate_signature(aws_secret_access_key, str)
     digest = OpenSSL::HMAC::digest(OpenSSL::Digest::Digest.new("SHA1"), aws_secret_access_key, str)
     Base64.encode64(digest).strip
   end
 
-  # build the headers required with every S3 request
+  # Build the headers required with every S3 request (Date and Content-Type); 
   # options hash can contain extra header settings, as follows:
   # :date and :content_type are required headers, and set to defaults if not supplied
   def add_default_headers(headers, options={})
-
     # set the default headers required by AWS
     missing_headers = REQUIRED_HEADERS - headers.keys
 
     if missing_headers.include?('Content-Type')
-      content_type = options[:content_type] || ''
-      headers['Content-Type'] = content_type
+      headers['Content-Type'] = options[:content_type] || ''
     end
 
     if missing_headers.include?('Date')
@@ -122,8 +124,8 @@ module S3
     headers
   end
 
-  # add metadata headers to the request, correctly prefixing them first
-  # returns the headers with the metadata headers appended
+  # Add metadata headers, correctly prefixing them first.
+  # Returns headers with the metadata headers appended.
   def metadata_headers(headers, metadata={})
     unless metadata.empty?
       metadata.each { |key, value| headers[METADATA_PREFIX + key] = value }
@@ -131,36 +133,34 @@ module S3
     headers
   end
 
-  # add a canned ACL setter
+  # Add a canned ACL setter header.
   def canned_acl_header(canned_acl, headers={})
     unless canned_acl.nil?
       unless CANNED_ACLS.include?(canned_acl)
-        raise S3Exception::UnsupportedCannedACL, "The canned ACL #{canned_acl} is not supported"
+        raise S33rException::UnsupportedCannedACL, "The canned ACL #{canned_acl} is not supported"
       end
       headers[AWS_HEADER_PREFIX + 'acl'] = canned_acl
     end
     headers
   end
 
-  # guess a file's mime type
-  # NB if the mime_type for a file cannot be guessed, "text/plain" is used
+  # Guess a file's mime type.
+  # If the mime_type for a file cannot be guessed, "text/plain" is used.
   def guess_mime_type(file_name)
     mime_type = MIME::Types.type_for(file_name)[0]
-    mime_type = MIME::Types['text/plain'][0] unless mime_type
+    mime_type ||= MIME::Types['text/plain'][0]
     mime_type
   end
 
-  # ensure that a bucket_name is well-formed (no leading or trailing slash)
+  # Ensure that a bucket_name is well-formed (no leading or trailing slash).
   def bucket_name_valid?(bucket_name)
-    if '/' == bucket_name[0,1]
-      raise S3Exception::MalformedBucketName, "Bucket name cannot have a leading slash"
-    elsif '/' == bucket_name[-1,1]
-      raise S3Exception::MalformedBucketName, "Bucket name cannot have a trailing slash"
+    if ('/' == bucket_name[0,1] || '/' == bucket_name[-1,1])
+      raise S33rException::MalformedBucketName, "Bucket name cannot have a leading or trailing slash"
     end
   end
 
-  # convert a hash of name/value pairs to querystring variables
-  # names can be strings or symbols
+  # Convert a hash of name/value pairs to querystring variables.
+  # Name for a variable can be a string or symbol.
   def generate_querystring(pairs={})
     str = ''
     if pairs.size > 0
@@ -169,7 +169,9 @@ module S3
     str
   end
 
-  # put / between args, but only if it's not already there
+  # Build URLs from fragments.
+  # Does similar job to File.join but puts forward slash between arguments
+  # (only if it's not already there).
   def url_join(*args)
     url_start = ''
     url_end = args.join('/')
@@ -185,14 +187,15 @@ module S3
     # replace any multiple forward slashes (except those in the scheme)
     url_end = url_end.gsub(/\/{2,}/, '/')
     
-    return url_start + url_end
+    url_start + url_end
   end
   
-  def s3_absolute_url(bucket_name, resource_key)
+  # The public URL for this key (which only works if public-read ACL is set).
+  def s3_public_url(bucket_name, resource_key)
     "http://" + HOST + '/' + bucket_name + '/' + resource_key
   end
   
-  # generate a gettable URL for an S3 resource key which passes authentication in querystring
+  # Generate a get-able URL for an S3 resource key which passes authentication in querystring.
   # int expires: when the URL expires (seconds since the epoch)
   def s3_authenticated_url(aws_access_key, aws_secret_access_key, bucket_name, resource_key, 
   expires)
@@ -204,6 +207,15 @@ module S3
     querystring = generate_querystring({ 'Signature' => signature, 'Expires' => expires,
     'AWSAccessKeyId' => aws_access_key })
     
-    return s3_absolute_url(bucket_name, resource_key) + querystring
+    return s3_public_url(bucket_name, resource_key) + querystring
+  end
+  
+  # Turn keys in a hash hsh into symbols.
+  # Returns a hash with 'symbolised' keys.
+  def S33r.keys_to_symbols(hsh)
+    symbolised = hsh.inject({}) do |symbolised, key_value|
+      symbolised.merge({key_value[0].to_sym => key_value[1]})
+    end
+    symbolised
   end
 end

data/lib/s33r/libxml_extensions.rb

@@ -1,8 +1,11 @@
-# convenience methods for libxml classes
+# Convenience methods for libxml classes.
 module XML
-  # find first matching element and return its content
-  # xpath: XPath query
-  # returns nil if no element matches xpath
+  # Find first matching element and return its content
+  # (intended for use as a mixed-in method for Document instances).
+  # 
+  # +xpath+: XPath query to run against self.
+  # 
+  # Returns nil if no element matches +xpath+.
   def xget(xpath)
     nodes = self.find(xpath).to_a
     if nodes.empty?
@@ -12,17 +15,19 @@ module XML
     end
   end
 
-  # parse an XML string into an XML::Document instance
+  # Parse an XML string into an XML::Document instance.
   def XML.get_xml_doc(xml_str)
     parser = XML::Parser.new
     parser.string = xml_str
     parser.parse
   end
 
+  # Make custom methods accessible to Document class.
   class Document
     include XML
   end
 
+  # Make custom methods accessible to Node class.
   class Node
     include XML
   end

data/lib/s33r/mimetypes.rb

@@ -1,4 +1,5 @@
-#--
+# TODO: clarify license
+# 
 # MIME::Types for Ruby
 # Version 1.15
 #
@@ -7,7 +8,7 @@
 # $Id: types.rb,v 1.4 2006/02/12 21:27:22 austin Exp $
 #
 # The ChangeLog contains all details on revisions.
-#++
+#
 
 # The namespace for MIME applications, tools, and libraries.
 module MIME

data/lib/s33r/named_bucket.rb

@@ -1,52 +1,117 @@
-# TODO: provide access to metadata on the enclosed bucket_listing
+#-- TODO: provide access to metadata on the enclosed bucket_listing
+require File.join(File.dirname(__FILE__), 'client')
 
-module S3
-  # a client for dealing with a single bucket
+module S33r
+  # Wraps the S33r::Client class to make it more convenient for use with a single bucket.
   class NamedBucket < Client
-    attr_accessor :bucket_name
+    attr_accessor :bucket_name, :strict, :public_contents, :dump_requests
+    
+    # Initialize an instance from a config_file. The config. file can include a separate
+    # +options+ section specifying options specific to NamedBucket instances (see the initialize method
+    # for more details).
+    # Other options are as for S33r::Client.init.
+    def NamedBucket.init(config_file)
+      aws_access_key, aws_secret_access_key, options, _ = super.class.load_config(config_file)
+      NamedBucket.new(aws_access_key, aws_secret_access_key, options)
+    end
   
-    # options available:
-    # :public_contents => true: all items put into bucket are made public (can be overridden per request)
-    # :strict => true: check whether the bucket exists before attempting to initialize
-    def initialize(aws_access_key, aws_secret_access_key, bucket_name, options={}, &block)
-      @bucket_name = bucket_name
+    # Initialize a NamedBucket instance.
+    # 
+    # +options+ is a hash of options for this instance:
+    # * <tt>:default_bucket => 'xxxx'</tt>: name of the bucket this client is attached to.
+    # * <tt>:public_contents => true</tt>: all items put into bucket are made public (can be overridden per request).
+    # * <tt>:strict => true</tt>: check whether the bucket exists before attempting to initialize; initialization \
+    # fails if the bucket does not exist
+    def initialize(aws_access_key, aws_secret_access_key, options={}, &block)    
+      super(aws_access_key, aws_secret_access_key, options)
+      
+      @bucket_name = options[:default_bucket]
 
       # holds a BucketListing instance
       @bucket_listing = nil
 
       # all content should be created as public-read
-      @client_headers.merge!(canned_acl_header('public-read')) if options[:public_contents]
-    
-      super(aws_access_key, aws_secret_access_key, options)
+      @public_contents = (true == options[:public_contents])
+      @client_headers.merge!(canned_acl_header('public-read')) if @public_contents
       
-      if true == options[:strict]
-        raise S3Exception::MissingResource unless bucket_exists?(bucket_name)
+      @strict = (true == options[:strict])
+      if ((@strict && !bucket_exists?(@bucket_name)) || @bucket_name.nil?)
+        raise S33rException::MissingResource, "The specified bucket name #{@bucket_name} does not exist or is invalid"
       end
+      
+      yield self if block_given?
     end
-
-    def metadata
-      # TODO: get bucket metadata from the bucket_listing
+    
+    # Are all objects added to this bucket made public by default?
+    def public_contents?
+      @public_contents
     end
-
-    def contents
-      # TODO: S3Object instances inside the bucket_listing
+    
+    # Is this a strict bucket (i.e. the target bucket must exist on S3)?
+    def strict?
+      @strict
+    end
+    
+    # Get a single object from a bucket as a blob.
+    def [](key)
+      get_resource(@bucket_name, key).body
     end
 
+    # Get a BucketListing instance for the content of this bucket.
     def listing
-      # TODO: build a bucket_listing whose objects are associated with the bucket
-      # and set the @bucket_listing instance variable
-      list_bucket(@bucket_name)
+      _, @bucket_listing = list_bucket(@bucket_name)
+      @bucket_listing
+    end
+    
+    # Get a pretty list of the keys in the bucket.
+    def keys
+      listing.pretty
+    end
+    
+    # List content of the bucket, and attach each item to this bucket as it is yielded.
+    def each_item
+      listing.contents.each_value { |item| item.named_bucket = self; yield item }
+    end
+    
+    # Does this bucket exist?
+    # Returns true if the bucket this NamedBucket is mapped to exists.
+    def exists?
+      bucket_exists?(@bucket_name)
     end
 
+    # Put a string into a key inside the bucket.
     def put_text(string, resource_key, headers={})
       super(string, @bucket_name, resource_key, headers)
     end
 
+    # Put a file into the bucket.
     def put_file(filename, resource_key=nil, headers={}, options={})
       super(filename, @bucket_name, resource_key, headers, options)
     end
     
-    # expires: time in secs since the epoch when 
+    # Put a generic resource (e.g. from a data stream) into the bucket.
+    def put_stream(data, resource_key, headers={})
+      put_resource(@bucket_name, resource_key, data, headers)
+    end
+    
+    # Delete the bucket.
+    def delete(headers={}, options={})
+      delete_bucket(@bucket_name, headers, options)
+    end
+    
+    # Delete an object from the bucket.
+    # 
+    # Returns boolean (true = deletion worked OK).
+    def delete_key(resource_key, headers={})
+      response = delete_resource(@bucket_name, resource_key, headers)
+      listing
+      response.ok?
+    end
+    
+    # Generate an authenticated URL (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/) 
+    # for an object inside this bucket.
+    # 
+    # +expires+: time in secs since the epoch when the link should become invalid.
     def s3_authenticated_url(resource_key, expires=(Time.now.to_i + DEFAULT_EXPIRY_SECS))
       super(@aws_access_key, @aws_secret_access_key, @bucket_name, resource_key, expires)
     end

data/lib/s33r/{s3_exception.rb → s33r_exception.rb}

@@ -1,5 +1,5 @@
-module S3
-  module S3Exception
+module S33r
+  module S33rException
 
     class MethodNotAvailable < Exception
     end

data/lib/s33r/{net_http_overrides.rb → s33r_http.rb}

@@ -1,21 +1,22 @@
 require 'net/http'
 
-# overrides for the default Net::HTTP classes
+# Addtions to the Net::HTTP classes
 
-# add some convenience functions for checking response status
+# Adds some convenience functions for checking response status.
 class Net::HTTPResponse
   attr_accessor :success
+  attr_writer :body
 
   def ok?
     success
   end
 
   def success
-    response.is_a?(Net::HTTPOK)
+    code == '200'
   end
 
   def not_found
-    response.is_a?(Net::HTTPNotFound)
+    code = '404'
   end
 
   def to_s
@@ -23,13 +24,34 @@ class Net::HTTPResponse
   end
 end
 
-# modified to enable larger chunk sizes to be used
-# when streaming data from large files
+# Modification of Net::HTTP base class to enable larger chunk sizes to be used
+# when streaming data from large files.
+# 
+# N.B. does _not_ alter the behaviour of this class unless you set chunk_size
+# using the accessor.
 class Net::HTTPGenericRequest
+  # Size of chunks (in bytes) to send when streaming body data.
   attr_accessor :chunk_size
 
+  # Dump initial line and raw request headers (useful for visual debugging).
+  # 
+  # Switch on request debugging by passing <tt>:dump_requests => true</tt> to S33r::Client
+  # constructor.
+  def to_s
+    str = "*******\n" +
+    "#{self.class::METHOD} #{@path} HTTP/1.1\n" +
+    "Host: #{S33r::HOST}\n"
+
+    self.each_capitalized do |key, value|
+      str += "#{key}: #{value}\n"
+    end
+    str += "*******\n\n"
+    str
+  end
+
   private
-  def send_request_with_body_stream(sock, ver, path, f)
+  # Minor alterations to original source to enable variable chunk sizes.
+  def send_request_with_body_stream(sock, ver, path, f) # :doc:
     raise ArgumentError, "Content-Length not given and Transfer-Encoding is not `chunked'" unless content_length() or chunked?
     unless content_type()
       warn 'net/http: warning: Content-Type did not set; using application/x-www-form-urlencoded' if $VERBOSE
@@ -48,18 +70,4 @@ class Net::HTTPGenericRequest
       end
     end
   end
-end
-
-class Net::HTTPRequest
-  def to_s
-    str = "*******\n" +
-    "#{self.class::METHOD} #{@path} HTTP/1.1\n" +
-    "Host: #{S3::HOST}\n"
-    
-    self.each_capitalized do |key, value|
-      str += "#{key}: #{value}\n"
-    end
-    str += "*******\n\n"
-    str
-  end
 end