s33r 0.1 → 0.2

data/lib/s33r/core.rb

@@ -1,18 +1,39 @@
+# Parts of this code are heavily based on Amazon's code. Here's their license:
+#
+#  This software code is made available "AS IS" without warranties of any
+#  kind.  You may copy, display, modify and redistribute the software
+#  code either by itself or as incorporated into your code; provided that
+#  you do not remove any proprietary notices.  Your use of this software
+#  code is at your own risk and you waive any claim against Amazon
+#  Digital Services, Inc. or its affiliates with respect to your use of
+#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
+#  affiliates.
+
 require 'base64'
 require 'time'
+require 'net/http'
 require 'net/https'
 require 'openssl'
+require 'xml/libxml'
 
+# this module handles operations which don't require an internet connection,
+# i.e. data validation and request building operations;
+# it also holds all the constants relating to S3
 module S3
   HOST = 's3.amazonaws.com'
   PORT = 443
+  NON_SSL_PORT = 80
   METADATA_PREFIX = 'x-amz-meta-'
   AWS_HEADER_PREFIX = 'x-amz-'
   AWS_AUTH_HEADER_VALUE = "AWS %s:%s"
   INTERESTING_HEADERS = ['content-md5', 'content-type', 'date']
   REQUIRED_HEADERS = ['Content-Type', 'Date']
   CANNED_ACLS = ['private', 'public-read', 'public-read-write', 'authenticated-read']
-  METHOD_VERBS = ['GET', 'PUT', 'HEAD', 'POST']
+  METHOD_VERBS = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE']
+  # maximum number which can be passed in max-keys parameter when GETting bucket list
+  BUCKET_LIST_MAX_MAX_KEYS = 1000
+  # default number of seconds an authenticated URL will last for (15 minutes)
+  DEFAULT_EXPIRY_SECS = 60 * 15
 
   # builds the canonical string for signing;
   # modified (slightly) from the Amazon sample code
@@ -75,7 +96,7 @@ module S3
 
   # encode the given string with the aws_secret_access_key, by taking the
   # hmac sha1 sum, and then base64 encoding it
-  def generate_signature(aws_secret_access_key, str, urlencode=false)
+  def generate_signature(aws_secret_access_key, str)
     digest = OpenSSL::HMAC::digest(OpenSSL::Digest::Digest.new("SHA1"), aws_secret_access_key, str)
     Base64.encode64(digest).strip
   end
@@ -128,4 +149,61 @@ module S3
     mime_type = MIME::Types['text/plain'][0] unless mime_type
     mime_type
   end
+
+  # ensure that a bucket_name is well-formed (no leading or trailing slash)
+  def bucket_name_valid?(bucket_name)
+    if '/' == bucket_name[0,1]
+      raise S3Exception::MalformedBucketName, "Bucket name cannot have a leading slash"
+    elsif '/' == bucket_name[-1,1]
+      raise S3Exception::MalformedBucketName, "Bucket name cannot have a trailing slash"
+    end
+  end
+
+  # convert a hash of name/value pairs to querystring variables
+  # names can be strings or symbols
+  def generate_querystring(pairs={})
+    str = ''
+    if pairs.size > 0
+      str += "?" + pairs.map { |key, value| "#{key}=#{CGI::escape(value.to_s)}" }.join('&')
+    end
+    str
+  end
+
+  # put / between args, but only if it's not already there
+  def url_join(*args)
+    url_start = ''
+    url_end = args.join('/')
+    
+    # string index where the scheme of the URL (xxxx://) ends
+    scheme_ends_at = (url_end =~ /:\/\//)
+    unless scheme_ends_at.nil?
+      scheme_ends_at = scheme_ends_at + 1
+      url_start = url_end[0..scheme_ends_at]
+      url_end = url_end[(scheme_ends_at + 1)..-1]
+    end
+    
+    # replace any multiple forward slashes (except those in the scheme)
+    url_end = url_end.gsub(/\/{2,}/, '/')
+    
+    return url_start + url_end
+  end
+  
+  def s3_absolute_url(bucket_name, resource_key)
+    "http://" + HOST + '/' + bucket_name + '/' + resource_key
+  end
+  
+  # generate a gettable URL for an S3 resource key which passes authentication in querystring
+  # int expires: when the URL expires (seconds since the epoch)
+  def s3_authenticated_url(aws_access_key, aws_secret_access_key, bucket_name, resource_key, 
+  expires)
+    path = '/' + bucket_name + '/' + resource_key
+    
+    canonical_string = generate_canonical_string('GET', path, {}, expires)
+    signature = generate_signature(aws_secret_access_key, canonical_string)
+    
+    querystring = generate_querystring({ 'Signature' => signature, 'Expires' => expires,
+    'AWSAccessKeyId' => aws_access_key })
+    
+    return s3_absolute_url(bucket_name, resource_key) + querystring
+  end
 end

data/lib/s33r/libxml_extensions.rb

@@ -0,0 +1,29 @@
+# convenience methods for libxml classes
+module XML
+  # find first matching element and return its content
+  # xpath: XPath query
+  # returns nil if no element matches xpath
+  def xget(xpath)
+    nodes = self.find(xpath).to_a
+    if nodes.empty?
+      return nil
+    else
+      return nodes.first.content
+    end
+  end
+
+  # parse an XML string into an XML::Document instance
+  def XML.get_xml_doc(xml_str)
+    parser = XML::Parser.new
+    parser.string = xml_str
+    parser.parse
+  end
+
+  class Document
+    include XML
+  end
+
+  class Node
+    include XML
+  end
+end

data/lib/s33r/named_bucket.rb

@@ -1,19 +1,40 @@
+# TODO: provide access to metadata on the enclosed bucket_listing
+
 module S3
   # a client for dealing with a single bucket
   class NamedBucket < Client
     attr_accessor :bucket_name
-
-    # TODO: check bucket exists before setting it
+  
     # options available:
-    # :public_contents => true: all items put into bucket are made public
+    # :public_contents => true: all items put into bucket are made public (can be overridden per request)
+    # :strict => true: check whether the bucket exists before attempting to initialize
     def initialize(aws_access_key, aws_secret_access_key, bucket_name, options={}, &block)
-      super(aws_access_key, aws_secret_access_key)
       @bucket_name = bucket_name
+
+      # holds a BucketListing instance
+      @bucket_listing = nil
+
+      # all content should be created as public-read
       @client_headers.merge!(canned_acl_header('public-read')) if options[:public_contents]
-      yield self if block_given?
+    
+      super(aws_access_key, aws_secret_access_key, options)
+      
+      if true == options[:strict]
+        raise S3Exception::MissingResource unless bucket_exists?(bucket_name)
+      end
     end
 
-    def root
+    def metadata
+      # TODO: get bucket metadata from the bucket_listing
+    end
+
+    def contents
+      # TODO: S3Object instances inside the bucket_listing
+    end
+
+    def listing
+      # TODO: build a bucket_listing whose objects are associated with the bucket
+      # and set the @bucket_listing instance variable
       list_bucket(@bucket_name)
     end
 
@@ -21,8 +42,13 @@ module S3
       super(string, @bucket_name, resource_key, headers)
     end
 
-    def put_file(filename, resource_key=nil, headers={})
-      super(filename, @bucket_name, resource_key, headers)
+    def put_file(filename, resource_key=nil, headers={}, options={})
+      super(filename, @bucket_name, resource_key, headers, options)
+    end
+    
+    # expires: time in secs since the epoch when 
+    def s3_authenticated_url(resource_key, expires=(Time.now.to_i + DEFAULT_EXPIRY_SECS))
+      super(@aws_access_key, @aws_secret_access_key, @bucket_name, resource_key, expires)
     end
   end
 end

data/lib/s33r/net_http_overrides.rb

@@ -1,5 +1,8 @@
 require 'net/http'
 
+# overrides for the default Net::HTTP classes
+
+# add some convenience functions for checking response status
 class Net::HTTPResponse
   attr_accessor :success
 
@@ -11,11 +14,17 @@ class Net::HTTPResponse
     response.is_a?(Net::HTTPOK)
   end
 
+  def not_found
+    response.is_a?(Net::HTTPNotFound)
+  end
+
   def to_s
     body
   end
 end
 
+# modified to enable larger chunk sizes to be used
+# when streaming data from large files
 class Net::HTTPGenericRequest
   attr_accessor :chunk_size
 
@@ -39,4 +48,18 @@ class Net::HTTPGenericRequest
       end
     end
   end
+end
+
+class Net::HTTPRequest
+  def to_s
+    str = "*******\n" +
+    "#{self.class::METHOD} #{@path} HTTP/1.1\n" +
+    "Host: #{S3::HOST}\n"
+    
+    self.each_capitalized do |key, value|
+      str += "#{key}: #{value}\n"
+    end
+    str += "*******\n\n"
+    str
+  end
 end

data/lib/s33r/s3_exception.rb

@@ -1,20 +1,29 @@
 module S3
   module S3Exception
-  
+
     class MethodNotAvailable < Exception
     end
-    
+
     class MissingRequiredHeaders < Exception
     end
-    
+
     class UnsupportedCannedACL < Exception
     end
-    
+
     class UnsupportedHTTPMethod < Exception
     end
-    
+
     class MalformedBucketName < Exception
     end
 
+    class MissingResource < Exception
+    end
+
+    class BucketListingMaxKeysError < Exception
+    end
+
+    class InvalidBucketListing < Exception
+    end
+
   end
 end

data/lib/s33r/sync.rb

@@ -0,0 +1,11 @@
+module S3
+  module Sync
+
+    # get the MD5 checksum for a file (comparable to the ETag on S3 objects)
+    def md5sum(filename)
+      f = File.open(filename).binmode
+      OpenSSL::Digest::MD5.hexdigest(f.read)
+    end
+
+  end
+end

data/test/cases/spec_bucket_listing.rb

@@ -0,0 +1,125 @@
+base = File.dirname(__FILE__)
+require base + '/../s3_test_constants'
+require 'set'
+
+context 'S3 bucket listing' do
+  setup do
+    xml_file = File.join(base, '../files/bucket_listing.xml')
+    @with_bucket_listing_xml = File.open(xml_file) { |f| f.read }
+    xml_file2 = File.join(base, '../files/bucket_listing2.xml')
+    @with_bucket_listing_xml2 = File.open(xml_file2) { |f| f.read }
+    xml_file3 = File.join(base, '../files/bucket_listing3.xml')
+    @with_empty_bucket_listing_xml = File.open(xml_file3) { |f| f.read }
+    xml_file4 = File.join(base, '../files/bucket_listing_broken.xml')
+    @with_broken_bucket_listing_xml = File.open(xml_file4) { |f| f.read }
+    @bucket_listing = BucketListing.new(@with_bucket_listing_xml)
+    @bucket_properties = %w(name prefix marker max_keys is_truncated)
+    @bucket_property_setters = @bucket_properties.map { |prop| prop + "=" }
+  end
+
+  specify 'can only be created if bucket listing XML supplied' do
+    lambda { BucketListing.new }.should.raise ArgumentError
+  end
+
+  specify 'cannot be created from invalid XML' do
+    lambda { BucketListing.new(nil) }.should.raise S3Exception::InvalidBucketListing
+  end
+
+  specify 'should recover gracefully from broken bucket listing XML' do
+    lambda { BucketListing.new(@with_broken_bucket_listing_xml) }.should.raise S3Exception::InvalidBucketListing
+  end
+
+  specify 'should cope if bucket is empty (i.e. no <Contents> elements)' do
+    @bucket_listing.set_listing_xml(@with_empty_bucket_listing_xml)
+  end
+
+  specify 'can return the raw XML used to initialise it' do
+    @bucket_listing.listing_xml.should.equal(@with_bucket_listing_xml)
+  end
+
+  specify 'can have the bucket listing XML reset' do
+    @bucket_listing.should.respond_to :set_listing_xml
+  end
+
+  specify 'should present bucket metadata as typed properties' do
+    @bucket_listing.name.should.equal('testingtesting')
+    @bucket_listing.prefix.should.equal('')
+    @bucket_listing.marker.should.equal('')
+    @bucket_listing.max_keys.should.equal(1000)
+    @bucket_listing.is_truncated.should.equal(false)
+    @bucket_listing.delimiter.should.be nil
+  end
+
+  specify 'when listing XML is reset, should update all properties correctly' do
+    @bucket_listing.set_listing_xml(@with_bucket_listing_xml2)
+    @bucket_listing.name.should.equal('testing2')
+    @bucket_listing.prefix.should.equal('/home/ell/')
+    @bucket_listing.marker.should.equal('')
+    @bucket_listing.max_keys.should.equal(100)
+    @bucket_listing.delimiter.should.equal('/')
+    @bucket_listing.is_truncated.should.equal(false)
+  end
+
+  specify 'should provide private setters for metadata' do
+    # all private methods
+    methods = @bucket_listing.private_methods.to_set
+    # names of the setters
+    setters = @bucket_property_setters
+    # all methods should be a superset of the private setters
+    methods.superset?(setters.to_set).should.equal true
+  end
+
+  specify 'should store resources (<Contents> elements) in a hash' do
+    @bucket_listing.contents.size.should_be 10
+    first_obj = @bucket_listing.contents['/home/ell/dir1/four.txt']
+    first_obj.should_be_instance_of S3Object
+  end
+
+  specify 'should enable access to metadata for a resource by its key' do
+    obj = @bucket_listing['/home/ell/dir1/four.txt']
+    obj.should_be_instance_of S3Object
+    obj.etag.should_equal '24ce59274b89287b3960c184153ac24b'
+  end
+  
+  specify 'should be able to build a full representation given full object XML from GET on resource key' do
+    fix
+  end
+
+  specify 'should provide easy access to <CommonPrefixes> elements as a hash' do
+    fix
+  end
+end
+
+context 'S3 object' do
+  setup do
+    @s3_object_xml = File.open(File.join(base, '../files/s3_object.xml')).read
+    @s3obj = S3Object.new
+    @s3obj.set_from_xml_string(@s3_object_xml)
+  end
+
+  specify 'can be initialised from XML fragment with correct data types' do
+    @s3obj.key.should.equal '/home/ell/dir1/four.txt'
+    d = @s3obj.last_modified
+    [d.year, d.month, d.day, d.hour, d.min, d.sec].should.equal [2006, 8, 19, 22, 53, 29]
+    @s3obj.etag.should.equal '24ce59274b89287b3960c184153ac24b'
+    @s3obj.size.should.equal 14
+  end
+
+  specify 'should treat the owner as an object in his/her own right' do
+    [@s3obj.owner.id, @s3obj.owner.display_name].should.equal \
+    ['56efddfead5aa65da942f156fb2b294f44d78fd932d701331edc5fba19620fd4', 'elliotsmith3']
+    @s3obj.owner.should_be_instance_of S3User
+  end
+
+  specify 'can be associated with a NamedBucket' do
+    fix
+  end
+
+  specify 'can be saved by proxing through the NamedBucket it is associated with' do
+    fix
+  end
+
+  specify 'cannot be saved unless associated with a NamedBucket' do
+    fix
+  end
+end

data/test/cases/spec_core.rb

@@ -0,0 +1,130 @@
+require File.dirname(__FILE__) + '/../s3_test_constants'
+
+context 'S3 core' do
+
+  setup do
+    @for_request_method = 'PUT'
+    @for_request_path = "/quotes/nelson"
+    @for_request_headers = {
+    "Content-Md5" => "c8fdb181845a4ca6b8fec737b3581d76",
+    "Content-Type" => "text/html",
+    "Date" => "Thu, 17 Nov 2005 18:49:58 GMT",
+    "X-Amz-Meta-Author" => "foo@bar.com",
+    "X-Amz-Magic" => "abracadabra"
+    }
+
+    # create broken request header hash
+    @for_incomplete_headers = @for_request_headers.clone.delete_if do |key,value|
+      'Content-Type' == key or 'Date' == key
+    end
+
+    @correct_canonical_string = "PUT\nc8fdb181845a4ca6b8fec737b3581d76\n" +
+    "text/html\nThu, 17 Nov 2005 18:49:58 GMT\nx-amz-magic:abracadabra\n" +
+    "x-amz-meta-author:foo@bar.com\n/quotes/nelson"
+    @correct_signature = "jZNOcbfWmD/A/f3hSvVzXZjM2HU="
+    @correct_auth_header = "AWS #{S3Testing::ACCESS_KEY}:#{@correct_signature}"
+
+    @with_invalid_bucket_name = '/badbucket'
+    @with_invalid_bucket_name2 = 'badbucket/'
+    
+    @correct_authenticated_url = "http://s3.amazonaws.com/quotes/nelson?Signature=vjbyPxybdZaNmGa%2ByT272YEAiv4%3D&"+
+    "AWSAccessKeyId=44CF9590006BF252F707&Expires=1141889120"
+  end
+
+  specify 'should generate correct canonical strings' do
+    generate_canonical_string(@for_request_method, @for_request_path,
+    @for_request_headers).should.equal @correct_canonical_string
+  end
+
+  specify 'should generate correct signatures' do
+    generate_signature(S3Testing::SECRET_ACCESS_KEY,
+    @correct_canonical_string).should.equal @correct_signature
+  end
+
+  specify 'should generate correct auth headers' do
+    generate_auth_header_value(@for_request_method, @for_request_path, @for_request_headers,
+    S3Testing::ACCESS_KEY, S3Testing::SECRET_ACCESS_KEY).should.equal @correct_auth_header
+  end
+
+  specify 'should not generate auth header if bad HTTP method passed' do
+    lambda { generate_auth_header_value('duff', nil, nil, nil, nil) }.should.raise \
+    S3Exception::MethodNotAvailable
+  end
+
+  specify 'should not generate auth header if required headers missing' do
+    lambda { generate_auth_header_value('PUT', '/', @for_incomplete_headers,
+    nil, nil) }.should.raise S3Exception::MissingRequiredHeaders
+  end
+
+  specify 'when generating auth header, should allow addition of Date and Content-Type headers' do
+    now = Time.now
+
+    fixed_headers = add_default_headers(@for_incomplete_headers, :date => now,
+    :content_type => 'text/html')
+
+    fixed_headers['Date'].should.equal now.httpdate
+    fixed_headers['Content-Type'].should.equal 'text/html'
+  end
+
+  specify 'should not generate canned ACL header if invalid canned ACL supplied' do
+    lambda { canned_acl_header('duff') }.should.raise \
+    S3Exception::UnsupportedCannedACL
+  end
+
+  specify 'should correctly add canned ACL headers' do
+    new_headers = canned_acl_header('private', { 'Content-Type' => 'text/html' })
+    new_headers.should.have(2).keys
+    new_headers.keys.should.include 'Content-Type'
+    new_headers.keys.should.include 'x-amz-acl'
+    new_headers['x-amz-acl'].should.equal 'private'
+  end
+
+  specify 'should set sensible defaults for missing Content-Type and Date headers' do
+    fixed_headers = add_default_headers(@for_incomplete_headers)
+    fixed_headers['Content-Type'].should.equal ''
+    fixed_headers.include?('Date').should.not.be nil
+  end
+
+  specify 'should default to text/plain mimetype for unknown file types' do
+    guess_mime_type('hello.madeup').should.equal('text/plain')
+  end
+
+  specify 'should recognise invalid bucket names' do
+    lambda { bucket_name_valid?(@with_invalid_bucket_name) }.should.raise \
+    S3Exception::MalformedBucketName
+    lambda { bucket_name_valid?(@with_invalid_bucket_name2) }.should.raise \
+    S3Exception::MalformedBucketName
+  end
+
+  specify 'should return empty string if generating querystring with no key/value pairs' do
+    generate_querystring({}).should_equal ''
+  end
+
+  specify 'should correctly format querystring key/value pairs' do
+    generate_querystring({'message' => 'Hello world', 'id' => 1, 'page' => '[2,4]'}).should_equal \
+    '?message=Hello+world&id=1&page=%5B2%2C4%5D'
+  end
+
+  specify 'should allow symbols as names for querystring variables when generating querystrings' do
+    generate_querystring({ :prefix => '/home/ell' }).should.equal('?prefix=%2Fhome%2Fell')
+  end
+
+  specify 'should convert integers to strings when generating querystrings' do
+    generate_querystring({ 'max-keys' => 400 }).should.equal('?max-keys=400')
+  end
+
+  specify 'should be able to construct correct URL paths from path fragments' do
+    url_join('http://', 'localhost/', 'test').should.equal('http://localhost/test')
+    url_join('ftp://test.com', '/public/', '/files').should.equal('ftp://test.com/public/files')
+    url_join('http://localhost/', '/index.html').should.equal('http://localhost/index.html')
+    url_join('http://', 'localhost', 'test').should.equal('http://localhost/test')
+    url_join('http://localhost:8080/test/', '/path/', 'element.html').should.equal('http://localhost:8080/test/path/element.html')
+    url_join('/test/', 'index.html').should.equal '/test/index.html'
+  end
+  
+  specify 'should generate URLs with authentication parameters' do
+    s3_authenticated_url(S3Testing::ACCESS_KEY, S3Testing::SECRET_ACCESS_KEY, 'quotes', 'nelson', \
+    1141889120).should.equal @correct_authenticated_url
+  end
+
+end