s33r 0.1 → 0.2

data/LICENSE.txt

@@ -4,6 +4,19 @@ It incorporates code from the mimetypes library (http://raa.ruby-lang.org/projec
 which is under a compatible license (same as Ruby).
 
 It is also heavily based on the sample Ruby code provided by Amazon
-(http://developer.amazonwebservices.com/connect/entry.jspa?externalID=135&categoryID=47).
+(http://developer.amazonwebservices.com/connect/entry.jspa?externalID=135&categoryID=47). Here's
+the Amazon licence:
 
-I picked up a couple of ideas from http://rubyforge.org/projects/rsh3ll/ too.
+***
+This software code is made available "AS IS" without warranties of any
+kind.  You may copy, display, modify and redistribute the software
+code either by itself or as incorporated into your code; provided that
+you do not remove any proprietary notices.  Your use of this software
+code is at your own risk and you waive any claim against Amazon
+Digital Services, Inc. or its affiliates with respect to your use of
+this software code. (c) 2006 Amazon Digital Services, Inc. or its
+affiliates.
+***
+
+I picked up a couple of ideas from http://rubyforge.org/projects/rsh3ll/ too, e.g.
+swopping out the HMAC library for OpenSSL

data/README.txt

@@ -13,4 +13,7 @@ line client which you can use to post a file to S3 and email a link to the file
 big files to people without clogging their email inbox with enormous files. It is also intended as a demonstration
 of how to use the library. Full instructions are included at the top of the file.
 
-By the way, I've tested this on Linux, but not on Windows or Mac.
+By the way, I've tested this on Linux, but not on Windows or Mac.
+
+Thanks for bug reports to:
+Keaka (http://keakaj.com/products.htm)

data/bin/config.yml

@@ -1,5 +1,5 @@
 access_key: 'youraccesskey'
 secret_key: 'yoursecretkey'
 bucket: 'name-of-bucket'
-from_email: 'elliot@example.com'
-to_email: 'elliot@example.com'
+from_email: 'you@example.com'
+to_email: 'you@example.com'

data/bin/s3cli.rb

@@ -19,14 +19,13 @@ require 'yaml'
 require 'rubygems'
 require_gem 's33r'
 require 'net/smtp'
+require 'time'
+require 'parsedate'
 
 filename = ARGV[0]
 to_email = ARGV[1]
+# set to the path to your config. file
 config_file = '/home/ell/.s33r'
-if '/path/to/your/config/file' == config_file
-  puts 'Please set the config_file variable to the path to your config. file'
-  exit
-end
 
 # load config. file
 options = YAML::load_file(config_file)
@@ -36,12 +35,27 @@ from_email = options['from_email']
 to_email ||= options['to_email']
 bucket = options['bucket']
 
+# expires can be a date/time string or 'forever';
+# if not set, defaults to 15 minutes
+expires = options['default_expires']
+base_expires = Time.now.to_i
+if 'forever' == expires
+  # 20 years (same as forever in computer terms)
+  expires = base_expires + (60 * 60 * 24 * 365.25 * 20).to_i
+elsif expires.is_a?(String)
+  datetime_parts = ParseDate.parsedate(expires)
+  expires = Time.gm(*datetime_parts).to_i
+elsif !expires
+  # default to 15 minutes from now if not set
+  expires = base_expires + (15*60)
+end
+
 # check for bucket
 if !bucket
   require 'readline'
   bucket = Readline.readline('No bucket found; please enter name for new bucket: ', true)
   client = S3::Client.new(access_key, secret_key)
-  response = client.create_bucket(bucket, client.canned_acl_header('public-read'))
+  response = client.create_bucket(bucket)
   if response.ok?
     puts 'Created new bucket'
   else
@@ -70,27 +84,32 @@ puts str
 start_time_secs = Time.now.to_i
 
 # a client pointing at the specified bucket
-S3::NamedBucket.new(access_key, secret_key, bucket, :public_contents => true) do |client|
-  response = client.put_file(filename)
-end
-
-time_taken_secs = Time.now.to_i - start_time_secs
-puts "Aah, it appears to have taken %d seconds" % time_taken_secs
-
-# post-put report
-if response.ok?
-  puts "File #{filename} transferred OK"
-  url = File.join("http://", S3::HOST, client.bucket_name, '/') + filename
-  puts "Available at URL:"
-  puts url
-
-  if to_email
-		message = "From:#{from_email}\r\nTo:#{to_email}\r\nSubject:You were sent a file\r\n\r\nFetch it from\n#{url}"
-
-		Net::SMTP.start('localhost') do |smtp|
-  		smtp.send_message message, from_email, to_email
-		end
+S3::NamedBucket.new(access_key, secret_key, bucket) do |c|
+  response = c.put_file(filename, filename)
+  url = c.s3_authenticated_url(filename, expires)
+  
+  time_taken_secs = Time.now.to_i - start_time_secs
+  puts "Aah, it appears to have taken %d seconds" % time_taken_secs
+  
+  # post-put report
+  if response.ok?
+    puts "File #{filename} transferred OK"
+    
+    pretty_expires = Time.at(expires).httpdate
+    
+    puts "Available at URL:"
+    puts url
+    puts "This link will expire at %s" % pretty_expires
+  
+    if to_email
+  		message = "From:#{from_email}\r\nTo:#{to_email}\r\nSubject:You were sent a file\r\n\r\nFetch it from\n#{url}\n\n" +
+  		"This link will expire at %s" % pretty_expires
+  
+  		Net::SMTP.start('localhost') do |smtp|
+    		smtp.send_message message, from_email, to_email
+  		end
+    end
+  else
+    puts 'File transfer failed'
   end
-else
-  puts 'File transfer failed'
 end

data/lib/s33r.rb

@@ -1,7 +1,2 @@
-require 's33r/core'
-require 's33r/client'
-require 's33r/list_bucket_result'
-require 's33r/net_http_overrides'
-require 's33r/named_bucket'
-require 's33r/s3_exception'
-require 's33r/external/mimetypes'
+base = File.join(File.dirname(__FILE__), 's33r')
+Dir[File.join(base, "*.rb")].each { |f| require f }

data/lib/s33r/bucket_listing.rb

@@ -0,0 +1,141 @@
+# use prefix to limit keys to some subset of all available keys;
+# use delimiter to group keys
+
+require 'date'
+require 'xml/libxml'
+
+module S3
+  class BucketListing
+    attr_reader :listing_xml
+
+    # properties of the bucket are publically read-only;
+    # can only be reset by resetting the listing_xml via listing_xml=
+    # (which parses the bucket listing XML)
+    attr_reader :name, :delimiter, :prefix, :marker, :max_keys, :is_truncated, :contents
+
+    def initialize(bucket_listing_xml)
+      @contents = {}
+      @common_prefixes = {}
+      set_listing_xml(bucket_listing_xml)
+    end
+
+    def set_listing_xml(bucket_listing_xml)
+      # remove the namespace declaration: libxml doesn't like it
+      bucket_listing_xml.gsub!(/ xmlns="http:\/\/s3.amazonaws.com\/doc\/2006-03-01\/"/, '')
+      @listing_xml = bucket_listing_xml
+      parse_listing(bucket_listing_xml)
+    rescue
+      message = "Cannot create bucket listing from supplied XML"
+      message += " (was nil)" if bucket_listing_xml.nil?
+      raise S3Exception::InvalidBucketListing, message
+    end
+
+    def parse_listing(bucket_listing_xml)
+      doc = XML.get_xml_doc(bucket_listing_xml)
+
+      prop_setter = lambda do |prop, path|
+        node = doc.find("//ListBucketResult/#{path}").to_a.first
+        self.send("#{prop}=", node.content) if node
+      end
+
+      # metadata
+      prop_setter.call(:name, 'Name')
+      prop_setter.call(:delimiter, 'Delimiter')
+      prop_setter.call(:prefix, 'Prefix')
+      prop_setter.call(:marker, 'Marker')
+      prop_setter.call(:max_keys, 'MaxKeys')
+      prop_setter.call(:is_truncated, 'IsTruncated')
+
+      # contents
+      doc.find('//Contents').to_a.each do |node|
+        obj = S3Object.new
+        obj.set_from_node(node)
+        @contents[obj.key] = obj
+      end
+    end
+    
+    # return an object in this bucket
+    def [](key)
+      @contents[key]
+    end
+
+    # setters
+    private
+      def name=(val)
+        @name = string_prop_normalise(val)
+      end
+
+      def prefix=(val)
+        @prefix = string_prop_normalise(val)
+      end
+
+      def delimiter=(val)
+        @delimiter = string_prop_normalise(val)
+      end
+
+      def marker=(val)
+        @marker = string_prop_normalise(val)
+      end
+
+      def max_keys=(val)
+        @max_keys = val.to_i
+      end
+
+      def is_truncated=(val)
+        @is_truncated = false
+        @is_truncated = true if ('true' == val || true == val || 'True' == val)
+      end
+
+      # normalise string properties:
+      # if value for XML element is nil, set property to empty string
+      def string_prop_normalise(val)
+        val = '' if val.nil?
+        val
+      end
+
+  end
+
+  class S3Object
+    attr_reader :bucket_name
+    attr_accessor :key, :last_modified, :etag, :size, :owner, :storage_class
+
+    # bucket: NamedBucket which is the parent of this object
+    def initialize(named_bucket=nil)
+      @named_bucket = named_bucket
+    end
+
+    # set properties from an XML string
+    # if using this, should be an XML document containing a <Contents> element as root
+    def set_from_xml_string(xml_str)
+      doc = XML.get_xml_doc(xml_str)
+      set_from_node(doc)
+    end
+
+    # set properties from an XML document
+    # XML::Document doc: XML document to parse
+    # options:
+    # :full => true: set all properties (default is to only set those in bucket listing)
+    def set_from_node(doc, options={})
+      @key = doc.xget('Key')
+      @last_modified = DateTime.parse(doc.xget('LastModified'))
+      @etag = doc.xget('ETag').gsub("\"", "")
+      @size = doc.xget('Size').to_i
+      @owner = S3User.new(doc.find('Owner').to_a.first)
+      
+      if options[:full]
+        # TODO: if setting from a full object listing (GET on a resource key),
+        # do additional field setting here (e.g. x-amz-meta- headers)
+      end
+    end
+  end
+
+  class S3User
+    attr_accessor :id, :display_name
+
+    # XML::Document owner_xml_doc: <Owner> node from inside ListBucketResult <Contents> element
+    def initialize(owner_xml_doc)
+      @id = owner_xml_doc.xget('//ID')
+      @display_name = owner_xml_doc.xget('//DisplayName')
+    end
+  end
+end

data/lib/s33r/client.rb

@@ -1,23 +1,35 @@
 require 'net/https'
 require 'cgi'
 
-# this is a very thin layer over the S3 API
-# TODO: need to wrap XML returned into object representation
 module S3
   include Net
 
+  # the client actually performs operations over the network,
+  # using the core to build request headers and content;
+  # only client-specific headers are managed here: other headers
+  # can be handled by the core
+  # TODO: need to wrap XML returned into object representation
+  # TODO: use customisable thread pool for requests
   class Client
     include S3
-    attr_accessor :chunk_size, :default_headers
-
-    def initialize(aws_access_key, aws_secret_access_key)
-      @client = HTTP.new(HOST, PORT)
 
-      # turn off SSL certificate verification
-      @client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    attr_accessor :chunk_size, :default_headers
 
-      # always use SSL
-      @client.use_ssl = true
+    # options: hash of optional client config.
+    # :ssl => false : only use plain HTTP
+    # :dump_requests => true: dump each request's initial line and headers
+    def initialize(aws_access_key, aws_secret_access_key, options={})
+      if false == options[:ssl]
+        @client = HTTP.new(HOST, NON_SSL_PORT)
+        @client.use_ssl = false
+      else
+        @client = HTTP.new(HOST, PORT)
+        # turn off SSL certificate verification
+        @client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        @client.use_ssl = true
+      end
+      
+      @dump_requests = (true == options[:dump_requests])
 
       # set default chunk size for streaming request body (1 Mb)
       @chunk_size = 1048576
@@ -28,6 +40,13 @@ module S3
 
       # headers sent with every request made by this client
       @client_headers = {}
+      
+      yield self if block_given?
+    end
+
+    # wrapper round embedded client use_ssl accessor
+    def use_ssl?
+      @client.use_ssl
     end
 
     # send a request over the wire
@@ -48,20 +67,24 @@ module S3
         req[key] = value
       end
 
-      @client.start do
-        if req.request_body_permitted?
-          # for streaming large files
-          if data.respond_to?(:read)
-            req.body_stream = data
-            req['Content-Length'] = data.stat.size.to_s
-            return @client.request(req, nil)
-          # simple text strings etc.
-          else
-            return @client.request(req, data)
-          end
-        else
-          return @client.request(req)
+      if req.request_body_permitted?
+        # for streaming files; NB Content-Length will be set by Net::HTTP
+        # for character-based body content
+        if data.respond_to?(:stat)
+          req.body_stream = data
+          req['Content-Length'] = data.stat.size.to_s
+          data = nil
         end
+      else
+        data = nil
+      end
+      
+      if @dump_requests
+        puts req.to_s
+      end
+
+      @client.start do
+        return @client.request(req, data)
       end
 
     end
@@ -73,7 +96,7 @@ module S3
 
     # head
     def do_head(path='/', headers={})
-      do_request('HEAD', path, headers)
+      do_request('HEAD', path, nil, headers)
     end
 
     # post
@@ -86,43 +109,60 @@ module S3
       do_request('PUT', path, data, headers)
     end
 
+    # delete
+    def do_delete(path, headers={})
+      do_request('DELETE', path, nil, headers)
+    end
+
     # return an instance of an appropriate request class
     def get_requester(method, path)
       raise S3Exception::UnsupportedHTTPMethod, "The #{method} HTTP method is not supported" if !(METHOD_VERBS.include?(method))
       eval("HTTP::" + method[0,1].upcase + method[1..-1].downcase + ".new('#{path}')")
     end
 
-    # convert a hash of name/value pairs to querystring variables
-    def get_querystring(pairs={})
-      str = ''
-      if pairs.size > 0
-        str += "?" + pairs.map { |key, value| "#{key}=#{CGI::escape(value.to_s)}" }.join('&')
-      end
-      str
-    end
-
     # list all buckets
     def list_all_buckets
       do_get('/')
     end
 
+    # list entries in a bucket
+    # query_params: hash of options on the bucket listing, passed as querystring parameters to S3
+    # :prefix => 'some_string' : restrict results to keys beginning with 'some_string'
+    # :marker => 'some_string' : restict results to keys occurring lexicographically after 'some_string'
+    # :max_keys => 1000 : return at most this number of keys (maximum possible value is 1000)
+    # :delimiter => 'some_string' :
+    def list_bucket(bucket_name, query_params={})
+      if query_params[:max_keys]
+        max_keys = query_params[:max_keys].to_i
+        raise S3Exception::BucketListingMaxKeysError, "max_keys option to list bucket cannot be > #{BUCKET_LIST_MAX_MAX_KEYS}" \
+        if max_keys > BUCKET_LIST_MAX_MAX_KEYS
+
+        # take out the max_keys parameter and move it to max-keys
+        query_params['max-keys'] = query_params.delete(:max_keys)
+      end
+      do_get("/#{bucket_name}" + generate_querystring(query_params))
+    end
+
     # create a bucket
     def create_bucket(bucket_name, headers={})
-      bucket_name_valid?(bucket_name)
-      bucket_exists?(bucket_name)
       do_put("/#{bucket_name}", nil, headers)
     end
 
-    # list entries in a bucket
-    def list_bucket(bucket_name)
-      bucket_name_valid?(bucket_name)
-      bucket_exists?(bucket_name)
-      do_get("/#{bucket_name}")
+    # delete a bucket
+    # TODO: enable deletion of keys inside the bucket
+    # TODO: maybe delete keys matching a partial path
+    def delete_bucket(bucket_name, headers={})
+      do_delete("/#{bucket_name}", headers)
+    end
+
+    # return true if bucket exists
+    def bucket_exists?(bucket_name)
+      list_bucket(bucket_name).ok?
     end
 
-    # put some resource onto S3
-    def put_resource(data, bucket_name, resource_key, headers={})
-      do_put(File.join("/#{bucket_name}", "#{CGI::escape(resource_key)}"), data, headers)
+    # put some generic resource onto S3
+    def put_resource(bucket_name, resource_key, data, headers={})
+      do_put("/#{bucket_name}/" + "#{CGI::escape(resource_key)}", data, headers)
     end
 
     # put a string onto S3
@@ -132,11 +172,19 @@ module S3
     end
 
     # put a file onto S3
-    def put_file(filename, bucket_name, resource_key=nil, headers={})
+    # options: to simplify setting of some headers with specific meaning to S3
+    # :render_as_attachment => true: set the Content-Disposition for this file to "attachment" and set 
+    # the default filename for saving the file (when accessed by a web browser) to _filename_
+    def put_file(filename, bucket_name, resource_key=nil, headers={}, options={})
       # default to the file path as the resource key if none explicitly set
       if resource_key.nil?
         resource_key = filename
       end
+      
+      # set Content-Disposition header
+      if options[:render_as_attachment]
+        headers['Content-Disposition'] = "attachment; filename=#{File.basename(filename)}"
+      end
 
       # content type is explicitly set in the headers
       if headers[:content_type]
@@ -153,29 +201,12 @@ module S3
       # the data we want to put (handle to file, so we can stream from it)
       File.open(filename) do |data|
         # send the put request
-        put_resource(data, bucket_name, resource_key, headers)
-      end
-    end
-
-    # guess a file's mime type
-    # NB if the mime_type for a file cannot be guessed, "text/plain" is used
-    def guess_mime_type(filename)
-      mime_type = MIME::Types.type_for(filename)[0]
-      mime_type ||= MIME::Types['text/plain'][0]
-      mime_type
-    end
-
-    # ensure that a bucket_name is well-formed
-    def bucket_name_valid?(bucket_name)
-      if '/' == bucket_name[0,1]
-        raise S3Exception::MalformedBucketName, "Bucket name cannot have a leading slash"
+        put_resource(bucket_name, resource_key, data, headers)
       end
     end
-
-    # TODO: proper check for existence of bucket;
-    # throw error if bucket does not exist (see bucket_name_valid? for example)
-    def bucket_exists?(bucket_name)
-      false
+    
+    # TODO: delete resource by bucket and key
+    def delete_resource(bucket_name, resource_key)
     end
 
     # add any default headers which should be sent with every request from the client;