s33r 0.1

data/LICENSE.txt

@@ -0,0 +1,9 @@
+This software is distributed under the MIT license (see MIT-LICENSE).
+
+It incorporates code from the mimetypes library (http://raa.ruby-lang.org/project/mime-types/),
+which is under a compatible license (same as Ruby).
+
+It is also heavily based on the sample Ruby code provided by Amazon
+(http://developer.amazonwebservices.com/connect/entry.jspa?externalID=135&categoryID=47).
+
+I picked up a couple of ideas from http://rubyforge.org/projects/rsh3ll/ too.

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+Copyright (C) 2006 Elliot Smith
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.txt

@@ -0,0 +1,16 @@
+This is a first pass at a Ruby library for talking to Amazon's S3 service.
+
+It is less complete than my previous effort (http://townx.org/blog/elliot/s3_rails), but more solid, more elegant,
+and with a test suite (which uses RSpec - see http://rubyforge.org/projects/rspec - I haven't added RSpec as a gem
+dependency (you can ignore the tests if you like), but for reference I developed the tests using 0.5.12).
+
+To use it from inside your Ruby script:
+
+  require 's33r'
+
+If you want to see an example program which uses the library, have a look at bin/s3cli.rb. This is a simple command
+line client which you can use to post a file to S3 and email a link to the file to someone. Useful for sending
+big files to people without clogging their email inbox with enormous files. It is also intended as a demonstration
+of how to use the library. Full instructions are included at the top of the file.
+
+By the way, I've tested this on Linux, but not on Windows or Mac.

data/bin/config.yml

@@ -0,0 +1,5 @@
+access_key: 'youraccesskey'
+secret_key: 'yoursecretkey'
+bucket: 'name-of-bucket'
+from_email: 'elliot@example.com'
+to_email: 'elliot@example.com'

data/bin/s3cli.rb

@@ -0,0 +1,96 @@
+# SIMPLE COMMAND LINE S3 CLIENT
+# You need an Amazon S3 account to use this: get one at http://aws.amazon.com/
+#
+# Create a config. file called based on the config.yml example in this directory;
+# then change the config_file setting below to set the path to your config. file.
+# You will be prompted to create a bucket if your setting for bucket in the config.
+# file is blank.
+#
+# Note that this script doesn't give you any means of deleting anything; you will
+# need to write your own program to do this (I use my own s33r server,
+# which I am currently re-implementing to work with this library).
+#
+# Call this script with two arguments:
+# filename: file to upload; the file is uploaded to S3 and assigned a key
+# based on filename supplied (including path)
+# to_email: email address to send to (optional)
+
+require 'yaml'
+require 'rubygems'
+require_gem 's33r'
+require 'net/smtp'
+
+filename = ARGV[0]
+to_email = ARGV[1]
+config_file = '/home/ell/.s33r'
+if '/path/to/your/config/file' == config_file
+  puts 'Please set the config_file variable to the path to your config. file'
+  exit
+end
+
+# load config. file
+options = YAML::load_file(config_file)
+access_key = options['access_key']
+secret_key = options['secret_key']
+from_email = options['from_email']
+to_email ||= options['to_email']
+bucket = options['bucket']
+
+# check for bucket
+if !bucket
+  require 'readline'
+  bucket = Readline.readline('No bucket found; please enter name for new bucket: ', true)
+  client = S3::Client.new(access_key, secret_key)
+  response = client.create_bucket(bucket, client.canned_acl_header('public-read'))
+  if response.ok?
+    puts 'Created new bucket'
+  else
+    puts 'Could not create bucket (HTTP problem)'
+    exit
+  end
+end
+
+# estimate upload time
+filesize = FileTest.size(filename)
+filesize_mb = (filesize / (1000*1000)).to_i
+secs_per_mb = 12
+str = "Transferring file; size %d bytes" % filesize
+str += " (about %d Mb)" % filesize_mb if filesize_mb > 0
+str += "\nUploading normally takes at least %d seconds per Mb\n" % secs_per_mb
+
+if filesize_mb > 0
+	str += "So this should take about %d seconds" % (filesize_mb * secs_per_mb)
+else
+  str += "So this should be done in no time"
+end
+
+puts str
+
+# time the put
+start_time_secs = Time.now.to_i
+
+# a client pointing at the specified bucket
+S3::NamedBucket.new(access_key, secret_key, bucket, :public_contents => true) do |client|
+  response = client.put_file(filename)
+end
+
+time_taken_secs = Time.now.to_i - start_time_secs
+puts "Aah, it appears to have taken %d seconds" % time_taken_secs
+
+# post-put report
+if response.ok?
+  puts "File #{filename} transferred OK"
+  url = File.join("http://", S3::HOST, client.bucket_name, '/') + filename
+  puts "Available at URL:"
+  puts url
+
+  if to_email
+		message = "From:#{from_email}\r\nTo:#{to_email}\r\nSubject:You were sent a file\r\n\r\nFetch it from\n#{url}"
+
+		Net::SMTP.start('localhost') do |smtp|
+  		smtp.send_message message, from_email, to_email
+		end
+  end
+else
+  puts 'File transfer failed'
+end

data/lib/s33r.rb

@@ -0,0 +1,7 @@
+require 's33r/core'
+require 's33r/client'
+require 's33r/list_bucket_result'
+require 's33r/net_http_overrides'
+require 's33r/named_bucket'
+require 's33r/s3_exception'
+require 's33r/external/mimetypes'

data/lib/s33r/client.rb

@@ -0,0 +1,188 @@
+require 'net/https'
+require 'cgi'
+
+# this is a very thin layer over the S3 API
+# TODO: need to wrap XML returned into object representation
+module S3
+  include Net
+
+  class Client
+    include S3
+    attr_accessor :chunk_size, :default_headers
+
+    def initialize(aws_access_key, aws_secret_access_key)
+      @client = HTTP.new(HOST, PORT)
+
+      # turn off SSL certificate verification
+      @client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+      # always use SSL
+      @client.use_ssl = true
+
+      # set default chunk size for streaming request body (1 Mb)
+      @chunk_size = 1048576
+
+      # Amazon S3 developer keys
+      @aws_access_key = aws_access_key
+      @aws_secret_access_key = aws_secret_access_key
+
+      # headers sent with every request made by this client
+      @client_headers = {}
+    end
+
+    # send a request over the wire
+    def do_request(method, path, data=nil, headers={})
+      req = get_requester(method, path)
+      req.chunk_size = @chunk_size
+
+      # add the S3 headers which are always required
+      headers = add_default_headers(headers)
+
+      # add any client-specific default headers
+      headers = add_client_headers(headers)
+
+      headers['Authorization'] = generate_auth_header_value(method, path, headers,
+        @aws_access_key, @aws_secret_access_key)
+
+      headers.each do |key, value|
+        req[key] = value
+      end
+
+      @client.start do
+        if req.request_body_permitted?
+          # for streaming large files
+          if data.respond_to?(:read)
+            req.body_stream = data
+            req['Content-Length'] = data.stat.size.to_s
+            return @client.request(req, nil)
+          # simple text strings etc.
+          else
+            return @client.request(req, data)
+          end
+        else
+          return @client.request(req)
+        end
+      end
+
+    end
+
+    # get
+    def do_get(path='/', headers={})
+      do_request('GET', path, headers)
+    end
+
+    # head
+    def do_head(path='/', headers={})
+      do_request('HEAD', path, headers)
+    end
+
+    # post
+    def do_post(path='/', data=nil, headers={})
+      do_request('POST', path, data, headers)
+    end
+
+    # put
+    def do_put(path='/', data=nil, headers={})
+      do_request('PUT', path, data, headers)
+    end
+
+    # return an instance of an appropriate request class
+    def get_requester(method, path)
+      raise S3Exception::UnsupportedHTTPMethod, "The #{method} HTTP method is not supported" if !(METHOD_VERBS.include?(method))
+      eval("HTTP::" + method[0,1].upcase + method[1..-1].downcase + ".new('#{path}')")
+    end
+
+    # convert a hash of name/value pairs to querystring variables
+    def get_querystring(pairs={})
+      str = ''
+      if pairs.size > 0
+        str += "?" + pairs.map { |key, value| "#{key}=#{CGI::escape(value.to_s)}" }.join('&')
+      end
+      str
+    end
+
+    # list all buckets
+    def list_all_buckets
+      do_get('/')
+    end
+
+    # create a bucket
+    def create_bucket(bucket_name, headers={})
+      bucket_name_valid?(bucket_name)
+      bucket_exists?(bucket_name)
+      do_put("/#{bucket_name}", nil, headers)
+    end
+
+    # list entries in a bucket
+    def list_bucket(bucket_name)
+      bucket_name_valid?(bucket_name)
+      bucket_exists?(bucket_name)
+      do_get("/#{bucket_name}")
+    end
+
+    # put some resource onto S3
+    def put_resource(data, bucket_name, resource_key, headers={})
+      do_put(File.join("/#{bucket_name}", "#{CGI::escape(resource_key)}"), data, headers)
+    end
+
+    # put a string onto S3
+    def put_text(string, bucket_name, resource_key, headers={})
+      headers["Content-Type"] = "text/plain"
+      put_resource(bucket_name, resource_key, string, headers)
+    end
+
+    # put a file onto S3
+    def put_file(filename, bucket_name, resource_key=nil, headers={})
+      # default to the file path as the resource key if none explicitly set
+      if resource_key.nil?
+        resource_key = filename
+      end
+
+      # content type is explicitly set in the headers
+      if headers[:content_type]
+        # use the first MIME type corresponding to this content type string
+        # (MIME::Types returns an array of possible MIME types)
+        mime_type = MIME::Types[headers[:content_type]][0]
+      else
+        mime_type = guess_mime_type(filename)
+      end
+      content_type = mime_type.simplified
+      headers['Content-Type'] = content_type
+      headers['Content-Transfer-Encoding'] = 'binary' if mime_type.binary?
+
+      # the data we want to put (handle to file, so we can stream from it)
+      File.open(filename) do |data|
+        # send the put request
+        put_resource(data, bucket_name, resource_key, headers)
+      end
+    end
+
+    # guess a file's mime type
+    # NB if the mime_type for a file cannot be guessed, "text/plain" is used
+    def guess_mime_type(filename)
+      mime_type = MIME::Types.type_for(filename)[0]
+      mime_type ||= MIME::Types['text/plain'][0]
+      mime_type
+    end
+
+    # ensure that a bucket_name is well-formed
+    def bucket_name_valid?(bucket_name)
+      if '/' == bucket_name[0,1]
+        raise S3Exception::MalformedBucketName, "Bucket name cannot have a leading slash"
+      end
+    end
+
+    # TODO: proper check for existence of bucket;
+    # throw error if bucket does not exist (see bucket_name_valid? for example)
+    def bucket_exists?(bucket_name)
+      false
+    end
+
+    # add any default headers which should be sent with every request from the client;
+    # any headers passed into this method override the defaults in @client_headers
+    def add_client_headers(headers)
+      headers.merge!(@client_headers) { |key, arg, default| arg }
+    end
+
+  end
+end

data/lib/s33r/core.rb

@@ -0,0 +1,131 @@
+require 'base64'
+require 'time'
+require 'net/https'
+require 'openssl'
+
+module S3
+  HOST = 's3.amazonaws.com'
+  PORT = 443
+  METADATA_PREFIX = 'x-amz-meta-'
+  AWS_HEADER_PREFIX = 'x-amz-'
+  AWS_AUTH_HEADER_VALUE = "AWS %s:%s"
+  INTERESTING_HEADERS = ['content-md5', 'content-type', 'date']
+  REQUIRED_HEADERS = ['Content-Type', 'Date']
+  CANNED_ACLS = ['private', 'public-read', 'public-read-write', 'authenticated-read']
+  METHOD_VERBS = ['GET', 'PUT', 'HEAD', 'POST']
+
+  # builds the canonical string for signing;
+  # modified (slightly) from the Amazon sample code
+  def generate_canonical_string(method, path, headers={}, expires=nil)
+    interesting_headers = {}
+    headers.each do |key, value|
+      lk = key.downcase
+      if (INTERESTING_HEADERS.include?(lk) or lk =~ /^#{AWS_HEADER_PREFIX}/o)
+        interesting_headers[lk] = value
+      end
+    end
+
+    # these fields get empty strings if they don't exist.
+    interesting_headers['content-type'] ||= ''
+    interesting_headers['content-md5'] ||= ''
+
+    # if you're using expires for query string auth, then it trumps date
+    if not expires.nil?
+      interesting_headers['date'] = expires
+    end
+
+    buf = "#{method}\n"
+    interesting_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value|
+      if key =~ /^#{AWS_HEADER_PREFIX}/o
+        buf << "#{key}:#{value}\n"
+      else
+        buf << "#{value}\n"
+      end
+    end
+
+    # ignore everything after the question mark...
+    buf << path.gsub(/\?.*$/, '')
+
+    # ...unless there is an acl or torrent parameter
+    if path =~ /[&?]acl($|&|=)/
+      buf << '?acl'
+    elsif path =~ /[&?]torrent($|&|=)/
+      buf << '?torrent'
+    end
+
+    return buf
+  end
+
+  # get the header value for AWS authentication
+  def generate_auth_header_value(method, path, headers, aws_access_key, aws_secret_access_key)
+    raise S3Exception::MethodNotAvailable, "Method %s not available" % method if !METHOD_VERBS.include?(method)
+
+    # check the headers needed for authentication have been set
+    missing_headers = REQUIRED_HEADERS - headers.keys
+    if !(missing_headers.empty?)
+      raise S3Exception::MissingRequiredHeaders,
+        "Headers required for AWS auth value are missing: " + missing_headers.join(', ')
+    end
+
+    # get the AWS header
+    canonical_string = generate_canonical_string(method, path, headers)
+    signature = generate_signature(aws_secret_access_key, canonical_string)
+    AWS_AUTH_HEADER_VALUE % [aws_access_key, signature]
+  end
+
+  # encode the given string with the aws_secret_access_key, by taking the
+  # hmac sha1 sum, and then base64 encoding it
+  def generate_signature(aws_secret_access_key, str, urlencode=false)
+    digest = OpenSSL::HMAC::digest(OpenSSL::Digest::Digest.new("SHA1"), aws_secret_access_key, str)
+    Base64.encode64(digest).strip
+  end
+
+  # build the headers required with every S3 request
+  # options hash can contain extra header settings, as follows:
+  # :date and :content_type are required headers, and set to defaults if not supplied
+  def add_default_headers(headers, options={})
+
+    # set the default headers required by AWS
+    missing_headers = REQUIRED_HEADERS - headers.keys
+
+    if missing_headers.include?('Content-Type')
+      content_type = options[:content_type] || ''
+      headers['Content-Type'] = content_type
+    end
+
+    if missing_headers.include?('Date')
+      date = options[:date] || Time.now
+      headers['Date'] = date.httpdate
+    end
+
+    headers
+  end
+
+  # add metadata headers to the request, correctly prefixing them first
+  # returns the headers with the metadata headers appended
+  def metadata_headers(headers, metadata={})
+    unless metadata.empty?
+      metadata.each { |key, value| headers[METADATA_PREFIX + key] = value }
+    end
+    headers
+  end
+
+  # add a canned ACL setter
+  def canned_acl_header(canned_acl, headers={})
+    unless canned_acl.nil?
+      unless CANNED_ACLS.include?(canned_acl)
+        raise S3Exception::UnsupportedCannedACL, "The canned ACL #{canned_acl} is not supported"
+      end
+      headers[AWS_HEADER_PREFIX + 'acl'] = canned_acl
+    end
+    headers
+  end
+
+  # guess a file's mime type
+  # NB if the mime_type for a file cannot be guessed, "text/plain" is used
+  def guess_mime_type(file_name)
+    mime_type = MIME::Types.type_for(file_name)[0]
+    mime_type = MIME::Types['text/plain'][0] unless mime_type
+    mime_type
+  end
+end