s3lib 0.1.0

data/lib/object_take1.rb

@@ -0,0 +1,150 @@
+# object.rb
+require 'rexml/document'
+
+module S3Lib
+  
+  class ObjectDoesNotExist < StandardError
+  end
+  
+  class ObjectAccessForbidden < StandardError
+  end  
+  
+  class NoContentError < S3Lib::S3ResponseError
+  end
+
+  class S3Object
+    
+    DEFAULT_CONTENT_TYPE = 'binary/octect-stream'
+    
+    attr_reader :key, :bucket
+        
+    # This is just an alias for S3Object.new
+    def self.find(bucket, key, options = {})
+      S3Object.new(bucket, key, options)
+    end
+    
+    def self.create(bucket, key, value = "", options = {})    
+      options.merge!({:body => value || "", 'content-type' => DEFAULT_CONTENT_TYPE})
+      begin
+        response = S3Lib.request(:put, S3Object.url(bucket, key), options)
+      rescue S3Lib::S3ResponseError => error
+        case error.amazon_error_type
+        when 'NoSuchBucket': raise S3Lib::BucketNotFoundError.new("The bucket '#{bucket}' does not exist.", error.io, error.s3requester)
+        when 'AccessDenied': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by someone else.", error.io, error.s3requester)
+        when 'MissingContentLength': raise S3Lib::NoContentError.new("You must provide a value to put in the object.\nUsage: S3Lib::S3Object.create(bucket, key, value, options)", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+      response.status[0] == "200" ? S3Object.new(bucket, key) : false
+    end
+    
+    # Delete an object given the object's bucket and key.
+    # No error will be raised if the object does not exist.
+    def self.delete(bucket, key, options = {})
+      begin
+        response = S3Lib.request(:delete, S3Object.url(bucket, key), options)
+      rescue S3Lib::S3ResponseError => error
+        case error.amazon_error_type
+        when 'NoSuchBucket': raise S3Lib::BucketNotFoundError.new("The bucket '#{bucket}' does not exist.", error.io, error.s3requester)
+        when 'NotSignedUp': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by somebody else", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+      puts response.status
+    end
+    
+    def delete
+      S3Object.delete(@bucket, @key, @options)
+    end
+    
+    def self.value(bucket, key, options = {})
+      request = S3Object.object_request(:get, S3Object.url(bucket, key), options)      
+      request.read
+    end
+    
+    # bucket can be either a Bucket object or a string containing the bucket's name
+    def self.url(bucket, key)
+      bucket_name = bucket.respond_to?(:name) ? bucket.name : bucket
+      File.join(bucket_name, key)
+    end
+    
+    # Both metadata and value are loaded lazily if options[:lazy_load] is true
+    # This is used by Bucket.find so you don't make a request for every object in the bucket
+    # The bucket can be either a bucket object or a string containing the bucket's name
+    # The key is a string.
+    def initialize(bucket, key, options = {})
+      options.merge!(:lazy_load => false)
+      bucket = Bucket.find(bucket) unless bucket.respond_to?(:name)
+      @bucket = bucket
+      @key = key
+      @options = options
+      get_metadata unless options.delete(:lazy_load)      
+    end   
+    
+    def url
+      S3Object.url(@bucket.name, @key)
+    end 
+    
+    def metadata
+      @metadata || get_metadata
+    end
+    
+    def value
+      @value || get_value
+    end
+    
+    def value=(value)
+      S3Object.object_request(:put, value)
+      @value = value
+      refresh_metadata
+    end
+    
+    def refresh
+      get_value
+    end
+    
+    def refresh_metadata
+      get_metadata
+    end
+    
+    def content_type
+      metadata["content-type"]
+    end
+    
+    def etag
+      metadata["etag"]
+    end
+    
+    private
+    
+    def self.object_request(verb, url, options = {})
+      begin
+        S3Lib.request(verb, url, options)
+      rescue S3Lib::S3ResponseError => error
+        case error.amazon_error_type
+        when 'NoSuchBucket': raise S3Lib::BucketNotFoundError.new("The bucket '#{bucket}' does not exist.", error.io, error.s3requester)
+        when 'NotSignedUp': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by somebody else", error.io, error.s3requester)
+        when 'AccessDenied': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by someone else.", error.io, error.s3requester)
+        when 'MissingContentLength': raise S3Lib::NoContentError.new("You must provide a value to put in the object.\nUsage: S3Lib::S3Object.create(bucket, key, value, options)", error.io, error.s3requester)          
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+    end
+        
+    def get_metadata
+      request = S3Object.object_request(:head, url)
+      @metadata = request.meta
+    end
+    
+    def get_value
+      request = S3Object.object_request(:get, url)
+      @metadata = request.meta      
+      @value = request.read
+    end
+    
+  end
+
+end

data/lib/object_with_acl_mixin.rb

@@ -0,0 +1,131 @@
+# s3_object.rb
+
+module S3Lib
+
+  class S3Object
+    
+    DEFAULT_CONTENT_TYPE = 'binary/octect-stream'
+    
+    attr_reader :key, :bucket
+    
+    include S3Lib::AclAccess    
+        
+    # This is just an alias for S3Object.new
+    def self.find(bucket, key, options = {})
+      S3Object.new(bucket, key, options)
+    end
+    
+    def self.create(bucket, key, value = "", options = {})    
+			# translate from :access to 'x-amz-acl'
+      options['x-amz-acl'] = options.delete(:access) if options[:access]      
+      options.merge!({:body => value || "", 'content-type' => DEFAULT_CONTENT_TYPE})
+      response = S3Object.object_request(:put, S3Object.url(bucket, key), options)
+      response.status[0] == "200" ? S3Object.new(bucket, key, options) : false
+    end
+    
+    # Delete an object given the object's bucket and key.
+    # No error will be raised if the object does not exist.
+    def self.delete(bucket, key, options = {})
+      S3Object.object_request(:delete, S3Object.url(bucket, key), options)
+    end
+    
+    def delete
+      S3Object.delete(@bucket, @key, @options)
+    end
+    
+    def self.value(bucket, key, options = {})
+      request = S3Object.object_request(:get, S3Object.url(bucket, key), options)      
+      request.read
+    end
+    
+    # Both metadata and value are loaded lazily if options[:lazy_load] is true
+    # This is used by Bucket.find so you don't make a request for every object in the bucket
+    # The bucket can be either a bucket object or a string containing the bucket's name
+    # The key is a string.
+    def initialize(bucket, key, options = {})
+      bucket = Bucket.find(bucket) unless bucket.respond_to?(:name)
+      @bucket = bucket
+      @key = key
+      @options = options
+      get_metadata unless options[:lazy_load]
+    end  
+    
+    # bucket can be either a Bucket object or a string containing the bucket's name
+    def self.url(bucket, key)
+      bucket_name = bucket.respond_to?(:name) ? bucket.name : bucket
+      File.join(bucket_name, key)
+    end     
+    
+    def url
+      S3Object.url(@bucket.name, @key)
+    end 
+    
+    def metadata
+      @metadata || get_metadata
+    end
+    
+    def value(params = {})
+      refresh if params[:refresh]
+      @value || get_value
+    end
+    
+    def value=(value)
+      S3Object.create(@bucket, @key, value, @options)
+      @value = value
+      refresh_metadata
+    end
+    
+    def refresh
+      get_value
+    end
+    
+    def refresh_metadata
+      get_metadata
+    end
+    
+    def content_type
+      metadata["content-type"]
+    end
+    
+    # strip off the leading and trailing double-quotes
+    def etag
+      metadata["etag"].sub(/\A\"/,'').sub(/\"\Z/, '')
+    end
+    
+    def length
+      metadata["content-length"].to_i
+    end
+    
+    private
+    
+    def self.object_request(verb, url, options = {})
+      begin
+        options.delete(:lazy_load)
+        response = S3Lib.request(verb, url, options)
+      rescue S3Lib::S3ResponseError => error
+        case error.amazon_error_type
+        when 'NoSuchBucket': raise S3Lib::BucketNotFoundError.new("The bucket '#{bucket}' does not exist.", error.io, error.s3requester)
+        when 'NotSignedUp': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by somebody else", error.io, error.s3requester)
+        when 'AccessDenied': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by someone else.", error.io, error.s3requester)
+        when 'MissingContentLength': raise S3Lib::NoContentError.new("You must provide a value to put in the object.\nUsage: S3Lib::S3Object.create(bucket, key, value, options)", error.io, error.s3requester)          
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+      response
+    end
+        
+    def get_metadata
+      request = S3Object.object_request(:head, url, @options)
+      @metadata = request.meta
+    end
+    
+    def get_value
+      request = S3Object.object_request(:get, url, @options)
+      @metadata = request.meta      
+      @value = request.read
+    end
+    
+  end
+
+end

data/lib/put_with_curl_test.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+require 's3lib'
+
+module S3Lib
+  
+  class AuthenticatedRequest
+    
+    def public_authorization_string
+      authorization_string
+    end
+    
+  end
+  
+end
+
+value = 'testing'
+key = 'test.txt'
+auth_string = nil
+date = Time.now.httpdate
+begin
+  S3Lib.request(:put, "spatten_test_bucket/#{key}", :body => value, 'date' => date)
+rescue => e
+  puts e.response
+  puts "authorization string:"
+  puts e.s3requester.public_authorization_string
+  auth_string = e.s3requester.public_authorization_string
+end
+  
+puts "date: #{date}"
+puts "Auth String:"
+puts auth_string
+
+puts "doing curl"
+puts `curl -X PUT -d body=#{value} -d 'Authorization=#{auth_string}' -d 'date=#{date}' http://s3.amazonaws.com/spatten_test_bucket/#{key}` 
+puts "end of curl"
+
+obj = S3Lib::S3Object.find('spatten_test_bucket', key)
+puts "Content type: #{obj.content_type}"
+

data/lib/s3_authenticator.rb

@@ -0,0 +1,155 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'rest-open-uri'
+
+require 'base64'
+require 'digest/sha1'
+require 'openssl'
+require 'pp'
+
+class Hash
+  
+  def downcase_keys
+    res = {}
+    each do |key, value|
+      key = key.downcase if key.respond_to?(:downcase)
+      res[key] = value
+    end
+    res
+  end
+  
+  def join_values(separator = ',')    
+    res = {}
+    each do |key, value|
+      res[key] = value.respond_to?(:join) ? value.join(separator) : value
+    end
+    res
+  end
+  
+end
+
+module S3Lib
+  
+  def self.request(verb, request_path, headers = {})
+    begin
+      s3requester = AuthenticatedRequest.new()
+      req = s3requester.make_authenticated_request(verb, request_path, headers)
+    rescue OpenURI::HTTPError=> e
+      raise S3Lib::S3ResponseError.new(e.message, e.io, s3requester)
+    end
+  end
+
+  class AuthenticatedRequest
+  
+    POSITIONAL_HEADERS = ['content-md5', 'content-type', 'date']  
+    AMAZON_HEADER_PREFIX = 'x-amz-'
+    HOST = 's3.amazonaws.com'  
+    BUCKET_LIST_PARAMS = [:max_keys, :prefix, :marker, :delimiter] 
+    SUB_RESOURCE_TYPES = ['acl', 'torrent', 'logging']
+  
+    def make_authenticated_request(verb, request_path, headers = {})
+      @verb = verb
+      @request_path = request_path.gsub(/^\//,'') # Strip off the leading '/'
+    
+      @amazon_id = ENV['AMAZON_ACCESS_KEY_ID']
+      @amazon_secret = ENV['AMAZON_SECRET_ACCESS_KEY']    
+    
+      @headers = headers.downcase_keys.join_values    
+      get_bucket_list_params          
+      get_bucket_name
+      fix_date
+      
+      req = open(uri_with_bucket_list_params, @headers.merge(:method => @verb, 'Authorization' => authorization_string))
+    end
+    
+    def canonical_string
+      "#{@verb.to_s.upcase}\n#{canonicalized_headers}#{canonicalized_resource}"
+    end  
+  
+    def get_bucket_name
+      @bucket = ""
+      return unless @headers.has_key?('host')
+      @headers['host'] = @headers['host'].downcase
+      return if @headers['host'] == 's3.amazonaws.com'
+      if @headers['host'] =~ /^([^.]+)(:\d\d\d\d)?\.#{HOST}$/
+        @bucket = $1.gsub(/\/$/,'') + '/'
+      else
+        @bucket = @headers['host'].gsub(/(:\d\d\d\d)$/, '').gsub(/\/$/,'') + '/'
+      end    
+    end
+  
+    def fix_date
+      @headers['date'] ||= Time.now.httpdate
+      @headers.delete('date') if @headers.has_key?('x-amz-date')    
+    end
+  
+    def uri
+      host = @headers['host'] || HOST
+      "http://" + File.join(host, URI.escape(@request_path))
+    end
+        
+    def get_bucket_list_params
+      @bucket_list_params = {}
+      @headers.each do |key, value|
+        @bucket_list_params[key] = @headers.delete(key) if BUCKET_LIST_PARAMS.include?(key)
+      end
+    end
+    
+    def uri_with_bucket_list_params
+      return uri if @bucket_list_params.empty?
+      uri_with_params = uri
+      bucket_list_string = @bucket_list_params.collect {|key, value| "#{key.to_s.gsub('_', '-')}=#{value}"}.join('&')
+      uri_with_params.sub(/\/$/, '') # remove trailing slash
+      uri_with_params += '?' unless uri =~ /\?$/ # Add trailing ?
+      uri_with_params += bucket_list_string # add bucket list params
+      uri_with_params
+    end    
+  
+    def authorization_string
+      generator = OpenSSL::Digest::Digest.new('sha1')
+      encoded_canonical = Base64.encode64(OpenSSL::HMAC.digest(generator, @amazon_secret, canonical_string)).strip
+
+      "AWS #{@amazon_id}:#{encoded_canonical}"
+    end
+  
+    def canonicalized_headers
+      canonicalized_positional_headers + canonicalized_amazon_headers
+    end
+  
+    def canonicalized_positional_headers
+      POSITIONAL_HEADERS.collect do |header|
+        (@headers[header] || "") + "\n"
+      end.join
+    end
+  
+    def canonicalized_amazon_headers
+    
+      # select all headers that start with x-amz-
+      amazon_headers = @headers.select do |header, value|
+        header =~ /^x-amz-/
+      end
+    
+      # Sort them alpabetically by key
+      amazon_headers = amazon_headers.sort do |a, b|
+        a[0] <=> b[0]
+      end
+    
+      # Collect all of the amazon headers like this:
+      # {key}:{value}\n
+      # The value has to have any whitespace on the left stripped from it 
+      # and any new-lines replaced by a single space.
+      # Finally, return the headers joined together as a single string and return it.
+      amazon_headers.collect do |header, value|
+        "#{header}:#{value.lstrip.gsub("\n"," ")}\n"
+      end.join
+    end
+  
+    def canonicalized_resource
+      canonicalized_resource_string = "/"
+      canonicalized_resource_string += @bucket
+      canonicalized_resource_string += @request_path  
+      canonicalized_resource_string  
+    end
+    
+  end  
+end

data/lib/s3_authenticator_dev.rb

@@ -0,0 +1,117 @@
+class Hash
+  
+  def downcase_keys
+    res = {}
+    each do |key, value|
+      key = key.downcase if key.respond_to?(:downcase)
+      res[key] = value
+    end
+    res
+  end
+  
+  def join_values(separator = ',')    
+    res = {}
+    each do |key, value|
+      res[key] = value.respond_to?(:join) ? value.join(separator) : value
+    end
+    res
+  end
+  
+end
+
+module S3Lib
+  require 'time'
+  require 'base64'
+  require 'digest/sha1'
+  require 'openssl'  
+  
+  def self.request(verb, request_path, headers = {})
+    s3requester = AuthenticatedRequest.new()
+    s3requester.make_authenticated_request(verb, request_path, headers)
+  end  
+  
+  class AuthenticatedRequest
+    
+    attr_reader :headers
+    POSITIONAL_HEADERS = ['content-md5', 'content-type', 'date']      
+    HOST = 's3.amazonaws.com'       
+    
+    def make_authenticated_request(verb, request_path, headers = {})
+      @verb = verb
+      @request_path = request_path.gsub(/^\//,'') # Strip off the leading '/'      
+      @amazon_id = ENV['AMAZON_ACCESS_KEY_ID']
+      @amazon_secret = ENV['AMAZON_SECRET_ACCESS_KEY']          
+      @headers = headers.downcase_keys.join_values
+      fix_date
+      get_bucket_name      
+    end    
+    
+    def fix_date
+      @headers['date'] ||= Time.now.httpdate
+      @headers.delete('date') if @headers.has_key?('x-amz-date')          
+    end
+  
+    def canonical_string
+      "#{@verb.to_s.upcase}\n#{canonicalized_headers}#{canonicalized_resource}"
+    end
+    
+    def canonicalized_headers
+      "#{canonicalized_positional_headers}#{canonicalized_amazon_headers}"
+    end
+    
+    def canonicalized_positional_headers
+      POSITIONAL_HEADERS.collect do |header|
+        (@headers[header] || "") + "\n"
+      end.join
+    end 
+    
+    def canonicalized_amazon_headers
+    
+      # select all headers that start with x-amz-
+      amazon_headers = @headers.select do |header, value|
+        header =~ /^x-amz-/
+      end
+    
+      # Sort them alpabetically by key
+      amazon_headers = amazon_headers.sort do |a, b|
+        a[0] <=> b[0]
+      end
+    
+      # Collect all of the amazon headers like this:
+      # {key}:{value}\n
+      # The value has to have any whitespace on the left stripped from it 
+      # and any new-lines replaced by a single space.
+      # Finally, return the headers joined together as a single string and return it.
+      amazon_headers.collect do |header, value|
+        "#{header}:#{value.lstrip.gsub("\n"," ")}\n"
+      end.join
+    end   
+    
+    def canonicalized_resource
+      canonicalized_resource_string = "/"
+      canonicalized_resource_string += @bucket
+      canonicalized_resource_string += @request_path  
+      canonicalized_resource_string  
+    end
+        
+    def get_bucket_name
+      @bucket = ""
+      return unless @headers.has_key?('host')
+      @headers['host'] = @headers['host'].downcase
+      return if @headers['host'] == 's3.amazonaws.com'
+      if @headers['host'] =~ /^([^.]+)(:\d\d\d\d)?\.#{HOST}$/ # Virtual hosting
+        @bucket = $1.gsub(/\/$/,'') + '/'
+      else
+        @bucket = @headers['host'].gsub(/(:\d\d\d\d)$/, '').gsub(/\/$/,'') + '/' # CNAME Virtual hosting
+      end    
+    end   
+    
+    def authorization_string
+      generator = OpenSSL::Digest::Digest.new('sha1')
+      encoded_canonical = Base64.encode64(OpenSSL::HMAC.digest(generator, @amazon_secret, canonical_string)).strip
+
+      "AWS #{@amazon_id}:#{encoded_canonical}"
+    end       
+  
+  end
+end

data/lib/s3_authenticator_dev_private.rb

@@ -0,0 +1,40 @@
+module S3Lib
+  require 'time'
+  
+  def self.request(verb, request_path, headers = {})
+    s3requester = AuthenticatedRequest.new()
+    s3requester.make_authenticated_request(verb, request_path, headers)
+  end  
+  
+  class AuthenticatedRequest
+    
+    POSITIONAL_HEADERS = ['content-md5', 'content-type', 'date']      
+    
+    def make_authenticated_request(verb, request_path, headers = {})
+      @verb = verb
+      @headers = headers
+      fix_date
+    end
+    
+    private
+    
+    def fix_date
+      @headers['date'] ||= Time.now.httpdate
+    end
+  
+    def canonical_string
+      "#{@verb.to_s.upcase}\n#{canonicalized_headers}"
+    end
+    
+    def canonicalized_headers
+      "#{canonicalized_positional_headers}"
+    end
+    
+    def canonicalized_positional_headers
+      POSITIONAL_HEADERS.collect do |header|
+        (@headers[header] || "") + "\n"
+      end.join
+    end    
+  
+  end
+end

data/lib/s3_errors.rb

@@ -0,0 +1,58 @@
+module S3Lib
+  
+  
+  class S3ResponseError < StandardError
+    attr_reader :response, :amazon_error_type, :status, :s3requester, :io
+    def initialize(message, io, s3requester)
+      @io = io
+      # Get the response and status from the IO object
+      @io.rewind
+      @response = @io.read
+      @io.rewind 
+      @status = io.status
+      
+      # The Amazon Error type will always look like <Code>AmazonErrorType</Code>.  Find it with a RegExp.
+      @response =~ /<Code>(.*)<\/Code>/
+      @amazon_error_type = $1
+      
+      # Make the AuthenticatedRequest instance available as well
+      @s3requester = s3requester
+      
+      # Call the standard Error initializer
+      # if you put '%s' in the message it will be replaced by the amazon_error_type
+      message += "\namazon error type: %s" unless message =~ /\%s/
+      super(message % @amazon_error_type)
+    end
+  end  
+  
+  # Bucket errors
+  
+  class NotYourBucketError < S3Lib::S3ResponseError
+  end
+  
+  class BucketNotFoundError < S3Lib::S3ResponseError
+  end
+  
+  class BucketNotEmptyError < S3Lib::S3ResponseError
+  end
+  
+  # Object errors
+  
+  class ObjectDoesNotExist < StandardError
+  end
+  
+  class ObjectAccessForbidden < StandardError
+  end  
+  
+  class NoContentError < S3Lib::S3ResponseError
+  end 
+  
+  # ACL errors
+  class MalformedACLError < S3Lib::S3ResponseError
+  end
+  
+  # Grant errors
+  class BadGrantTypeError < StandardError
+  end
+  
+end

data/lib/s3lib.rb

@@ -0,0 +1,10 @@
+require File.join(File.dirname(__FILE__), 's3_authenticator')
+require File.join(File.dirname(__FILE__), 's3_errors')
+require File.join(File.dirname(__FILE__), 'service')
+require File.join(File.dirname(__FILE__), 'object')
+require File.join(File.dirname(__FILE__), 'bucket')
+require File.join(File.dirname(__FILE__), 'acl')
+require File.join(File.dirname(__FILE__), 'grant')
+require 'rexml/document'
+require 'rubygems'
+require 'builder'