net-amazon-s3 0.1.0

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2006 Ryan Grove <ryan@wonko.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of this project nor the names of its contributors may be
+      used to endorse or promote products derived from this software without
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/lib/net/amazon/s3.rb

@@ -0,0 +1,361 @@
+require 'base64'
+require 'digest/md5'
+require 'net/https'
+require 'net/amazon/s3/bucket'
+require 'net/amazon/s3/errors'
+require 'net/amazon/s3/object'
+require 'openssl'
+require 'rexml/document'
+
+module Net; module Amazon
+
+  # = Net::Amazon::S3
+  # 
+  # This library implements the Amazon S3 REST API (Rubyfied for your pleasure).
+  # Its usage is hopefully pretty straightforward. See below for examples.
+  # 
+  # Author::    Ryan Grove (mailto:ryan@wonko.com)
+  # Version::   0.1.0
+  # Copyright:: Copyright (c) 2006 Ryan Grove. All rights reserved.
+  # License::   New BSD License (http://opensource.org/licenses/bsd-license.php)
+  # Website::   http://wonko.com/software/net-amazon-s3
+  # 
+  # == A Brief Overview of Amazon S3
+  # 
+  # Amazon S3 stores arbitrary values (objects) identified by keys and organized
+  # into buckets. An S3 bucket is essentially a glorified Hash. Object values
+  # can be up to 5 GB in size, and objects can also have up to 2 KB of metadata
+  # associated with them.
+  # 
+  # Bucket names share a global namespace and must be unique across all of
+  # S3, but object keys only have to be unique within the bucket in which they
+  # are stored.
+  # 
+  # For more details, visit http://s3.amazonaws.com
+  # 
+  # == Installation
+  # 
+  #   gem install net-amazon-s3
+  # 
+  # == Examples
+  # 
+  # === Create an instance of the S3 client
+  # 
+  #   require 'rubygems'
+  #   require 'net/amazon/s3'
+  #   
+  #   access_key_id     = 'DXM37ARQ25519H34E6W2'
+  #   secret_access_key = '43HM88c+8kYr/UeFp+shjTnzFgisO5AZzpEO06FU'
+  #   
+  #   s3 = Net::Amazon::S3.new(access_key_id, secret_access_key)
+  #   
+  # === Create a bucket and add an object to it
+  # 
+  #   foo = s3.create_bucket('foo')
+  #   foo['bar'] = 'baz'            # create object 'bar' and assign it the
+  #                                 # value 'baz'
+  # 
+  # === Upload a large file to the bucket
+  # 
+  #   File.open('mybigmovie.avi', 'rb') do |file|
+  #     foo['mybigmovie.avi'] = file
+  #   end
+  # 
+  # === Download a large file from the bucket
+  # 
+  #   File.open('mybigmovie.avi', 'wb') do |file|
+  #     foo['mybigmovie.avi'].value {|chunk| file.write(chunk) }
+  #   end
+  # 
+  # === Get a hash containing all objects in the bucket
+  # 
+  #   objects = foo.get_objects
+  # 
+  # === Get all objects in the bucket whose keys begin with "my"
+  # 
+  #   my_objects = foo.get_objects('my')
+  # 
+  # === Delete the bucket and everything in it
+  # 
+  #   s3.delete_bucket('foo', true)
+  # 
+  # == TODO
+  # 
+  # * Owner support
+  # * Object metadata support
+  # * ACLs
+  # * Logging configuration
+  # 
+  class S3
+    include Enumerable
+    
+    REST_ENDPOINT = 's3.amazonaws.com'
+  
+    attr_accessor :access_key_id, :secret_access_key
+    attr_reader   :options
+    
+    # Creates and returns a new S3 client. The following options are available:
+    # 
+    # [<tt>:enable_cache</tt>] Set to +true+ to enable intelligent caching of
+    #                          frequently-used requests. This can improve
+    #                          performance, but may result in staleness if other
+    #                          clients are simultaneously modifying the buckets
+    #                          and objects in this S3 account. Default is
+    #                          +true+.
+    # [<tt>:ssl</tt>] Set to +true+ to use SSL for all requests. No verification
+    #                 is performed on the server's certificate when SSL is used.
+    #                 Default is +true+.
+    def initialize(access_key_id, secret_access_key, options = {})
+      @access_key_id     = access_key_id
+      @secret_access_key = secret_access_key
+
+      @options = {
+        :enable_cache => true,
+        :ssl          => true
+      }
+
+      @options.merge!(options)
+      
+      @cache = {}
+    end
+    
+    # Returns +true+ if a bucket with the specified +bucket_name+ exists in this
+    # S3 account, +false+ otherwise.
+    def bucket_exist?(bucket_name)
+      return get_buckets.has_key?(bucket_name)
+    end
+    
+    alias has_bucket? bucket_exist?
+
+    # Creates a new bucket with the specified +bucket_name+ and returns a
+    # Bucket object representing it.
+    def create_bucket(bucket_name)
+      error?(request_put("/#{bucket_name}"))
+      @cache.delete(:buckets)
+      return get_bucket(bucket_name)
+    end
+
+    # Deletes the bucket with the specified +bucket_name+. If +recursive+ is
+    # +true+, all objects contained in the bucket will also be deleted. If
+    # +recursive+ is +false+ and the bucket is not empty, a
+    # S3Error::BucketNotEmpty error will be raised.
+    def delete_bucket(bucket_name, recursive = false)
+      unless bucket = get_bucket(bucket_name)
+        raise S3Error::NoSuchBucket, 'The specified bucket does not exist'
+      end
+    
+      if recursive
+        bucket.each {|object| bucket.delete_object(object.name) }
+      end
+      
+      @cache.delete(:buckets)
+      
+      return true unless error?(request_delete("/#{bucket_name}"))
+    end
+
+    # Iterates through the list of buckets.
+    def each
+      get_buckets.each {|key, value| yield key, value }
+    end
+
+    # Raises the appropriate error if the specified Net::HTTPResponse object
+    # contains an Amazon S3 error; returns +false+ otherwise.
+    def error?(response)
+      return false if response.is_a?(Net::HTTPSuccess)
+
+      xml = REXML::Document.new(response.body)
+      
+      unless xml.root.name == 'Error'
+        raise S3Error, "Unknown error: #{response.body}"
+      end
+      
+      error_code    = xml.root.elements['Code'].text
+      error_message = xml.root.elements['Message'].text
+
+      if S3Error.const_defined?(error_code)
+        raise S3Error.const_get(error_code), error_message
+      else
+        raise S3Error, "#{error_code}: #{error_message}"
+      end
+    end
+
+    # Returns a Bucket object representing the specified bucket, or +nil+ if the
+    # bucket doesn't exist.
+    def get_bucket(bucket_name)
+      return nil unless bucket_exist?(bucket_name)
+      return @cache[:buckets][bucket_name]
+    end
+    
+    alias [] get_bucket
+
+    # Gets a list of all buckets owned by this account. Returns a Hash of
+    # Bucket objects indexed by bucket name.
+    def get_buckets
+      if @options[:enable_cache] && !@cache[:buckets].nil?
+        return @cache[:buckets]
+      end
+      
+      response = request_get('/')
+      error?(response)
+      
+      xml = REXML::Document.new(response.body)
+      
+      buckets = {}
+
+      xml.root.elements.each('Buckets/Bucket') do |element|
+        bucket_name   = element.elements['Name'].text
+        creation_date = Time.parse(element.elements['CreationDate'].text)
+        
+        buckets[bucket_name] = Bucket.new(self, bucket_name, creation_date)
+      end
+      
+      return @cache[:buckets] = buckets
+    end
+    
+    # Sends a properly-signed DELETE request to the specified S3 path and
+    # returns a Net::HTTPResponse object.
+    def request_delete(path, headers = {})
+      http = Net::HTTP.new(REST_ENDPOINT, @options[:ssl] ? 443 : 80)
+
+      http.use_ssl     = @options[:ssl]
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      
+      http.start do |http|
+        req = sign_request(Net::HTTP::Delete.new(path), nil, headers)
+        return http.request(req)
+      end
+    end
+    
+    # Sends a properly-signed GET request to the specified S3 path and returns
+    # a Net::HTTPResponse object.
+    # 
+    # When called with a block, yields a Net::HTTPResponse object whose body has
+    # not been read; the caller can process it using
+    # Net::HTTPResponse.read_body.
+    def request_get(path, headers = {})
+      http = Net::HTTP.new(REST_ENDPOINT, @options[:ssl] ? 443 : 80)
+
+      http.use_ssl     = @options[:ssl]
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      
+      http.start do |http|
+        req = sign_request(Net::HTTP::Get.new(path), nil, headers)
+        
+        if block_given?
+          http.request(req) {|response| yield response }
+        else
+          return http.request(req)
+        end
+      end
+    end
+    
+    # Sends a properly-signed HEAD request to the specified S3 path and returns
+    # a Net::HTTPResponse object.
+    def request_head(path, headers = {})
+      http = Net::HTTP.new(REST_ENDPOINT, @options[:ssl] ? 443 : 80)
+
+      http.use_ssl     = @options[:ssl]
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      
+      http.start do |http|
+        req = sign_request(Net::HTTP::Head.new(path), nil, headers)
+        return http.request(req)
+      end
+    end
+    
+    # Sends a properly-signed PUT request to the specified S3 path and returns a
+    # Net::HTTPResponse object.
+    # 
+    # If +content+ is an open IO stream, the body of the request will be read
+    # from the stream.
+    def request_put(path, content = nil, headers = {})
+      http = Net::HTTP.new(REST_ENDPOINT, @options[:ssl] ? 443 : 80)
+      
+      http.use_ssl     = @options[:ssl]
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      
+      http.start do |http|
+        req = sign_request(Net::HTTP::Put.new(path), content, headers)
+        
+        if content.is_a?(IO)
+          req.body_stream = content
+        else
+          req.body = content
+        end
+        
+        response = http.request(req)
+        
+        return response
+      end
+    end
+    
+    private
+    
+    # Adds an appropriately-signed +Authorization+ header to the
+    # Net::HTTPRequest +request+.
+    # 
+    # If +content+ is an open IO stream, the body of the request will be read
+    # from the stream.
+    def sign_request(request, content = nil, headers = {})
+      unless request.is_a?(Net::HTTPRequest)
+        raise ArgumentError,
+            "Expected Net::HTTPRequest, not #{request.class}"
+      end
+
+      unless request.path =~ /^(\/.*?)(?:\?.*)?$/i
+        raise S3Error, "Invalid request path: #{request.path}"
+      end
+      
+      path = $1
+      
+      request['Host'] = REST_ENDPOINT
+      request['Date'] = Time.new.httpdate()
+      
+      if content.nil?
+        request['Content-Length'] = 0
+      elsif content.is_a?(IO)
+        # Generate an MD5 hash of the stream's contents.
+        md5 = Digest::MD5.new
+        content.rewind
+        
+        while buffer = content.read(65536) do
+          md5 << buffer
+        end
+        
+        content.rewind
+        
+        # Set headers.
+        request['Content-Type']   = 'binary/octet-stream'
+        request['Content-Length'] = content.stat.size
+        request['Content-MD5']    = Base64.encode64(md5.digest).strip
+      else
+        request['Content-Type']   = 'binary/octet-stream'
+        request['Content-Length'] = content.length
+        request['Content-MD5']    = Base64.encode64(Digest::MD5.digest(
+            content)).strip
+      end
+
+      headers.each {|key, value| request[key] = value }
+      
+      hmac = OpenSSL::HMAC.new(@secret_access_key, OpenSSL::Digest::SHA1.new)
+      hmac << "#{request.method}\n#{request['Content-MD5']}\n".toutf8 +
+          "#{request['Content-Type']}\n#{request['Date']}\n".toutf8
+      
+      request.to_hash.keys.sort.each do |key|
+        if key =~ /^x-amz-/i
+          hmac << "#{key.downcase.strip}:#{request[key].strip}\n".toutf8
+        end
+      end
+      
+      hmac << path.toutf8
+      
+      signature = Base64.encode64(hmac.digest).strip
+      
+      request['Authorization'] = "AWS #{@access_key_id}:#{signature}"
+      
+      return request
+    end
+
+  end
+
+end; end

data/lib/net/amazon/s3/bucket.rb

@@ -0,0 +1,141 @@
+require 'rexml/document'
+require 'time'
+require 'uri'
+
+module Net; module Amazon; class S3
+
+  # Represents an Amazon S3 bucket. This class should only be instantiated
+  # through one of the methods in the S3 class.
+  class Bucket
+    include Comparable
+    include Enumerable
+  
+    attr_reader :name, :creation_date
+  
+    # Creates and returns a new Bucket object. You should never create new
+    # Bucket objects directly. Instead, use one of the methods in the S3 class.
+    def initialize(s3, bucket_name, creation_date)
+      @s3            = s3
+      @name          = bucket_name
+      @creation_date = creation_date
+
+      @cache = {}
+    end
+    
+    # Compares two buckets by name.
+    def <=>(bucket)
+      return @name <=> bucket.name
+    end
+    
+    # Creates and returns a new S3::Object with the specified +object_key+ and
+    # +value+. If this bucket already contains an object with the specified key,
+    # that object will be overwritten.
+    # 
+    # If +value+ is an open IO stream, the value of the object will be read from
+    # the stream.
+    def create_object(object_key, value, metadata = {})
+      object_key_escaped = S3::Object.escape_key(object_key)
+      
+      headers = {}
+      metadata.each {|key, value| headers["x-amz-meta-#{key}"] = value }
+      
+      response = @s3.request_put("/#{@name}/#{object_key_escaped}", value,
+          headers)
+      @s3.error?(response)
+
+      @cache.delete(:objects)
+
+      return get_object(object_key)
+    end
+    
+    alias []= create_object
+
+    # Deletes the specified object from this bucket.
+    def delete_object(object_key)
+      object_key_escaped = S3::Object.escape_key(object_key)
+    
+      unless object = get_object(object_key)
+        raise S3Error::NoSuchKey, 'The specified key does not exist'
+      end
+      
+      @cache.delete(:objects)
+      
+      return true unless @s3.error?(@s3.request_delete(
+          "/#{@name}/#{object_key_escaped}"))
+    end
+    
+    # Iterates through the list of objects.
+    def each
+      get_objects.each {|key, value| yield key, value }
+    end
+  
+    # Returns a S3::Object representing the specified +object_key+, or +nil+ if
+    # the object doesn't exist in this bucket.
+    def get_object(object_key)
+      return get_objects(object_key)[object_key]
+    end
+    
+    alias [] get_object
+
+    # Gets a list of all objects in this bucket whose keys begin with +prefix+.
+    # Returns a Hash of S3::Object objects indexed by object key.
+    def get_objects(prefix = '')
+      prefix = prefix.toutf8
+    
+      if @s3.options[:enable_cache] && !@cache[:objects].nil? &&
+          !@cache[:objects][prefix].nil?
+        return @cache[:objects][prefix]
+      end
+      
+      if @cache[:objects].nil?
+        @cache[:objects] = {}
+      end
+      
+      objects      = {}
+      request_uri  = "/#{@name}?prefix=#{URI.escape(prefix)}"
+      is_truncated = true
+      
+      # The request is made in a loop because the S3 API limits results to pages
+      # of 1,000 objects by default, so if there are more than 1,000 objects,
+      # we'll have to send more requests to get them all.
+      while is_truncated do
+        response = @s3.request_get(request_uri)
+        @s3.error?(response)
+        
+        xml = REXML::Document.new(response.body)
+        
+        if xml.root.elements['IsTruncated'].text == 'false'
+          is_truncated = false
+        else
+          request_uri = "/#{@name}?prefix=#{URI.escape(prefix)}&marker=" +
+              xml.root.elements.to_a('Contents').last.elements['Key'].text
+        end
+        
+        next if xml.root.elements['Contents'].nil?
+  
+        xml.root.elements.each('Contents') do |element|
+          object_key           = element.elements['Key'].text
+          object_size          = element.elements['Size'].text
+          object_etag          = element.elements['ETag'].text
+          object_last_modified = Time.parse(
+              element.elements['LastModified'].text)
+          
+          objects[object_key] = S3::Object.new(@s3, self, object_key,
+              object_size, object_etag, object_last_modified)
+        end
+      end
+
+      return @cache[:objects][prefix] = objects
+    end
+    
+    # Returns +true+ if an object with the specified +object_key+ exists in
+    # this bucket, +false+ otherwise.
+    def object_exist?(object_key)
+      return get_objects.has_key?(object_key)
+    end
+    
+    alias has_object? object_exist?
+
+  end
+  
+end; end; end

data/lib/net/amazon/s3/errors.rb

@@ -0,0 +1,47 @@
+module Net; module Amazon; class S3; class S3Error < StandardError
+
+  class AccessDenied < S3Error; end
+  class AccountProblem < S3Error; end
+  class AllAccessDisabled < S3Error; end
+  class AmbiguousGrantByEmailAddress < S3Error; end
+  class OperationAborted < S3Error; end
+  class BadDigest < S3Error; end
+  class BucketAlreadyExists < S3Error; end
+  class BucketNotEmpty < S3Error; end
+  class CredentialsNotSupported < S3Error; end
+  class EntityTooLarge < S3Error; end
+  class IncompleteBody < S3Error; end
+  class InternalError < S3Error; end
+  class InvalidAccessKeyId < S3Error; end
+  class InvalidAddressingHeader < S3Error; end
+  class InvalidArgument < S3Error; end
+  class InvalidBucketName < S3Error; end
+  class InvalidDigest < S3Error; end
+  class InvalidRange < S3Error; end
+  class InvalidSecurity < S3Error; end
+  class InvalidStorageClass < S3Error; end
+  class InvalidTargetBucketForLogging < S3Error; end
+  class KeyTooLong < S3Error; end
+  class InvalidURI < S3Error; end
+  class MalformedACLError < S3Error; end
+  class MalformedXMLError < S3Error; end
+  class MaxMessageLengthExceeded < S3Error; end
+  class MetadataTooLarge < S3Error; end
+  class MethodNotAllowed < S3Error; end
+  class MissingContentLength < S3Error; end
+  class MissingSecurityHeader < S3Error; end
+  class NoLoggingStatusForKey < S3Error; end
+  class NoSuchBucket < S3Error; end
+  class NoSuchKey < S3Error; end
+  class NotImplemented < S3Error; end
+  class NotSignedUp < S3Error; end
+  class PreconditionFailed < S3Error; end
+  class RequestTimeout < S3Error; end
+  class RequestTimeTooSkewed < S3Error; end
+  class RequestTorrentOfBucketError < S3Error; end
+  class SignatureDoesNotMatch < S3Error; end
+  class TooManyBuckets < S3Error; end
+  class UnexpectedContent < S3Error; end
+  class UnresolvableGrantByEmailAddress < S3Error; end
+
+end; end; end; end

data/lib/net/amazon/s3/object.rb

@@ -0,0 +1,77 @@
+module Net; module Amazon; class S3
+
+  # Represents an Amazon S3 object. This class should only be instantiated
+  # through one of the methods in Bucket.
+  class Object
+    include Comparable
+  
+    #--
+    # Public Class Methods
+    #++
+    
+    # Escapes an object key for use in an S3 request path. This method should
+    # not be used to escape object keys for use in URL query parameters. Use
+    # URI.escape for that.
+    def self.escape_key(object_key)
+      return object_key.gsub(' ', '+').toutf8
+    end
+    
+    #--
+    # Public Instance Methods
+    #++
+
+    attr_reader :name, :size, :etag, :last_modified
+    
+    def initialize(s3, bucket, object_key, size, etag, last_modified)
+      @s3            = s3
+      @bucket        = bucket
+      @key           = object_key.toutf8
+      @size          = size
+      @etag          = etag
+      @last_modified = last_modified
+      
+      @cache = {}
+    end
+    
+    # Compares two objects by key.
+    def <=>(object)
+      return @key <=> object.key
+    end
+    
+    # Gets this object's value.
+    # 
+    # When called with a block, yields the value in chunks as it is read in from
+    # the socket.
+    def value
+      key_escaped = Object.escape_key(@key)
+      
+      if block_given?
+        @s3.request_get("/#{@bucket.name}/#{key_escaped}") do |response|
+          @s3.error?(response)
+          response.read_body {|chunk| yield chunk }
+        end
+      else
+        response = @s3.request_get("/#{@bucket.name}/#{key_escaped}")
+        @s3.error?(response)
+        
+        return response.body
+      end
+    end
+    
+    # Sets this object's value.
+    # 
+    # If +new_value+ is an open IO stream, the value will be read from the
+    # stream.
+    def value=(new_value)
+      key_escaped = Object.escape_key(@key)
+
+      response = @s3.request_put("/#{@bucket.name}/" +
+          "#{key_escaped}", new_value)
+      @s3.error?(response)
+      
+      return new_value
+    end
+    
+  end
+  
+end; end; end

metadata

@@ -0,0 +1,57 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: net-amazon-s3
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2006-11-25 00:00:00 -08:00
+summary: Amazon S3 library.
+require_paths: 
+- lib
+email: ryan@wonko.com
+homepage: http://wonko.com/software/net-amazon-s3
+rubyforge_project: net-amazon-s3
+description: 
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.8.4
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Ryan Grove
+files: 
+- lib/net
+- lib/net/amazon
+- lib/net/amazon/s3
+- lib/net/amazon/s3.rb
+- lib/net/amazon/s3/bucket.rb
+- lib/net/amazon/s3/errors.rb
+- lib/net/amazon/s3/object.rb
+- LICENSE
+test_files: []
+
+rdoc_options: 
+- --title
+- Net::Amazon::S3 Documentation
+- --main
+- Net::Amazon::S3
+- --line-numbers
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+