internuity-awsum 0.2 → 0.3

data/lib/error.rb

@@ -0,0 +1,55 @@
+require 'parser'
+
+module Awsum
+  class Error < StandardError
+    attr_reader :response_code, :code, :message, :request_id, :additional
+
+    def initialize(response)
+      @response_code = response.code
+      parser = ErrorParser.new
+      parser.parse(response.body)
+      @code = parser.code
+      @message = parser.message
+      @request_id = parser.request_id
+      @additional = parser.additional
+    end
+
+    def inspect
+      "#<Awsum::Error response_code=#{@response_code} code=#{@code} request_id=#{@request_id} message=#{@message}>"
+    end
+
+  private
+    class ErrorParser < Awsum::Parser #:nodoc:
+      attr_reader :code, :message, :request_id, :additional
+
+      def initialize
+        @additional = {}
+        @text = ""
+      end
+
+      def tag_start(tag, attributes)
+      end
+
+      def text(text)
+        @text << text unless @text.nil?
+      end
+
+      def tag_end(tag)
+        text = @text.strip
+        return if text.blank?
+
+        case tag
+          when 'Code'
+            @code = text
+          when 'Message'
+            @message = text
+          when 'RequestID', 'RequestId'
+            @request_id = text
+          else
+            @additional[tag] = text
+        end
+        @text = ''
+      end
+    end
+  end
+end

data/lib/net_fix.rb

@@ -0,0 +1,100 @@
+# Some fixes for the net/http libraries to better suppport S3
+module Net
+  class BufferedIO
+    #Increase the default read size for streaming from the socket
+    def rbuf_fill
+      timeout(@read_timeout) {
+        @rbuf << @io.sysread(1024 * 16)
+      }
+    end
+  end
+
+  class HTTPGenericRequest
+    @@local_read_size = 1024 * 16
+
+    # Added limit which can be one of :headers or :body to limit the sending of
+    # a request to either the headers or the body in order to make use of 
+    # 100-continue processing for S3
+    def exec(sock, ver, path, limit = nil)   #:nodoc: internal use only
+      if @body
+        send_request_with_body sock, ver, path, @body, limit
+      elsif @body_stream
+        send_request_with_body_stream sock, ver, path, @body_stream, limit
+      else
+        write_header sock, ver, path
+      end
+    end
+
+    private
+    
+    # Will send both headers and body unless limit is set to either 
+    # :headers or :body to restrict to one
+    def send_request_with_body(sock, ver, path, body, limit = nil)
+      self.content_length = body.length
+      delete 'Transfer-Encoding'
+      supply_default_content_type
+      write_header sock, ver, path unless limit == :body
+      sock.write body unless limit == :headers
+    end
+    
+    # Will send both headers and body unless limit is set to either 
+    # :headers or :body to restrict to one
+    #
+    # Increased the default read size for streaming from local streams to 1MB
+    def send_request_with_body_stream(sock, ver, path, f, limit = nil)
+      unless content_length() or chunked?
+        raise ArgumentError,
+            "Content-Length not given and Transfer-Encoding is not `chunked'"
+      end
+      supply_default_content_type
+      write_header sock, ver, path unless limit == :body
+      if limit != :headers
+        if chunked?
+          while s = f.read(1024 * 1024)
+            sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
+          end
+          sock.write "0\r\n\r\n"
+        else
+          while s = f.read(1024 * 1024)
+            sock.write s
+          end
+        end
+      end
+    end
+  end
+
+  class HTTP < Protocol
+    # Patched to handle 100-continue processing for S3
+    def request(req, body = nil, &block)  # :yield: +response+
+      unless started?
+        start {
+          req['connection'] ||= 'close'
+          return request(req, body, &block)
+        }
+      end
+      if proxy_user()
+        unless use_ssl?
+          req.proxy_basic_auth proxy_user(), proxy_pass()
+        end
+      end
+
+      req.set_body_internal body
+      begin_transport req
+        # Send only the headers if a 100-continue request
+        limit = ((req.is_a?(Post) || req.is_a?(Put)) && req['expect'] == '100-continue') ? :headers : nil
+        req.exec @socket, @curr_http_version, edit_path(req.path), limit
+        begin
+          res = HTTPResponse.read_new(@socket)
+          if res.is_a?(HTTPContinue) && limit && req['content-length'].to_i > 0
+            req.exec @socket, @curr_http_version, edit_path(req.path), :body
+          end
+        end while res.kind_of?(HTTPContinue)
+        res.reading_body(@socket, req.response_body_permitted?) {
+          yield res if block_given?
+        }
+      end_transport req, res
+
+      res
+    end
+  end
+end

data/lib/requestable.rb

@@ -190,6 +190,8 @@ module Awsum
           uri.host = new_uri.host
           uri.path = "#{new_uri.path}#{uri.path unless uri.path = '/'}"
           process_request(method, uri.to_s, headers, data, &block)
+        else
+          response
       end
     end
 

data/lib/s3/bucket.rb

@@ -0,0 +1,66 @@
+module Awsum
+  class S3
+    class Bucket
+      attr_reader :name, :creation_date
+
+      def initialize(s3, name, creation_date = nil)
+        @s3 = s3
+        @name = name
+        @creation_date = creation_date
+      end
+
+      # Delete this Bucket
+      def delete
+        @s3.delete_bucket(@name)
+      end
+
+      # Delete this Bucket, recursively deleting all keys first
+      def delete!
+        @s3.keys(@name).each do |key|
+          key.delete
+        end
+        delete
+      end
+    end
+
+    class BucketParser < Awsum::Parser #:nodoc:
+      def initialize(s3)
+        @s3 = s3
+        @buckets = []
+        @text = nil
+      end
+
+      def tag_start(tag, attributes)
+        case tag
+          when 'Bucket'
+            @current = {}
+            @text = ''
+        end
+      end
+
+      def text(text)
+        @text << text unless @text.nil?
+      end
+
+      def tag_end(tag)
+        case tag
+          when 'Bucket'
+            @buckets << Bucket.new(
+                          @s3,
+                          @current['Name'],
+                          Time.parse(@current['CreationDate'])
+                        )
+            @text = nil
+            @current = nil
+          else
+            text = @text.strip unless @text.nil?
+            @current[tag] = (text == '' ? nil : text) unless @current.nil?
+        end
+      end
+
+      def result
+        @buckets
+      end
+    end
+  end
+end

data/lib/s3/headers.rb

@@ -0,0 +1,24 @@
+module Awsum
+  class S3
+    class Headers #:nodoc:
+      def initialize(response)
+        @response = response
+      end
+
+      # Locking down to HTTPHeader methods only
+      def method_missing(method, *args, &block)
+        if !%w(body body_permitted? entity inspect read_body to_ary value).include?(method.to_s) && @response.respond_to?(method)
+          @response.send(method, *args, &block)
+        else
+          raise NoMethodError.new("undefined method `#{method}' for #{inspect}")
+        end
+      end
+
+      def inspect
+        headers = []
+        @response.canonical_each do |h,v| headers << h end
+        "#<Awsum::S3::Headers \"#{headers.join('", "')}\">"
+      end
+    end
+  end
+end

data/lib/s3/object.rb

@@ -0,0 +1,138 @@
+module Awsum
+  class S3
+    class Object
+      attr_reader :key, :bucket, :last_modified, :etag, :size, :owner, :storage_class
+
+      def initialize(s3, bucket, key, last_modified, etag, size, owner, storage_class)
+        @s3 = s3
+        @bucket = bucket
+        @key = key
+        @last_modified = last_modified
+        @etag = etag
+        @size = size
+        @owner = owner
+        @storage_class = storage_class
+      end
+
+      # Get the headers for this Object
+      #
+      # All header methods map directly to the Net::HTTPHeader module
+      def headers
+        @headers ||= @s3.object_headers(@bucket, @key)
+      end
+
+      # Retrieve the data stored for this Object
+      #
+      # You can get the data as a single call or add a block to retrieve the data in chunks
+      # ==Examples
+      #   content = object.data
+      #
+      # or
+      #
+      #   object.data do |chunk|
+      #     # handle chunk
+      #     puts chunk
+      #   end
+      def data(&block)
+        @s3.object_data @bucket, @key, &block
+      end
+
+      # Delete this Key
+      def delete
+        @s3.delete_object(@bucket, @key)
+      end
+
+      # Make a copy of this Object with a new key
+      def copy(new_key, headers = nil, meta_headers = nil)
+        @s3.copy_object(@bucket, @key, nil, new_key, headers, meta_headers)
+      end
+
+      # Rename or move this Object to a new key
+      def rename(new_key, headers = nil, meta_headers = nil)
+        copied = @s3.copy_object(@bucket, @key, nil, new_key, headers, meta_headers)
+        @s3.delete_object(@bucket, @key) if copied
+      end
+      alias_method :move, :rename
+
+      # Copy this Object to another Bucket
+      #
+      def copy_to(new_bucket, new_key = nil, headers = nil, meta_headers = nil)
+        @s3.copy_object(@bucket, @key, new_bucket, new_key, headers, meta_headers)
+      end
+
+      # Move this Object to another Bucket
+      def move_to(new_bucket, new_key = nil, headers = nil, meta_headers = nil)
+        copied = @s3.copy_object(@bucket, @key, new_bucket, new_key, headers, meta_headers)
+        @s3.delete_object(@bucket, @key) if copied
+      end
+    end
+
+#TODO: Create a more advanced array which can deal with pagination
+    class ObjectParser < Awsum::Parser #:nodoc:
+      def initialize(s3)
+        @s3 = s3
+        @bucket = ''
+        @objects = []
+        @text = nil
+        @stack = []
+      end
+
+      def tag_start(tag, attributes)
+        case tag
+          when 'ListBucketResult'
+            @stack << tag
+            @text = ''
+          when 'Contents'
+            @stack << tag
+            @current = {}
+            @text = ''
+          when 'Owner'
+            @owner = {}
+            @stack << tag
+        end
+      end
+
+      def text(text)
+        @text << text unless @text.nil?
+      end
+
+      def tag_end(tag)
+        case tag
+          when 'Name'
+            if @stack[-1] == 'ListBucketResult'
+              @bucket = @text.strip
+            end
+          when 'Contents'
+            @objects << Object.new(
+                          @s3,
+                          @bucket,
+                          @current['Key'],
+                          Time.parse(@current['LastModified']),
+                          @current['ETag'],
+                          @current['Size'].to_i,
+                          {'id' => @owner['ID'], 'name' => @owner['DisplayName']},
+                          @current['StorageClass']
+                        )
+            @current = nil
+            @text = nil
+            @stack.pop
+          when 'Owner'
+            @stack.pop
+          else
+            text = @text.strip unless @text.nil?
+            case @stack[-1]
+              when 'Owner'
+                @owner[tag] = (text == '' ? nil : text) unless @owner.nil?
+              when 'Contents'
+                @current[tag] = (text == '' ? nil : text) unless @current.nil?
+            end
+            @text = ''
+        end
+      end
+
+      def result
+        @objects
+      end
+    end
+  end
+end

data/lib/s3/s3.rb

@@ -0,0 +1,219 @@
+require 's3/bucket'
+require 's3/object'
+require 's3/headers'
+
+module Awsum
+  # Handles all interaction with Amazon S3
+  #
+  #--
+  # TODO: Change this to S3
+  # ==Getting Started
+  # Create an Awsum::Ec2 object and begin calling methods on it.
+  #   require 'rubygems'
+  #   require 'awsum'
+  #   ec2 = Awsum::Ec2.new('your access id', 'your secret key')
+  #   images = ec2.my_images
+  #   ...
+  #
+  # All calls to EC2 can be done directly in this class, or through a more 
+  # object oriented way through the various returned classes
+  #
+  # ==Examples
+  #   ec2.image('ami-ABCDEF').run
+  #
+  #   ec2.instance('i-123456789').volumes.each do |vol|
+  #     vol.create_snapsot
+  #   end
+  #
+  #   ec2.regions.each do |region|
+  #     region.use
+  #       images.each do |img|
+  #         puts "#{img.id} - #{region.name}"
+  #       end
+  #     end
+  #   end
+  #
+  # ==Errors
+  # All methods will raise an Awsum::Error if an error is returned from Amazon
+  #
+  # ==Missing Methods
+  # If you need any of this functionality, please consider getting involved 
+  # and help complete this library.
+  class S3
+    include Awsum::Requestable
+
+    # Create an new S3 instance
+    #
+    # The access_key and secret_key are both required to do any meaningful work.
+    #
+    # If you want to get these keys from environment variables, you can do that 
+    # in your code as follows:
+    #   s3 = Awsum::S3.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'])
+    def initialize(access_key = nil, secret_key = nil)
+      @access_key = access_key
+      @secret_key = secret_key
+    end
+
+    # List all the Bucket(s)
+    def buckets
+      response = send_s3_request
+      parser = Awsum::S3::BucketParser.new(self)
+      parser.parse(response.body)
+    end
+
+    def bucket(name)
+      Bucket.new(self, name)
+    end
+
+    # Create a new Bucket
+    #
+    # ===Parameters
+    # * <tt>bucket_name</tt> - The name of the new bucket
+    # * <tt>location</tt> <i>(optional)</i> - Can be <tt>:default</tt>, <tt>:us</tt> or <tt>:eu</tt>
+    def create_bucket(bucket_name, location = :default)
+      raise ArgumentError.new('Bucket name cannot be in an ip address style') if bucket_name =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
+      raise ArgumentError.new('Bucket name can only have lowercase letters, numbers, periods (.), underscores (_) and dashes (-)') unless bucket_name =~ /^[\w\d][-a-z\d._]+[a-z\d._]$/
+      raise ArgumentError.new('Bucket name cannot contain a dash (-) next to a period (.)') if bucket_name =~ /\.-|-\./
+      raise ArgumentError.new('Bucket name must be between 3 and 63 characters') if bucket_name.size < 3 || bucket_name.size > 63
+
+      data = nil
+      if location == :eu
+        data = '<CreateBucketConfiguration><LocationConstraint>EU</LocationConstraint></CreateBucketConfiguration>'
+      end
+
+      response = send_s3_request('PUT', :bucket => bucket_name, :data => data)
+      response.is_a?(Net::HTTPSuccess)
+    end
+
+    def delete_bucket(bucket_name)
+      response = send_s3_request('DELETE', :bucket => bucket_name)
+      response.is_a?(Net::HTTPSuccess)
+    end
+
+    # List the Key(s) of a Bucket
+    #
+    # ===Parameters
+    # * <tt>bucket_name</tt> - The name of the bucket to search for keys
+    # ====Options
+    # * <tt>:prefix</tt> - Limits the response to keys which begin with the indicated prefix. You can use prefixes to separate a bucket into different sets of keys in a way similar to how a file system uses folders.
+    # * <tt>:marker</tt> - Indicates where in the bucket to begin listing. The list will only include keys that occur lexicographically after marker. This is convenient for pagination: To get the next page of results use the last key of the current page as the marker.
+    # * <tt>:max_keys</tt> - The maximum number of keys you'd like to see in the response body. The server might return fewer than this many keys, but will not return more.
+    # * <tt>:delimeter</tt> - Causes keys that contain the same string between the prefix and the first occurrence of the delimiter to be rolled up into a single result element in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere in the response.
+    def keys(bucket_name, options = {})
+      paramters = {}
+      paramters['prefix'] = options[:prefix] if options[:prefix]
+      paramters['marker'] = options[:marker] if options[:marker]
+      paramters['max_keys'] = options[:max_keys] if options[:max_keys]
+      paramters['prefix'] = options[:prefix] if options[:prefix]
+
+      response = send_s3_request('GET', :bucket => bucket_name, :paramters => paramters)
+      parser = Awsum::S3::ObjectParser.new(self)
+      parser.parse(response.body)
+    end
+
+    # Create a new Object in the specified Bucket
+    #
+    # ===Parameters
+    # * <tt>bucket_name</tt> - The name of the Bucket in which to store the Key
+    # * <tt>key</tt> - The name/path of the Key to store
+    # * <tt>data</tt> - The data to be stored in this Object
+    # * <tt>headers</tt> - Standard HTTP headers to be sent along
+    # * <tt>meta_headers</tt> - Meta headers to be stored along with the key
+    # * <tt>acl</tt> - A canned access policy, can be one of <tt>:private</tt>, <tt>:public_read</tt>, <tt>:public_read_write</tt> or <tt>:authenticated_read</tt>
+    def create_object(bucket_name, key, data, headers = {}, meta_headers = {}, acl = :private)
+      headers = headers.dup
+      meta_headers.each do |k,v|
+        headers[k =~ /^x-amz-meta-/i ? k : "x-amz-meta-#{k}"] = v
+      end
+      headers['x-amz-acl'] = acl.to_s.gsub(/_/, '-')
+
+      response = send_s3_request('PUT', :bucket => bucket_name, :key => key, :headers => headers, :data => data)
+      response.is_a?(Net::HTTPSuccess)
+    end
+
+    # Retrieve the headers for this Object
+    #
+    # All header methods map directly to the Net::HTTPHeader module
+    def object_headers(bucket_name, key)
+      response = send_s3_request('HEAD', :bucket => bucket_name, :key => key)
+      Headers.new(response)
+    end
+
+    # Retrieve the data stored for the specified Object
+    #
+    # You can get the data as a single call or add a block to retrieve the data in chunks
+    # ==Examples
+    #   data = s3.object_data('test-bucket', 'key')
+    #
+    # or
+    #
+    #   s3.object_data('test-bucket', 'key') do |chunk|
+    #     # handle chunk
+    #     puts chunk
+    #   end
+    def object_data(bucket_name, key, &block)
+      send_s3_request('GET', :bucket => bucket_name, :key => key) do |response|
+        if block_given?
+          response.read_body &block
+          return true
+        else
+          return response.body
+        end
+      end
+    end
+
+    # Deletes an Object from a Bucket
+    def delete_object(bucket_name, key)
+      response = send_s3_request('DELETE', :bucket => bucket_name, :key => key)
+      response.is_a?(Net::HTTPSuccess)
+    end
+
+    # Copy the contents of an Object to another key and/or bucket
+    # 
+    # ===Parameters
+    # * <tt>source_bucket_name</tt> - The name of the Bucket from which to copy
+    # * <tt>source_key</tt> - The name of the Key from which to copy
+    # * <tt>destination_bucket_name</tt> - The name of the Bucket to which to copy (Can be nil if copying within the same bucket, or updating header data of existing Key)
+    # * <tt>destination_key</tt> - The name of the Key to which to copy (Can be nil if copying to a new bucket with same key, or updating header data of existing Key)
+    # * <tt>headers</tt> - If not nil, the headers are replaced with this information
+    # * <tt>meta_headers</tt> - If not nil, the meta headers are replaced with this information
+    #
+    #--
+    # TODO: Need to handle copy-if-... headers
+    def copy_object(source_bucket_name, source_key, destination_bucket_name = nil, destination_key= nil, headers = nil, meta_headers = nil)
+      raise ArgumentError.new('You must include one of destination_bucket_name, destination_key or headers to be replaced') if destination_bucket_name.nil? && destination_key.nil? && headers.nil? && meta_headers.nil?
+
+      headers = {
+          'x-amz-copy-source' => "/#{source_bucket_name}/#{source_key}",
+          'x-amz-metadata-directive' => (((destination_bucket_name.nil? && destination_key.nil?) || !(headers.nil? || meta_headers.nil?)) ? 'REPLACE' : 'COPY')
+        }.merge(headers||{})
+      meta_headers.each do |k,v|
+        headers[k =~ /^x-amz-meta-/i ? k : "x-amz-meta-#{k}"] = v
+      end unless meta_headers.nil?
+
+      destination_bucket_name ||= source_bucket_name
+      destination_key ||= source_key
+
+      response = send_s3_request('PUT', :bucket => destination_bucket_name, :key => destination_key, :headers => headers, :data => nil)
+      if response.is_a?(Net::HTTPSuccess)
+        #Check for delayed error (See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectCOPY.html#RESTObjectCOPY_Response)
+        response_body = response.body
+        if response_body =~ /<Error>/i
+          raise Awsum::Error.new(response)
+        else
+          true
+        end
+      end
+    end
+
+#private
+    #The host to make all requests against
+    def host
+      @host ||= 's3.amazonaws.com'
+    end
+
+    def host=(host)
+      @host = host
+    end
+  end
+end