awsum 0.5

data/lib/awsum/parser.rb

@@ -0,0 +1,18 @@
+require 'rexml/document'
+
+module Awsum
+  class Parser #:nodoc:
+    def parse(xml_text)
+      REXML::Document.parse_stream(xml_text, self)
+      result
+    end
+
+#--
+#Methods to be overridden by each Parser implementation
+    def result                      ; end
+    def tag_start(tag, attributes)  ; end
+    def text(text)                  ; end
+    def tag_end(tag)                ; end
+    def xmldecl(*args)              ; end
+  end
+end

data/lib/awsum/requestable.rb

@@ -0,0 +1,216 @@
+require 'base64'
+require 'cgi'
+require 'digest/md5'
+require 'openssl'
+require 'time'
+require 'uri'
+require 'awsum/error'
+
+require 'net/https'
+require 'awsum/net_fix'
+
+module Awsum
+  module Requestable #:nodoc:
+
+  private
+    # Sends a request with query parameters
+    #
+    # Used for EC2 requests
+    def send_query_request(params)
+      standard_options = {
+        'AWSAccessKeyId'   => @access_key,
+        'Version'          => API_VERSION,
+        'Timestamp'        => Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.000Z'),
+        'SignatureVersion' => SIGNATURE_VERSION,
+        'SignatureMethod'  => 'HmacSHA256'
+      }
+      params = standard_options.merge(params)
+
+      #Put parameters into query string format
+      params_string = params.delete_if{|k,v| v.nil?}.sort{|a,b|
+        a[0].to_s <=> b[0].to_s
+      }.collect{|key, val|
+        "#{CGI::escape(key.to_s)}=#{CGI::escape(val.to_s)}"
+      }.join('&')
+      params_string.gsub!('+', '%20')
+
+      #Create request signature
+      signature_string = "GET\n#{host}\n/\n#{params_string}"
+      signature = sign(signature_string, 'sha256')
+
+      #Attach signature to query string
+      params_string << "&Signature=#{CGI::escape(signature)}"
+
+      url = "https://#{host}/?#{params_string}"
+      response = process_request('GET', url)
+      if response.is_a?(Net::HTTPSuccess)
+        response
+      else
+        raise Awsum::Error.new(response)
+      end
+    end
+
+    # Sends a full request including headers and possible post data
+    #
+    # Used for S3 requests
+    def send_s3_request(method = 'GET', options = {}, &block)
+      bucket = options[:bucket] || ''
+      key = options[:key] || ''
+      parameters = options[:parameters] || {}
+      data = options[:data]
+      headers = {}
+      (options[:headers] || {}).each { |k,v| headers[k.downcase] = v }
+
+      #Add the host header
+      host_name = "#{bucket}#{bucket.blank? ? '' : '.'}#{host}"
+      headers['host'] = host_name
+
+      # Ensure required headers are in place
+      if !data.nil? && headers['content-md5'].nil?
+        if data.respond_to?(:read)
+          if data.respond_to?(:rewind)
+            digest = Digest::MD5.new
+            while chunk = data.read(1024 * 1024)
+              digest << chunk
+            end
+            data.rewind
+
+            headers['content-md5'] = Base64::encode64(digest.digest).gsub(/\n/, '')
+          end
+        else
+          headers['content-md5'] = Base64::encode64(Digest::MD5.digest(data)).gsub(/\n/, '')
+        end
+      end
+      if !data.nil? && headers['content-length'].nil?
+        headers['content-length'] = (data.respond_to?(:lstat) ? data.lstat.size : data.size).to_s
+      end
+      headers['date'] = Time.now.rfc822 if headers['date'].nil?
+      headers['content-type'] ||= ''
+      if !data.nil?
+        headers['expect'] = '100-continue'
+      end
+
+      signature_string = generate_rest_signature_string(method, bucket, key, parameters, headers)
+      puts "signature_string: \n#{signature_string}\n\n" if ENV['DEBUG']
+
+      signature = sign(signature_string)
+
+      headers['authorization'] = "AWS #{@access_key}:#{signature}"
+
+      url = "https://#{host_name}#{key[0..0] == '/' ? '' : '/'}#{key}#{parameters.size == 0 ? '' : "?#{parameters.collect{|k,v| v.nil? ? k.to_s : "#{CGI::escape(k.to_s)}=#{CGI::escape(v.to_s)}"}.join('&')}"}"
+
+      process_request(method, url, headers, data) do |response|
+        if response.is_a?(Net::HTTPSuccess)
+          if block_given?
+            block.call(response)
+          else
+            response
+          end
+        else
+          raise Awsum::Error.new(response)
+        end
+      end
+    end
+
+    def generate_rest_signature_string(method, bucket, key, parameters, headers)
+      canonicalized_amz_headers = headers.sort{|a,b| a[0].to_s.downcase <=> b[0].to_s.downcase }.collect{|k, v| "#{k.to_s.downcase}:#{(v.is_a?(Array) ? v.join(',') : v).gsub(/\n/, ' ')}" if k =~ /\Ax-amz-/i }.compact.join("\n")
+      canonicalized_resource = "#{bucket.blank? ? '' : '/'}#{bucket}#{key[0..0] == '/' ? '' : '/'}#{key}"
+      ['acl', 'torrent', 'logging', 'location'].each do |sub_resource|
+        if parameters.has_key?(sub_resource)
+          canonicalized_resource << "?#{sub_resource}"
+          break
+        end
+      end
+
+      signature_string = "#{method}\n#{headers['content-md5']}\n#{headers['content-type']}\n#{headers['x-amz-date'].nil? ? headers['date'] : ''}\n#{canonicalized_amz_headers}#{canonicalized_amz_headers.blank? ? '' : "\n"}#{canonicalized_resource}"
+    end
+
+    def generate_s3_signed_request_url(method, bucket, key, expires = nil)
+      signature_string = generate_rest_signature_string(method, bucket, key, {}, {'date' => expires})
+      signature = sign(signature_string)
+      "http://#{bucket}#{bucket.blank? ? '' : '.'}#{host}#{key[0..0] == '/' ? '' : '/'}#{key}?AWSAccessKeyId=#{@access_key}&Signature=#{CGI::escape(signature)}&Expires=#{expires}"
+    end
+
+    def process_request(method, url, headers = {}, data = nil, &block)
+      #TODO: Allow secure/non-secure
+      uri = URI.parse(url)
+      uri.scheme = 'https'
+      uri.port = 443
+
+      #puts uri.to_s
+
+      Net::HTTP.version_1_1
+      con = Net::HTTP.new(uri.host, uri.port)
+      con.use_ssl = true
+      con.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      con.start do |http|
+        case method.upcase
+          when 'GET'
+            request = Net::HTTP::Get.new("#{uri.path}#{"?#{uri.query}" if uri.query}")
+          when 'POST'
+            request = Net::HTTP::Post.new("#{uri.path}#{"?#{uri.query}" if uri.query}")
+          when 'PUT'
+            request = Net::HTTP::Put.new("#{uri.path}#{"?#{uri.query}" if uri.query}")
+          when 'DELETE'
+            request = Net::HTTP::Delete.new("#{uri.path}#{"?#{uri.query}" if uri.query}")
+          when 'HEAD'
+            request = Net::HTTP::Head.new("#{uri.path}#{"?#{uri.query}" if uri.query}")
+        end
+        request.initialize_http_header(headers)
+
+        unless data.nil?
+          if data.respond_to?(:read)
+            request.body_stream = data
+          else
+            request.body = data
+          end
+        end
+
+        if block_given?
+          http.request(request) do |response|
+            handle_response response, &block
+          end
+        else
+          response = http.request(request)
+          handle_response response
+        end
+      end
+    end
+
+    def handle_response(response, &block)
+      case response
+        when Net::HTTPSuccess
+          if block_given?
+            block.call(response)
+          else
+            response
+          end
+        when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPTemporaryRedirect
+          new_uri = URI.parse(response['location'])
+          uri.host = new_uri.host
+          uri.path = "#{new_uri.path}#{uri.path unless uri.path = '/'}"
+          process_request(method, uri.to_s, headers, data, &block)
+        else
+          response
+      end
+    end
+
+    # Converts an array of paramters into <param_name>.<num> format
+    def array_to_params(arr, param_name)
+      arr = [arr] unless arr.is_a?(Array)
+      params = {}
+      arr.each_with_index do |value,i|
+        params["#{param_name}.#{i+1}"] = value
+      end
+      params
+    end
+
+    # Sign a string with a digest, wrap in HMAC digest and base64 encode
+    #
+    # ===Returns
+    # base64 encoded string with newlines removed
+    def sign(string, digest = 'sha1')
+      Base64::encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new(digest), @secret_key, string)).gsub(/\n/, '')
+    end
+  end
+end

data/lib/awsum/s3.rb

@@ -0,0 +1,220 @@
+require 'awsum'
+require 'awsum/s3/bucket'
+require 'awsum/s3/object'
+require 'awsum/s3/headers'
+
+module Awsum
+  # Handles all interaction with Amazon S3
+  #
+  #--
+  # TODO: Change this to S3
+  # ==Getting Started
+  # Create an Awsum::Ec2 object and begin calling methods on it.
+  #   require 'rubygems'
+  #   require 'awsum'
+  #   ec2 = Awsum::Ec2.new('your access id', 'your secret key')
+  #   images = ec2.my_images
+  #   ...
+  #
+  # All calls to EC2 can be done directly in this class, or through a more
+  # object oriented way through the various returned classes
+  #
+  # ==Examples
+  #   ec2.image('ami-ABCDEF').run
+  #
+  #   ec2.instance('i-123456789').volumes.each do |vol|
+  #     vol.create_snapsot
+  #   end
+  #
+  #   ec2.regions.each do |region|
+  #     region.use
+  #       images.each do |img|
+  #         puts "#{img.id} - #{region.name}"
+  #       end
+  #     end
+  #   end
+  #
+  # ==Errors
+  # All methods will raise an Awsum::Error if an error is returned from Amazon
+  #
+  # ==Missing Methods
+  # If you need any of this functionality, please consider getting involved
+  # and help complete this library.
+  class S3
+    include Awsum::Requestable
+
+    # Create an new S3 instance
+    #
+    # The access_key and secret_key are both required to do any meaningful work.
+    #
+    # If you want to get these keys from environment variables, you can do that
+    # in your code as follows:
+    #   s3 = Awsum::S3.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'])
+    def initialize(access_key = nil, secret_key = nil)
+      @access_key = access_key
+      @secret_key = secret_key
+    end
+
+    # List all the Bucket(s)
+    def buckets
+      response = send_s3_request
+      parser = Awsum::S3::BucketParser.new(self)
+      parser.parse(response.body)
+    end
+
+    def bucket(name)
+      Bucket.new(self, name)
+    end
+
+    # Create a new Bucket
+    #
+    # ===Parameters
+    # * <tt>bucket_name</tt> - The name of the new bucket
+    # * <tt>location</tt> <i>(optional)</i> - Can be <tt>:default</tt>, <tt>:us</tt> or <tt>:eu</tt>
+    def create_bucket(bucket_name, location = :default)
+      raise ArgumentError.new('Bucket name cannot be in an ip address style') if bucket_name =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
+      raise ArgumentError.new('Bucket name can only have lowercase letters, numbers, periods (.), underscores (_) and dashes (-)') unless bucket_name =~ /^[\w\d][-a-z\d._]+[a-z\d._]$/
+      raise ArgumentError.new('Bucket name cannot contain a dash (-) next to a period (.)') if bucket_name =~ /\.-|-\./
+      raise ArgumentError.new('Bucket name must be between 3 and 63 characters') if bucket_name.size < 3 || bucket_name.size > 63
+
+      data = nil
+      if location == :eu
+        data = '<CreateBucketConfiguration><LocationConstraint>EU</LocationConstraint></CreateBucketConfiguration>'
+      end
+
+      response = send_s3_request('PUT', :bucket => bucket_name, :data => data)
+      response.is_a?(Net::HTTPSuccess)
+    end
+
+    def delete_bucket(bucket_name)
+      response = send_s3_request('DELETE', :bucket => bucket_name)
+      response.is_a?(Net::HTTPSuccess)
+    end
+
+    # List the Key(s) of a Bucket
+    #
+    # ===Parameters
+    # * <tt>bucket_name</tt> - The name of the bucket to search for keys
+    # ====Options
+    # * <tt>:prefix</tt> - Limits the response to keys which begin with the indicated prefix. You can use prefixes to separate a bucket into different sets of keys in a way similar to how a file system uses folders.
+    # * <tt>:marker</tt> - Indicates where in the bucket to begin listing. The list will only include keys that occur lexicographically after marker. This is convenient for pagination: To get the next page of results use the last key of the current page as the marker.
+    # * <tt>:max_keys</tt> - The maximum number of keys you'd like to see in the response body. The server might return fewer than this many keys, but will not return more.
+    # * <tt>:delimeter</tt> - Causes keys that contain the same string between the prefix and the first occurrence of the delimiter to be rolled up into a single result element in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere in the response.
+    def keys(bucket_name, options = {})
+      paramters = {}
+      paramters['prefix'] = options[:prefix] if options[:prefix]
+      paramters['marker'] = options[:marker] if options[:marker]
+      paramters['max_keys'] = options[:max_keys] if options[:max_keys]
+      paramters['prefix'] = options[:prefix] if options[:prefix]
+
+      response = send_s3_request('GET', :bucket => bucket_name, :paramters => paramters)
+      parser = Awsum::S3::ObjectParser.new(self)
+      parser.parse(response.body)
+    end
+
+    # Create a new Object in the specified Bucket
+    #
+    # ===Parameters
+    # * <tt>bucket_name</tt> - The name of the Bucket in which to store the Key
+    # * <tt>key</tt> - The name/path of the Key to store
+    # * <tt>data</tt> - The data to be stored in this Object
+    # * <tt>headers</tt> - Standard HTTP headers to be sent along
+    # * <tt>meta_headers</tt> - Meta headers to be stored along with the key
+    # * <tt>acl</tt> - A canned access policy, can be one of <tt>:private</tt>, <tt>:public_read</tt>, <tt>:public_read_write</tt> or <tt>:authenticated_read</tt>
+    def create_object(bucket_name, key, data, headers = {}, meta_headers = {}, acl = :private)
+      headers = headers.dup
+      meta_headers.each do |k,v|
+        headers[k =~ /^x-amz-meta-/i ? k : "x-amz-meta-#{k}"] = v
+      end
+      headers['x-amz-acl'] = acl.to_s.gsub(/_/, '-')
+
+      response = send_s3_request('PUT', :bucket => bucket_name, :key => key, :headers => headers, :data => data)
+      response.is_a?(Net::HTTPSuccess)
+    end
+
+    # Retrieve the headers for this Object
+    #
+    # All header methods map directly to the Net::HTTPHeader module
+    def object_headers(bucket_name, key)
+      response = send_s3_request('HEAD', :bucket => bucket_name, :key => key)
+      Headers.new(response)
+    end
+
+    # Retrieve the data stored for the specified Object
+    #
+    # You can get the data as a single call or add a block to retrieve the data in chunks
+    # ==Examples
+    #   data = s3.object_data('test-bucket', 'key')
+    #
+    # or
+    #
+    #   s3.object_data('test-bucket', 'key') do |chunk|
+    #     # handle chunk
+    #     puts chunk
+    #   end
+    def object_data(bucket_name, key, &block)
+      send_s3_request('GET', :bucket => bucket_name, :key => key) do |response|
+        if block_given?
+          response.read_body &block
+          return true
+        else
+          return response.body
+        end
+      end
+    end
+
+    # Deletes an Object from a Bucket
+    def delete_object(bucket_name, key)
+      response = send_s3_request('DELETE', :bucket => bucket_name, :key => key)
+      response.is_a?(Net::HTTPSuccess)
+    end
+
+    # Copy the contents of an Object to another key and/or bucket
+    #
+    # ===Parameters
+    # * <tt>source_bucket_name</tt> - The name of the Bucket from which to copy
+    # * <tt>source_key</tt> - The name of the Key from which to copy
+    # * <tt>destination_bucket_name</tt> - The name of the Bucket to which to copy (Can be nil if copying within the same bucket, or updating header data of existing Key)
+    # * <tt>destination_key</tt> - The name of the Key to which to copy (Can be nil if copying to a new bucket with same key, or updating header data of existing Key)
+    # * <tt>headers</tt> - If not nil, the headers are replaced with this information
+    # * <tt>meta_headers</tt> - If not nil, the meta headers are replaced with this information
+    #
+    #--
+    # TODO: Need to handle copy-if-... headers
+    def copy_object(source_bucket_name, source_key, destination_bucket_name = nil, destination_key= nil, headers = nil, meta_headers = nil)
+      raise ArgumentError.new('You must include one of destination_bucket_name, destination_key or headers to be replaced') if destination_bucket_name.nil? && destination_key.nil? && headers.nil? && meta_headers.nil?
+
+      headers = {
+          'x-amz-copy-source' => "/#{source_bucket_name}/#{source_key}",
+          'x-amz-metadata-directive' => (((destination_bucket_name.nil? && destination_key.nil?) || !(headers.nil? || meta_headers.nil?)) ? 'REPLACE' : 'COPY')
+        }.merge(headers||{})
+      meta_headers.each do |k,v|
+        headers[k =~ /^x-amz-meta-/i ? k : "x-amz-meta-#{k}"] = v
+      end unless meta_headers.nil?
+
+      destination_bucket_name ||= source_bucket_name
+      destination_key ||= source_key
+
+      response = send_s3_request('PUT', :bucket => destination_bucket_name, :key => destination_key, :headers => headers, :data => nil)
+      if response.is_a?(Net::HTTPSuccess)
+        #Check for delayed error (See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectCOPY.html#RESTObjectCOPY_Response)
+        response_body = response.body
+        if response_body =~ /<Error>/i
+          raise Awsum::Error.new(response)
+        else
+          true
+        end
+      end
+    end
+
+#private
+    #The host to make all requests against
+    def host
+      @host ||= 's3.amazonaws.com'
+    end
+
+    def host=(host)
+      @host = host
+    end
+  end
+end