right_aws 1.2.0 → 1.3.0

data/History.txt

@@ -47,4 +47,20 @@ r1581 | tve | 2007-08-24 16:21:45 -0700 (Fri, 24 Aug 2007) | 1 line
 Improved RightAws documentation
 ------------------------------------------------------------------------
 
+== 1.3.0 / 2007-09-26
+------------------------------------------------------------------------
+r1754 | todd | 2007-09-19 13:48:34 -0700 (Wed, 19 Sep 2007) | 6 lines
+
+(#487, #488) Consolidate a lot of code that was repeated in three places.
+Fix error handling path when using streaming GET interfaces with S3.
+Also add a stub for RightHttpConnection which
+allows more control over the unit tests.  Expand the unit tests and coverage
+tests. 
+------------------------------------------------------------------------
+r1755 | todd | 2007-09-19 14:29:19 -0700 (Wed, 19 Sep 2007) | 2 lines
+
+(#487) RDoc fixes after code consolidation
+
+------------------------------------------------------------------------
+
 

data/Manifest.txt

@@ -15,8 +15,10 @@ test/ec2/test_helper.rb
 test/ec2/test_right_ec2.rb
 test/s3/test_helper.rb
 test/s3/test_right_s3.rb
+test/s3/test_right_s3_stubbed.rb
 test/sqs/test_helper.rb
 test/sqs/test_right_sqs.rb
 test/ts_right_aws.rb
 test/test_credentials.rb
+test/http_connection.rb
 

data/Rakefile

@@ -7,6 +7,8 @@ require 'rcov/rcovtask'
 $: << File.dirname(__FILE__)
 require 'lib/right_aws.rb'
 
+testglobs =     ["test/ts_right_aws.rb"]
+
 Hoe.new('right_aws', RightAws::VERSION::STRING) do |p|
   p.rubyforge_name = 'rightaws'
   p.author = 'RightScale, Inc.'
@@ -17,14 +19,44 @@ Hoe.new('right_aws', RightAws::VERSION::STRING) do |p|
   p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
   p.remote_rdoc_dir = "/right_aws_gem_doc"
   p.extra_deps = [['right_http_connection','>= 0.1.4']]
-  p.test_globs = ['test/ts_right_aws.rb']
+  p.test_globs = testglobs 
 end
 
 desc "Analyze code coverage of the unit tests."
 Rcov::RcovTask.new do |t|
-  t.test_files = FileList["test/ts*.rb"]
+  t.test_files = FileList[testglobs]
   #t.verbose = true     # uncomment to see the executed command
 end
+ 
+desc "Test just the SQS interface"
+task :testsqs do
+  require 'test/test_credentials'
+  require 'test/http_connection'
+  TestCredentials.get_credentials
+  require 'test/sqs/test_right_sqs.rb'
+end
+
+desc "Test just the S3 interface"
+task :tests3 do
+  require 'test/test_credentials'
+  require 'test/http_connection'
+  TestCredentials.get_credentials
+  require 'test/s3/test_right_s3.rb'
+end
 
+desc "Test just the S3 interface using local stubs"
+task :tests3local do
+  require 'test/test_credentials'
+  require 'test/http_connection'
+  TestCredentials.get_credentials
+  require 'test/s3/test_right_s3_stubbed.rb'
+end
+
+desc "Test just the EC2 interface"
+task :testec2 do
+  require 'test/test_credentials'
+  TestCredentials.get_credentials
+  require 'test/ec2/test_right_ec2.rb'
+end
 
 # vim: syntax=Ruby

data/lib/awsbase/benchmark_fix.rb

@@ -25,7 +25,7 @@
 
 # A hack because there's a bug in add! in Benchmark::Tms
 module Benchmark  #:nodoc:
-  class Tms
+  class Tms #:nodoc:
     def add!(&blk)
       t = Benchmark::measure(&blk)
       @utime  = utime + t.utime

data/lib/awsbase/right_awsbase.rb

@@ -24,9 +24,30 @@
 # Test
 module RightAws
 
-  # Text, if found in an error message returned by AWS, indicates that this may be a transient
-  # error. Transient errors are automatically retried with exponential back-off.
-  AMAZON_PROBLEMS = [ 'internal service error', 
+  class AwsUtils #:nodoc:
+    def self.sign(aws_secret_access_key, auth_string)
+      Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new("sha1"), aws_secret_access_key, auth_string)).strip
+    end
+
+  end
+
+  class AwsBenchmarkingBlock #:nodoc:
+    attr_accessor :xml, :service
+    def initialize
+      # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
+      @service = Benchmark::Tms.new()
+      # Benchmark::Tms instance for XML parsing benchmarking.
+      @xml = Benchmark::Tms.new()
+    end
+  end
+
+  class RightAwsBase
+
+    # Amazon HTTP Error handling
+
+    # Text, if found in an error message returned by AWS, indicates that this may be a transient
+    # error. Transient errors are automatically retried with exponential back-off.
+    AMAZON_PROBLEMS = [ 'internal service error', 
                       'is currently unavailable', 
                       'no response from', 
                       'Please try again',
@@ -36,12 +57,141 @@ module RightAws
                       'This application is not currently available',
                       'InsufficientInstanceCapacity'
                     ]
+    @@amazon_problems = AMAZON_PROBLEMS
+      # Returns a list of Amazon service responses which are known to be transient problems. 
+      # We have to re-request if we get any of them, because the problem will probably disappear. 
+      # By default this method returns the same value as the AMAZON_PROBLEMS const.
+    def self.amazon_problems
+      @@amazon_problems
+    end
+    
+      # Sets the list of Amazon side problems.  Use in conjunction with the
+      # getter to append problems.
+    def self.amazon_problems=(problems_list)
+      @@amazon_problems = problems_list
+    end
+
+  end
+
+  module RightAwsBaseInterface
+
+      # Current aws_access_key_id
+    attr_reader :aws_access_key_id
+      # Last HTTP request object
+    attr_reader :last_request
+      # Last HTTP response object
+    attr_reader :last_response
+      # Last AWS errors list (used by AWSErrorHandler)
+    attr_accessor :last_errors
+      # Last AWS request id (used by AWSErrorHandler)
+    attr_accessor :last_request_id
+      # Logger object
+    attr_accessor :logger
+      # Initial params hash
+    attr_accessor :params
+
+    def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
+      @params = params
+      raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
+        if aws_access_key_id.blank? || aws_secret_access_key.blank?
+      @aws_access_key_id     = aws_access_key_id
+      @aws_secret_access_key = aws_secret_access_key
+      @params[:server]       ||= service_info[:default_host]
+      @params[:port]         ||= service_info[:default_port]
+      @params[:protocol]     ||= service_info[:default_protocol]
+      @params[:multi_thread] ||= defined?(AWS_DAEMON)
+      @logger = @params[:logger]
+      @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
+      @logger = Logger.new(STDOUT)   if !@logger
+      @logger.info "New #{self.class.name} using #{@params[:multi_thread] ? 'multi' : 'single'}-threaded mode"
+      @error_handler = nil
+    end
+
+    def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
+      AwsError::on_aws_exception(self, options)
+    end
+    
+      # Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
+    def multi_thread
+      @params[:multi_thread]
+    end
+
+    def request_info_impl(connection, benchblock, request, parser, &block) #:nodoc:
+      @last_request  = request[:request]
+      @last_response = nil
+      response=nil
+      blockexception = nil
+
+      if(block != nil)
+        # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
+        # an exception may get thrown in the block body (which is high-level
+        # code either here or in the application) but gets caught in the
+        # low-level code of HttpConnection.  The solution is not to let any
+        # exception escape the block that we pass to HttpConnection::request.
+        # Exceptions can originate from code directly in the block, or from user
+        # code called in the other block which is passed to response.read_body.
+        benchblock.service.add! do
+          responsehdr = connection.request(request) do |response|
+          #########
+            begin
+              @last_response = response
+              if response.is_a?(Net::HTTPSuccess)
+                @error_handler = nil
+                response.read_body(&block)
+              else
+                @error_handler = AWSErrorHandler.new(self, parser, self.class.amazon_problems) unless @error_handler
+                check_result   = @error_handler.check(request)
+                if check_result
+                  @error_handler = nil
+                  return check_result 
+                end
+                raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
+              end
+            rescue Exception => e
+              blockexception = e
+            end
+          end
+          #########
+
+          #OK, now we are out of the block passed to the lower level
+          if(blockexception)
+            raise blockexception
+          end
+          benchblock.xml.add! do
+            parser.parse(responsehdr)
+          end
+          return parser.result
+        end
+      else
+        benchblock.service.add!{ response = connection.request(request) }
+          # check response for errors...
+        @last_response = response
+        if response.is_a?(Net::HTTPSuccess)
+          @error_handler = nil
+          benchblock.xml.add! { parser.parse(response) }
+          return parser.result
+        else
+          @error_handler = AWSErrorHandler.new(self, parser, self.class.amazon_problems) unless @error_handler
+          check_result   = @error_handler.check(request)
+          if check_result
+            @error_handler = nil
+            return check_result 
+          end
+          raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
+        end
+      end
+    rescue
+      @error_handler = nil
+      raise
+    end
+
+  end
+
 
   # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
   # web services raise this type of error.
   # Attribute inherited by RuntimeError:
   #  message    - the text of the error, generally as returned by AWS in its XML response.
-
   class AwsError < RuntimeError
     
     # either an array of errors where each item is itself an array of [code, message]),

data/lib/ec2/right_ec2.rb

@@ -63,7 +63,8 @@ module RightAws
   # Error handling: all operations raise an RightAws::AwsError in case
   # of problems. Note that transient errors are automatically retried.
     
-  class Ec2
+  class Ec2 < RightAwsBase
+    include RightAwsBaseInterface
     
     SIGNATURE_VERSION = "1"
     # Amazon EC2 API version being used
@@ -75,43 +76,13 @@ module RightAws
     # Default addressing type (public=NAT, direct=no-NAT) used when launching instances.
     DEFAULT_ADDRESSING_TYPE =  'public'
     DNS_ADDRESSING_SET      = ['public','direct']
-    
-    # A list of Amazon problems we can handle by AWSErrorHandler.
-    @@amazon_problems = RightAws::AMAZON_PROBLEMS
-    
-    # Current aws_access_key_id
-    attr_reader :aws_access_key_id
-    # Last HTTP request object
-    attr_reader :last_request
-    # Last HTTP response object
-    attr_reader :last_response
-    # Last AWS errors list (used by AWSErrorHandler)
-    attr_accessor :last_errors
-    # Last AWS request id (used by AWSErrorHandler)
-    attr_accessor :last_request_id
-    # Logger object, used by this class to generate log messages
-    attr_accessor :logger
-    # Option params passed into new
-    attr_accessor :params
-    
-    @@bench_ec2 = Benchmark::Tms.new()
-    @@bench_xml = Benchmark::Tms.new()
-    
-    # Benchmark::Tms instance that accumulates time spent in requests to EC2.
-    def self.bench_ec2;  @@bench_ec2;  end
-    
-    # Benchmark::Tms instance that accumulates time spent in XML parsing.
-    def self.bench_xml; @@bench_xml; end
 
-    # Returns a list of Amazon service responses which are known to be transient errors.
-    # By default returns the same value as AMAZON_PROBLEMS const. See AWSErrorHandler.
-    def self.amazon_problems
-      @@amazon_problems
+    @@bench = AwsBenchmarkingBlock.new
+    def self.bench_xml
+      @@bench.xml
     end
-
-    # Sets the list of Amazon problems that are automatically retried. See AWSErrorHandler.
-    def self.amazon_problems=(problems_list)
-      @@amazon_problems = problems_list
+    def self.bench_ec2
+      @@bench.service
     end
     
     # Create a new handle to an EC2 account. All handles share the same per process or per thread
@@ -124,31 +95,10 @@ module RightAws
     # * <tt>:logger</tt>: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT
     #
     def initialize(aws_access_key_id, aws_secret_access_key, params={})
-      @params = params
-      raise AwsError.new("AWS access keys are required to operate on EC2") \
-        if aws_access_key_id.blank? || aws_secret_access_key.blank?
-      @aws_access_key_id     = aws_access_key_id
-      @aws_secret_access_key = aws_secret_access_key
-        # params
-      @params[:server]       ||= DEFAULT_HOST
-      @params[:port]         ||= DEFAULT_PORT
-      @params[:protocol]     ||= DEFAULT_PROTOCOL
-      @params[:multi_thread] ||= defined?(AWS_DAEMON)
-        # set logger
-      @logger = @params[:logger]
-      @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
-      @logger = Logger.new(STDOUT)   if !@logger
-      @logger.info "New #{self.class.name} using #{@params[:multi_thread] ? 'multi' : 'single'}-threaded mode"
+      init({:name=>'EC2', :default_host => DEFAULT_HOST, :default_port => DEFAULT_PORT, :default_protocol => DEFAULT_PROTOCOL}, 
+           aws_access_key_id, aws_secret_access_key, params)
     end
 
-    def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
-      AwsError::on_aws_exception(self, options)
-    end
-    
-      # Return +true+ if this RightEc2NativeQuery instance works in multi_thread mode and +false+ otherwise.
-    def multi_thread
-      @params[:multi_thread]
-    end
 
     def generate_request(action, param={}) #:nodoc:
       timestamp    = ( Time::now ).utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
@@ -159,7 +109,7 @@ module RightAws
                       "SignatureVersion"  => SIGNATURE_VERSION }
       request_hash.update(param)
       request_data   = request_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
-      request_hash.update('Signature' => Base64.encode64( OpenSSL::HMAC.digest( OpenSSL::Digest::Digest.new( "sha1" ), @aws_secret_access_key, request_data)).strip)
+      request_hash.update('Signature' =>  AwsUtils::sign(@aws_secret_access_key, request_data))
       request_params = request_hash.to_a.collect{|key,val| key + "=" + CGI::escape(val) }.join("&")
       request        = Net::HTTP::Get.new("/?#{request_params}")
         # prepare output hash
@@ -174,29 +124,7 @@ module RightAws
     def request_info(request, parser)  #:nodoc:
       thread = @params[:multi_thread] ? Thread.current : Thread.main
       thread[:ec2_connection] ||= Rightscale::HttpConnection.new(:exception => AwsError)
-      @last_request  = request[:request]
-      @last_response = nil
-      response=nil
-      
-      @@bench_ec2.add!{ response = thread[:ec2_connection].request(request) }
-        # check response for errors...
-      @last_response = response
-      if response.is_a?(Net::HTTPSuccess)
-        @error_handler = nil
-        @@bench_xml.add! { parser.parse(response) }
-        return parser.result
-      else
-        @error_handler = AWSErrorHandler.new(self, parser, @@amazon_problems) unless @error_handler
-        check_result   = @error_handler.check(request)
-        if check_result
-          @error_handler = nil
-          return check_result 
-        end
-        raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
-      end
-    rescue
-      @error_handler = nil
-      raise
+      request_info_impl(thread[:ec2_connection], @@bench, request, parser)
     end
 
 

data/lib/right_aws.rb

@@ -48,7 +48,7 @@ require 'sqs/right_sqs'
 module RightAws #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 1
-    MINOR = 2
+    MINOR = 3
     TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
@@ -59,7 +59,7 @@ end
 
 # We also want everything available in the Rightscale namespace for backward
 # compatibility reasons.
-module Rightscale
+module Rightscale #:nodoc:
   include RightAws
   extend RightAws
 end

data/lib/s3/right_s3_interface.rb

@@ -23,7 +23,8 @@
 
 module RightAws
 
-  class S3Interface
+  class S3Interface < RightAwsBase
+    include RightAwsBaseInterface
     
     DEFAULT_HOST           = 's3.amazonaws.com'
     DEFAULT_PORT           = 443
@@ -33,54 +34,15 @@ module RightAws
     AMAZON_HEADER_PREFIX   = 'x-amz-'
     AMAZON_METADATA_PREFIX = 'x-amz-meta-'
 
-      # A list if Amazons problems we can handle by AWSErrorHandler.
-    @@amazon_problems = RightAws::AMAZON_PROBLEMS
-   
-	# TODO TRB 6/19/07 - all the below accessors are shared in the 
-	# three service gems.  See if is it reasonable to stick these
-	# in an interface class in right_awsbase that we can mixin
-	#
-	# Same for the benchmarking code - all three service gems have
-	# the same.  Break out into a helper class.  Also look at the 
-	# benchmarking fix as a good thing to move to common code.
- 
-      # Current aws_access_key_id
-    attr_reader :aws_access_key_id
-      # Last HTTP request object
-    attr_reader :last_request
-      # Last HTTP response object
-    attr_reader :last_response
-      # Last AWS errors list (used by AWSErrorHandler)
-    attr_accessor :last_errors
-      # Last AWS request id (used by AWSErrorHandler)
-    attr_accessor :last_request_id
-      # Logger object
-    attr_accessor :logger
-      # Initial params hash
-    attr_accessor :params
-    
-    @@bench_s3  = Benchmark::Tms.new()
-    @@bench_xml = Benchmark::Tms.new()
-
-      # Benchmark::Tms instance for S3 access benchmark.
-    def self.bench_s3;  @@bench_s3;  end  
-    
-      # Benchmark::Tms instance for XML parsing benchmark.
-    def self.bench_xml; @@bench_xml; end  # For benchmark puposes.
-
-      # Returns a list of Amazon service responses which are known to be transient problems. 
-      # We have to re-request if we get any of them, because the problem will probably disappear. 
-      # By default this method returns the same value as the AMAZON_PROBLEMS const.
-    def self.amazon_problems
-      @@amazon_problems
+    @@bench = AwsBenchmarkingBlock.new
+    def self.bench_xml
+      @@bench.xml
     end
-
-      # Sets the list of Amazon side problems.  Use in conjunction with the
-      # getter to append problems.
-    def self.amazon_problems=(problems_list)
-      @@amazon_problems = problems_list
+    def self.bench_s3
+      @@bench.service
     end
 
+
       # Creates new RightS3 instance.
       #
       #  s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX', {:multi_thread => true, :logger => Logger.new('/tmp/x.log')}) #=> #<RightS3:0xb7b3c27c>
@@ -94,39 +56,10 @@ module RightAws
       #     :logger       => Logger Object}       # Logger instance: logs to STDOUT if omitted }
       #
     def initialize(aws_access_key_id, aws_secret_access_key, params={})
-      @params = params
-      raise AwsError.new("AWS access keys are required to operate on S3") \
-        if aws_access_key_id.blank? || aws_secret_access_key.blank?
-
-	# TODO TRB 6/19/07 - keys, basic params, and logger are all 
-	# candidates to break out into a helper class common to all 
-	# service gems.  Stick the helper in right_awsbase
-      @aws_access_key_id     = aws_access_key_id
-      @aws_secret_access_key = aws_secret_access_key
-        # params
-      @params[:server]       ||= DEFAULT_HOST
-      @params[:port]         ||= DEFAULT_PORT
-      @params[:protocol]     ||= DEFAULT_PROTOCOL
-      @params[:multi_thread] ||= defined?(AWS_DAEMON)
-        # set logger
-      @logger = @params[:logger]
-      @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
-      @logger = Logger.new(STDOUT)   if !@logger
-      @logger.info "New #{self.class.name} using #{@params[:multi_thread] ? 'multi' : 'single'}-threaded mode"
-      @error_handler = nil
-    end
-
-	# TODO TRB 6/19/07 - Service gem common method
-    def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
-      RightAws::AwsError::on_aws_exception(self, options)
+      init({:name=>'S3', :default_host => DEFAULT_HOST, :default_port => DEFAULT_PORT, :default_protocol => DEFAULT_PROTOCOL}, 
+           aws_access_key_id, aws_secret_access_key, params)
     end
 
-	# TODO TRB 6/19/07 - Service gem common method
-
-      # Return the +true+ if this RightS3 instance works in multi_thread state and +false+ otherwise.
-    def multi_thread
-      @params[:multi_thread]
-    end
 
   #-----------------------------------------------------------------
   #      Requests
@@ -172,7 +105,7 @@ module RightAws
       headers.each      { |key, value| request[key.to_s] = value }
         #generate auth strings
       auth_string = canonical_string(request.method, request.path, request.to_hash)
-      signature   = Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new("sha1"), @aws_secret_access_key, auth_string)).strip
+      signature   = AwsUtils::sign(@aws_secret_access_key, auth_string)
         # set other headers
       request['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}"
         # prepare output hash
@@ -184,59 +117,10 @@ module RightAws
 
       # Sends request to Amazon and parses the response.
       # Raises AwsError if any banana happened.
-      # TODO TRB 6/19/07: 
-      # request_info is a candidate to move to right_awsbase      
-      # because it currently appears (in identical form) in right_s3,
-      # right_ec2, and right_sqs
     def request_info(request, parser, &block) # :nodoc:
       thread = @params[:multi_thread] ? Thread.current : Thread.main
       thread[:s3_connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError)
-      @last_request  = request[:request]
-      @last_response = nil
-      response=nil
-
-      if(block != nil)
-        @@bench_s3.add! do
-          responsehdr = thread[:s3_connection].request(request) do |response|
-            if response.is_a?(Net::HTTPSuccess)
-              @error_handler = nil
-              response.read_body(&block)
-            else
-              @error_handler = AWSErrorHandler.new(self, parser, @@amazon_problems) unless @error_handler
-              check_result   = @error_handler.check(request)
-              if check_result
-                @error_handler = nil
-                return check_result 
-              end
-              raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
-            end
-          end
-          @@bench_xml.add! do
-            parser.parse(responsehdr)
-          end
-          return parser.result
-        end
-      else
-        @@bench_s3.add!{ response = thread[:s3_connection].request(request) }
-          # check response for errors...
-        @last_response = response
-        if response.is_a?(Net::HTTPSuccess)
-          @error_handler = nil
-          @@bench_xml.add! { parser.parse(response) }
-          return parser.result
-        else
-          @error_handler = AWSErrorHandler.new(self, parser, @@amazon_problems) unless @error_handler
-          check_result   = @error_handler.check(request)
-          if check_result
-            @error_handler = nil
-            return check_result 
-          end
-          raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
-        end
-      end
-    rescue
-      @error_handler = nil
-      raise
+      request_info_impl(thread[:s3_connection], @@bench, request, parser, &block)
     end
 
 

data/lib/sqs/right_sqs_interface.rb

@@ -23,7 +23,8 @@
 
 module RightAws
 
-  class SqsInterface
+  class SqsInterface < RightAwsBase
+    include RightAwsBaseInterface
     
     SIGNATURE_VERSION = "1"
     API_VERSION       = "2007-05-01"
@@ -32,45 +33,17 @@ module RightAws
     DEFAULT_PROTOCOL  = 'https'
     REQUEST_TTL       = 30
     DEFAULT_VISIBILITY_TIMEOUT = 30
-      # A list of Amazon problems we can handle via AWSErrorHandler.
-    @@amazon_problems = RightAws::AMAZON_PROBLEMS
-
-      # Current aws_access_key_id
-    attr_reader :aws_access_key_id
-      # Last HTTP request object
-    attr_reader :last_request
-      # Last HTTP response object
-    attr_reader :last_response
-      # Last AWS errors list (used by AWSErrorHandler)
-    attr_accessor :last_errors
-      # Last AWS request id (used by AWSErrorHandler)
-    attr_accessor :last_request_id
-      # Logger object
-    attr_accessor :logger
-      # Initial params hash
-    attr_accessor :params
-    
-    @@bench_sqs = Benchmark::Tms.new()
-    @@bench_xml = Benchmark::Tms.new()
-    
-      # Benchmark::Tms instance for SQS access benchmarking.
-    def self.bench_sqs; @@bench_sqs;  end  
-      # Benchmark::Tms instance for XML parsing benchmarking.
-    def self.bench_xml; @@bench_xml; end  # For benchmark purposes.
 
-      # Returns a list of Amazon service responses which are known to be transient problems. 
-      # We have to re-request if we get any of them, because the problem will probably disappear. 
-      # By default this method returns the same value as the AMAZON_PROBLEMS const.
-    def self.amazon_problems
-      @@amazon_problems
+
+    @@bench = AwsBenchmarkingBlock.new
+    def self.bench_xml
+      @@bench.xml
     end
-    
-      # Sets the list of Amazon side problems.  Use in conjunction with the
-      # getter to append problems.
-    def self.amazon_problems=(problems_list)
-      @@amazon_problems = problems_list
+    def self.bench_sqs
+      @@bench.service
     end
 
+
       # Creates a new SqsInterface instance.
       #
       #  sqs = RightAws::SqsInterface.new('1E3GDYEOGFJPIT75KDT40','hgTHt68JY07JKUY08ftHYtERkjgtfERn57DFE379', {:multi_thread => true, :logger => Logger.new('/tmp/x.log')}) #=> <RightSqs:0xb7af6264>
@@ -83,31 +56,10 @@ module RightAws
       #     :logger       => Logger Object}        # Logger instance: logs to STDOUT if omitted }
       #
     def initialize(aws_access_key_id, aws_secret_access_key, params={})
-      @params = params
-      raise AwsError.new("AWS access keys are required to operate on SQS") \
-        if aws_access_key_id.blank? || aws_secret_access_key.blank?
-      @aws_access_key_id     = aws_access_key_id
-      @aws_secret_access_key = aws_secret_access_key
-        # params
-      @params[:server]       ||= DEFAULT_HOST
-      @params[:port]         ||= DEFAULT_PORT
-      @params[:protocol]     ||= DEFAULT_PROTOCOL
-      @params[:multi_thread] ||= defined?(AWS_DAEMON)
-        # set logger
-      @logger = @params[:logger]
-      @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
-      @logger = Logger.new(STDOUT)   if !@logger
-      @logger.info "New #{self.class.name} using #{@params[:multi_thread] ? 'multi' : 'single'}-threaded mode"
-    end
-
-    def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
-      AwsError::on_aws_exception(self, options)
+      init({:name=>'SQS', :default_host => DEFAULT_HOST, :default_port => DEFAULT_PORT, :default_protocol => DEFAULT_PROTOCOL}, 
+           aws_access_key_id, aws_secret_access_key, params)
     end
 
-      # Return +true+ if this RightS3 instance is running in multi_thread state and +false+ otherwise.
-    def multi_thread
-      @params[:multi_thread]
-    end
 
   #-----------------------------------------------------------------
   #      Requests
@@ -129,7 +81,7 @@ module RightAws
                        "SignatureVersion" => SIGNATURE_VERSION }
       request_hash.update(param)
       request_data   = request_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
-      request_hash['Signature'] = Base64.encode64( OpenSSL::HMAC.digest( OpenSSL::Digest::Digest.new( "sha1" ), @aws_secret_access_key, request_data)).strip
+      request_hash['Signature'] = AwsUtils::sign(@aws_secret_access_key, request_data)
       request_params = request_hash.to_a.collect{|key,val| key.to_s + "=" + CGI::escape(val.to_s) }.join("&")
       request        = Net::HTTP::Get.new("#{queue_uri}?#{request_params}")
         # prepare output hash
@@ -156,7 +108,7 @@ module RightAws
       request['Date']         = Time.now.httpdate
         # generate authorization string
       auth_string = "#{method.upcase}\n#{request['content-md5']}\n#{request['Content-Type']}\n#{request['Date']}\n#{CGI::unescape(queue_uri)}"
-      signature   = Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new("sha1"), @aws_secret_access_key, auth_string)).strip
+      signature   = AwsUtils::sign(@aws_secret_access_key, auth_string)
         # set other headers
       request['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}"
       request['AWS-Version']   = API_VERSION
@@ -173,29 +125,7 @@ module RightAws
     def request_info(request, parser) # :nodoc:
       thread = @params[:multi_thread] ? Thread.current : Thread.main
       thread[:sqs_connection] ||= Rightscale::HttpConnection.new(:exception => AwsError)
-      @last_request  = request[:request]
-      @last_response = nil
-      response=nil
-      
-      @@bench_sqs.add!{ response = thread[:sqs_connection].request(request) }
-        # check response for errors...
-      @last_response = response
-      if response.is_a?(Net::HTTPSuccess)
-        @error_handler = nil
-        @@bench_xml.add! { parser.parse(response) }
-        return parser.result
-      else
-        @error_handler = AWSErrorHandler.new(self, parser, @@amazon_problems) unless @error_handler
-        check_result   = @error_handler.check(request)
-        if check_result
-          @error_handler = nil
-          return check_result 
-        end
-        raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
-      end
-    rescue
-      @error_handler = nil
-      raise
+      request_info_impl(thread[:sqs_connection], @@bench, request, parser)
     end
 
 

data/test/http_connection.rb

@@ -0,0 +1,85 @@
+=begin
+Copyright (c) 2007 RightScale, Inc. 
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+=end
+
+# Stub extension/redefinition of RightHttpConnection for testing purposes.
+require 'net/http'
+require 'right_http_connection'
+
+module Net
+  class HTTPResponse
+  alias_method :real_body, :body
+    def setmsg(msg)
+      @mymsg = msg
+    end
+
+    def body
+      @mymsg ? @mymsg : real_body
+    end
+  end
+end
+
+module Rightscale
+
+  class HttpConnection
+    @@response_stack = []
+
+    alias_method :real_request, :request
+
+    def request(request_params, &block)
+      if(@@response_stack.length == 0)
+        return real_request(request_params, &block)
+      end
+
+      if(block)
+        # Do something special
+      else
+        next_response = HttpConnection::pop() 
+        classname = Net::HTTPResponse::CODE_TO_OBJ["#{next_response[:code]}"]
+        response = classname.new("1.1", next_response[:code], next_response[:msg])
+        if(next_response[:msg])
+          response.setmsg(next_response[:msg])
+        end
+        response
+      end
+    end
+
+    def self.reset
+      @@response_stack = []
+    end
+
+    def self.push(code, msg=nil)
+      response = {:code => code, :msg => msg}
+      @@response_stack << response
+    end
+
+    def self.pop
+      @@response_stack.pop
+    end
+
+    def self.length
+      @@response_stack.length
+    end
+
+  end
+
+end

data/test/s3/test_right_s3.rb

@@ -25,7 +25,7 @@ class TestS3 < Test::Unit::TestCase
   end
 
   def test_03_list_empty_bucket
-    assert_equal 0, @s3.list_bucket(@bucket).size, "#{@bucket} must exist!"
+    assert_equal 0, @s3.list_bucket(@bucket).size, "#{@bucket} isn't empty, arrgh!"
   end
   
   def test_04_put
@@ -44,12 +44,31 @@ class TestS3 < Test::Unit::TestCase
   def test_06_head
     assert_equal 'Woohoo1!', @s3.head(@bucket,@key1)['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo1!'"
   end
+
+
+  def test_07_streaming_get
+    resp = String.new
+    assert_raise(Rightscale::AwsError) do 
+      @s3.get(@bucket, 'undefined/key') do |chunk|
+        resp += chunk
+      end
+    end
+
+    resp = String.new
+    data1 = @s3.get(@bucket, @key1) do |chunk|
+      resp += chunk
+    end
+    assert_equal RIGHT_OBJECT_TEXT, resp, "Object text must be equal to '#{RIGHT_OBJECT_TEXT}'"
+    assert_equal @s3.get_object(@bucket, @key1), resp, "Streaming iface must return same as non-streaming"
+    assert_equal 'Woohoo1!', data1[:headers]['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo1!'"
+  end
   
-  def test_07_delete_folder
+  def test_08_delete_folder
     assert_equal 1, @s3.delete_folder(@bucket, 'test').size, "Only one key(#{@key1}) must be deleted!"
   end
 
-  def test_08_delete_bucket
+
+  def test_09_delete_bucket
     assert_raise(Rightscale::AwsError) { @s3.delete_bucket(@bucket) }
     assert @s3.clear_bucket(@bucket), 'Clear_bucket fail'
     assert_equal 0, @s3.list_bucket(@bucket).size, 'Bucket must be empty'
@@ -57,6 +76,7 @@ class TestS3 < Test::Unit::TestCase
     assert !@s3.list_all_my_buckets.map{|bucket| bucket[:name]}.include?(@bucket), "#{@bucket} must not exist"
   end
 
+
   #---------------------------
   # Rightscale::S3 classes
   #---------------------------
@@ -212,6 +232,16 @@ class TestS3 < Test::Unit::TestCase
     assert key1.delete
     assert !key1.exists?
   end
-  
+
+  def test_36_set_amazon_problems
+    original_problems = RightAws::S3Interface.amazon_problems
+    assert(original_problems.length > 0)
+    RightAws::S3Interface.amazon_problems= original_problems << "A New Problem"
+    new_problems = RightAws::S3Interface.amazon_problems
+    assert_equal(new_problems, original_problems)
+
+    RightAws::S3Interface.amazon_problems= nil
+    assert_nil(RightAws::S3Interface.amazon_problems)
+  end
 
 end

data/test/s3/test_right_s3_stubbed.rb

@@ -0,0 +1,70 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestS3Stubbed < Test::Unit::TestCase
+
+  RIGHT_OBJECT_TEXT     = 'Right test message'
+  
+  def setup
+    @s3     = Rightscale::S3Interface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key)
+    @bucket = 'right_s3_awesome_test_bucket'
+    @key1   = 'test/woohoo1'
+    @key2   = 'test1/key/woohoo2'
+    @s      = Rightscale::S3.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key)
+    Rightscale::HttpConnection.reset
+  end
+
+  # Non-remote tests: these use the stub version of Rightscale::HTTPConnection
+  def test_101_create_bucket
+    Rightscale::HttpConnection.push(409, 'The named bucket you tried to create already exists')
+    Rightscale::HttpConnection.push(500, 'We encountered an internal error.  Please try again.')
+    Rightscale::HttpConnection.push(500, 'We encountered an internal error.  Please try again.')
+    assert_raise RightAws::AwsError do
+      @s3.create_bucket(@bucket)
+    end
+  end
+
+  def test_102_list_all_my_buckets_failure
+    Rightscale::HttpConnection.push(401, 'Unauthorized') 
+    assert_raise RightAws::AwsError do
+      @s3.list_all_my_buckets
+    end
+  end
+
+  def test_103_list_empty_bucket
+    Rightscale::HttpConnection.push(403, 'Access Denied') 
+    assert_raise RightAws::AwsError do
+      @s3.list_bucket(@bucket)
+    end
+  end
+  
+  def test_104_put
+    Rightscale::HttpConnection.push(400, 'Your proposed upload exceeds the maximum allowed object size.') 
+    Rightscale::HttpConnection.push(400, 'The Content-MD5 you specified was an invalid.') 
+    Rightscale::HttpConnection.push(409, 'Please try again') 
+    assert_raise RightAws::AwsError do
+      assert @s3.put(@bucket, @key1, RIGHT_OBJECT_TEXT, 'x-amz-meta-family'=>'Woohoo1!'), 'Put bucket fail'
+    end
+    assert_raise RightAws::AwsError do
+      assert @s3.put(@bucket, @key2, RIGHT_OBJECT_TEXT, 'x-amz-meta-family'=>'Woohoo2!'), 'Put bucket fail'
+    end
+  end
+  
+  def test_105_get_and_get_object
+    Rightscale::HttpConnection.push(404, 'not found') 
+    assert_raise(Rightscale::AwsError) { @s3.get(@bucket, 'undefined/key') }
+  end
+  
+  def test_106_head
+    Rightscale::HttpConnection.push(404, 'Good Luck!') 
+    assert_raise RightAws::AwsError do
+      @s3.head(@bucket,@key1)
+    end
+  end
+
+
+  def test_109_delete_bucket
+    Rightscale::HttpConnection.push(403, 'Good Luck!') 
+    assert_raise(Rightscale::AwsError) { @s3.delete_bucket(@bucket) }
+  end
+
+end

data/test/sqs/test_right_sqs.rb

@@ -218,4 +218,21 @@ class TestSqs < Test::Unit::TestCase
     assert queue.delete(true)
   end
 
+  def test_27_set_amazon_problems
+    original_problems = RightAws::SqsInterface.amazon_problems
+    assert(original_problems.length > 0)
+    RightAws::SqsInterface.amazon_problems= original_problems << "A New Problem"
+    new_problems = RightAws::SqsInterface.amazon_problems
+    assert_equal(new_problems, original_problems)
+
+    RightAws::SqsInterface.amazon_problems= nil
+    assert_nil(RightAws::SqsInterface.amazon_problems)
+  end
+
+  def test_28_check_threading_model
+    assert(!@sqs.multi_thread)
+    newsqs = Rightscale::SqsInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, {:multi_thread => true})
+    assert(newsqs.multi_thread)
+  end
+
 end

data/test/ts_right_aws.rb

@@ -3,8 +3,10 @@ $: << File.dirname(__FILE__)
 require 'test_credentials'
 TestCredentials.get_credentials
 
+require 'http_connection'
 require 'awsbase/test_right_awsbase.rb'
 require 'ec2/test_right_ec2.rb'
 require 's3/test_right_s3.rb'
+require 's3/test_right_s3_stubbed.rb'
 require 'sqs/test_right_sqs.rb'
 

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: right_aws
 version: !ruby/object:Gem::Version 
-  version: 1.2.0
-date: 2007-09-12 00:00:00 -07:00
+  version: 1.3.0
+date: 2007-09-26 00:00:00 -07:00
 summary: Interface classes for the Amazon EC2, SQS, and S3 Web Services
 require_paths: 
 - lib
@@ -46,10 +46,12 @@ files:
 - test/ec2/test_right_ec2.rb
 - test/s3/test_helper.rb
 - test/s3/test_right_s3.rb
+- test/s3/test_right_s3_stubbed.rb
 - test/sqs/test_helper.rb
 - test/sqs/test_right_sqs.rb
 - test/ts_right_aws.rb
 - test/test_credentials.rb
+- test/http_connection.rb
 test_files: 
 - test/ts_right_aws.rb
 rdoc_options: 
@@ -82,5 +84,5 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.2.1
+        version: 1.3.0
     version: