right_agent 2.0.8 → 2.1.0

data/lib/right_agent/exceptions.rb

@@ -48,32 +48,16 @@ module RightScale
     class Terminating < RuntimeError; end
 
     # Not authorized to make request
-    class Unauthorized < NestedException
-      def initialize(message, nested_exception = nil)
-        super(message, nested_exception)
-      end
-    end
+    class Unauthorized < NestedException; end
 
     # Cannot connect to service, lost connection to it, or it is out of service or too busy to respond
-    class ConnectivityFailure < NestedException
-      def initialize(message, nested_exception = nil)
-        super(message, nested_exception)
-      end
-    end
+    class ConnectivityFailure < NestedException; end
 
     # Request failed but potentially will succeed if retried
-    class RetryableError < NestedException
-      def initialize(message, nested_exception = nil)
-        super(message, nested_exception)
-      end
-    end
+    class RetryableError < NestedException; end
 
     # Database query failed
-    class QueryFailure < NestedException
-      def initialize(message, nested_exception = nil)
-        super(message, nested_exception)
-      end
-    end
+    class QueryFailure < NestedException; end
 
     # Error internal to specified server
     class InternalServerError < NestedException

data/lib/right_agent/http_exceptions.rb

@@ -0,0 +1,87 @@
+require 'restclient'
+
+module RightScale
+
+  # This code is largely borrowed from RestClient
+
+  # Container for response so that access response headers in same fashion as RestClient
+  class Response
+    attr_reader :headers
+
+    def initialize(headers)
+      @headers = headers
+    end
+  end
+
+  # Base HTTP exception class
+  class HttpException < RuntimeError
+    attr_writer :message
+    attr_reader :http_code, :http_body
+    attr_accessor :response
+
+    def initialize(code, body, response = nil)
+      @http_code = code
+      @http_body = body
+      @response = response
+    end
+
+    def inspect
+      "#{message}: #{http_body}"
+    end
+
+    def to_s
+      inspect
+    end
+
+    def message
+      @message || self.class.name
+    end
+
+  end
+
+  # Exceptions created for each status code defined in RestClient::STATUSES
+  # e.g., RightScale::HttpExceptions::ResourceNotFound for status code 404
+  module HttpExceptions
+
+    # Exception when request failed with an error code that is not managed here
+    class RequestFailed < HttpException
+
+      def to_s
+        message
+      end
+
+      def message
+        "HTTP status code #{http_code}"
+      end
+    end
+
+    # Map HTTP status codes to the corresponding exception class
+    HTTP_EXCEPTIONS_MAP = {}
+
+    # Create exception for given code
+    def self.create(code, body = "", headers = {})
+      if HttpExceptions::HTTP_EXCEPTIONS_MAP[code]
+        HttpExceptions::HTTP_EXCEPTIONS_MAP[code].new(code, body, RightScale::Response.new(headers))
+      else
+        RequestFailed.new(code, body, RightScale::Response.new(headers))
+      end
+    end
+
+    # Convert RestClient exception
+    def self.convert(e)
+      e2 = create(e.http_code, e.http_body, RightScale::Response.new((e.response && e.response.headers) || {}))
+      e2.message = e.message
+      e2
+    end
+
+    RestClient::STATUSES.each do |code, message|
+      klass = Class.new(HttpException) do
+        send(:define_method, :message) {"#{http_code ? "#{http_code} " : ''}#{message}"}
+      end
+      klass_constant = const_set(message.delete(' \-\''), klass)
+      HTTP_EXCEPTIONS_MAP[code] = klass_constant
+    end
+
+  end # HttpExceptions
+
+end # RightScale

data/lib/right_agent/minimal.rb

@@ -44,3 +44,4 @@ require ::File.normalize_path('command', RIGHT_AGENT_BASE_DIR)
 require ::File.normalize_path('log', RIGHT_AGENT_BASE_DIR)
 require ::File.normalize_path('pid_file', RIGHT_AGENT_BASE_DIR)
 require ::File.normalize_path('serialize/serializable', RIGHT_AGENT_BASE_DIR)
+require ::File.normalize_path('eventmachine_spawn', RIGHT_AGENT_BASE_DIR)

data/lib/right_agent/retryable_request.rb

@@ -88,7 +88,7 @@ module RightScale
     def initialize(operation, payload, options = {})
       raise ArgumentError.new("operation is required") unless (@operation = operation)
       raise ArgumentError.new("payload is required") unless (@payload = payload)
-      @retry_on_error = options[:retry_on_error] || false
+      @retry_on_error = options[:retry_on_error]
       @timeout = options[:timeout] || DEFAULT_TIMEOUT
       @retry_delay = options[:retry_delay] || DEFAULT_RETRY_DELAY
       @retry_delay_count = options[:retry_delay_count] || DEFAULT_RETRY_DELAY_COUNT

data/lib/right_agent/scripts/agent_controller.rb

@@ -287,7 +287,7 @@ module RightScale
     def dispatch(action, agent_name)
       # Setup the environment from config if necessary
       begin
-        eval("#{action}_agent(agent_name)")
+        send("#{action}_agent", agent_name)
       rescue SystemExit
         true
       rescue SignalException
@@ -332,6 +332,12 @@ module RightScale
     def start_agent(agent_name, agent_class = Agent)
       puts "#{human_readable_name} being started"
 
+      if @options[:fiber_pool_size]
+        require 'fiber_pool'
+        @options[:fiber_pool] = FiberPool.new(@options[:fiber_pool_size])
+        EM_S.fiber_pool = @options[:fiber_pool]
+      end
+
       EM.error_handler do |e|
         Log.error("EM block execution failed with exception", e, :trace)
         Log.error("\n\n===== Exiting due to EM block exception =====\n\n")
@@ -339,7 +345,7 @@ module RightScale
         exit(1)
       end
 
-      EM.run do
+      EM_S.run do
         begin
           @@agent = agent_class.start(@options)
         rescue SystemExit
@@ -364,9 +370,9 @@ module RightScale
     # (Boolean):: true if process was stopped, otherwise false
     def stop_agent(agent_name)
       res = false
-      if pid_file = AgentConfig.pid_file(agent_name)
+      if (pid_file = AgentConfig.pid_file(agent_name))
         name = human_readable_name(agent_name, pid_file.identity)
-        if pid = pid_file.read_pid[:pid]
+        if (pid = pid_file.read_pid[:pid])
           begin
             Process.kill('TERM', pid)
             res = true
@@ -405,7 +411,7 @@ module RightScale
         else
           puts "#{name} is not running but has a stale pid file at #{pid_file}"
         end
-      elsif identity = AgentConfig.agent_options(agent_name)[:identity]
+      elsif (identity = AgentConfig.agent_options(agent_name)[:identity])
         puts "#{human_readable_name(agent_name, identity)} is not running"
       end
       res
@@ -469,7 +475,7 @@ module RightScale
     def configure_agent(action, options)
       agent_type = options[:agent_type]
       agent_name = options[:agent_name]
-      if agent_name != agent_type && cfg = AgentConfig.load_cfg(agent_type)
+      if agent_name != agent_type && (cfg = AgentConfig.load_cfg(agent_type))
         base_id = (options[:base_id] || AgentIdentity.parse(cfg[:identity]).base_id.to_s).to_i
         unless (identity = AgentConfig.agent_options(agent_name)[:identity]) &&
                AgentIdentity.parse(identity).base_id == base_id

data/lib/right_agent/scripts/agent_deployer.rb

@@ -44,6 +44,7 @@
 #      --reconnect-interval SEC Set number of seconds between HTTP or AMQP reconnect attempts
 #      --grace-timeout SEC      Set number of seconds before graceful termination times out
 #      --[no-]dup-check         Set whether to check for and reject duplicate requests, .e.g., due to retries
+#      --fiber-pool-size, -f N  Set size of fiber pool
 #      --options, -o KEY=VAL    Set options that act as final override for any persisted configuration settings
 #      --monit                  Generate monit configuration file
 #      --test                   Build test deployment using default test settings
@@ -176,6 +177,10 @@ module RightScale
           options[:dup_check] = b
         end
 
+        opts.on('-f', '--fiber-pool-size N') do |n|
+          options[:fiber_pool_size] = n.to_i
+        end
+
         opts.on('--prefetch COUNT') do |count|
           options[:prefetch] = count.to_i
         end
@@ -312,6 +317,7 @@ module RightScale
       cfg[:dup_check]          = options[:dup_check].nil? ? true : options[:dup_check]
       cfg[:http_proxy]         = options[:http_proxy] if options[:http_proxy]
       cfg[:http_no_proxy]      = options[:http_no_proxy] if options[:http_no_proxy]
+      cfg[:fiber_pool_size]    = options[:fiber_pool_size] if options[:fiber_pool_size]
       cfg
     end
 

data/lib/right_agent/sender.rb

@@ -571,7 +571,9 @@ module RightScale
       true
     end
 
-    # Send request via HTTP and then immediately handle response
+    # Send request via HTTP
+    # Use next_tick for asynchronous response and to ensure
+    # that the request is sent using the main EM reactor thread
     #
     # === Parameters
     # kind(Symbol):: Kind of request: :send_push or :send_request
@@ -583,6 +585,33 @@ module RightScale
     # === Return
     # true:: Always return true
     def http_send(kind, target, packet, received_at, callback)
+      if @options[:async_response]
+        EM_S.next_tick do
+          begin
+            http_send_once(kind, target, packet, received_at, callback)
+          rescue Exception => e
+            Log.error("Failed sending or handling response for #{packet.trace} #{packet.type}", e, :trace)
+            @exception_stats.track("request", e)
+          end
+        end
+      else
+        http_send_once(kind, target, packet, received_at, callback)
+      end
+      true
+    end
+
+    # Send request via HTTP and then immediately handle response
+    #
+    # === Parameters
+    # kind(Symbol):: Kind of request: :send_push or :send_request
+    # target(Hash|String|nil):: Target for request
+    # packet(Push|Request):: Request packet to send
+    # received_at(Time):: Time when request received
+    # callback(Proc|nil):: Block used to process response
+    #
+    # === Return
+    # true:: Always return true
+    def http_send_once(kind, target, packet, received_at, callback)
       begin
         method = packet.class.name.split("::").last.downcase
         result = success_result(@agent.client.send(method, packet.type, packet.payload, target, packet.token))
@@ -619,11 +648,7 @@ module RightScale
         result = Result.new(packet.token, @identity, result, from = packet.target)
         result.received_at = received_at.to_f
         @pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
-        if @options[:async_response]
-          EM.next_tick { handle_response(result) }
-        else
-          handle_response(result)
-        end
+        handle_response(result)
       end
       true
     end

data/right_agent.gemspec

@@ -25,8 +25,8 @@ require 'rbconfig'
 
 Gem::Specification.new do |spec|
   spec.name      = 'right_agent'
-  spec.version   = '2.0.8'
-  spec.date      = '2014-03-03'
+  spec.version   = '2.1.0'
+  spec.date      = '2014-04-01'
   spec.authors   = ['Lee Kirchhoff', 'Raphael Simon', 'Tony Spataro', 'Scott Messier']
   spec.email     = 'lee@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_agent'

data/spec/agent_spec.rb

@@ -42,7 +42,6 @@ describe RightScale::Agent do
       flexmock(EM).should_receive(:next_tick).and_yield
       flexmock(EM).should_receive(:add_timer).and_yield
       @timer = flexmock("timer")
-      flexmock(EM::Timer).should_receive(:new).and_return(@timer)
       flexmock(EM::PeriodicTimer).should_receive(:new).and_return(@timer)
       @timer.should_receive(:cancel)
       @broker = flexmock("broker", :subscribe => ["b1"], :publish => ["b1"], :prefetch => true,
@@ -249,11 +248,9 @@ describe RightScale::Agent do
       @log = flexmock(RightScale::Log)
       @log.should_receive(:error).by_default.and_return { |m| raise RightScale::Log.format(*m) }
       flexmock(EM).should_receive(:next_tick).and_yield
-      flexmock(EM).should_receive(:add_timer).and_yield
-      @timer = flexmock("timer")
-      flexmock(EM::Timer).should_receive(:new).and_return(@timer).by_default
-      @timer.should_receive(:cancel).by_default
-      @periodic_timer = flexmock("timer")
+      flexmock(EM).should_receive(:add_timer).with(0, Proc).and_yield.by_default
+      flexmock(EM).should_receive(:add_timer).with(1, Proc).and_yield.by_default
+      @periodic_timer = flexmock("periodic timer")
       flexmock(EM::PeriodicTimer).should_receive(:new).and_return(@periodic_timer)
       @periodic_timer.should_receive(:cancel).by_default
       @broker_id = "rs-broker-123-1"
@@ -521,6 +518,12 @@ describe RightScale::Agent do
 
     describe "Terminating" do
 
+      before(:each) do
+        @timer = flexmock("timer")
+        flexmock(EM::Timer).should_receive(:new).and_return(@timer).by_default
+        @timer.should_receive(:cancel).by_default
+      end
+
       it "should log error for abnormal termination" do
         run_in_em do
           @agent = RightScale::Agent.new(:user => "me", :identity => @identity)

data/spec/clients/balanced_http_client_spec.rb

@@ -32,29 +32,37 @@ describe RightScale::BalancedHttpClient do
     @log = flexmock(RightScale::Log)
     @log.should_receive(:error).by_default.and_return { |m| raise RightScale::Log.format(*m) }
     @log.should_receive(:warning).by_default.and_return { |m| raise RightScale::Log.format(*m) }
+    @options = {}
     @url = "http://my.com"
     @urls = [@url]
+    @host = @url
     @path = "/foo/bar"
     @balancer = flexmock("balancer")
+    flexmock(RightSupport::Net::RequestBalancer).should_receive(:new).and_return(@balancer).by_default
+    @later = (@now = Time.now)
+    @tick = 0.01
+    flexmock(Time).should_receive(:now).and_return { @later += @tick }
   end
 
   context :initialize do
-    ['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy', 'ALL_PROXY'].each do |proxy|
-      it "initializes use of proxy if #{proxy} defined in environment" do
-        ENV[proxy] = "https://my.proxy.com"
-        flexmock(RightSupport::Net::RequestBalancer).should_receive(:new).and_return(@balancer)
-        flexmock(RestClient).should_receive(:proxy=).with("https://my.proxy.com").once
-        RightScale::BalancedHttpClient.new(@urls)
-        ENV.delete(proxy)
-      end
+    it "initializes parameter filter list" do
+      client = RightScale::BalancedHttpClient.new(@urls)
+      client.instance_variable_get(:@filter_params).should == []
+    end
 
-      it "prepends scheme to proxy address if #{proxy} defined in environment" do
-        ENV[proxy] = "1.2.3.4"
-        flexmock(RightSupport::Net::RequestBalancer).should_receive(:new).and_return(@balancer)
-        flexmock(RestClient).should_receive(:proxy=).with("http://1.2.3.4").once
-        RightScale::BalancedHttpClient.new(@urls)
-        ENV.delete(proxy)
-      end
+    it "stores specified parameter filters" do
+      client = RightScale::BalancedHttpClient.new(@urls, :filter_params => [:secret])
+      client.instance_variable_get(:@filter_params).should == ["secret"]
+    end
+
+    it "creates blocking HTTP client" do
+      client = RightScale::BalancedHttpClient.new(@urls)
+      client.instance_variable_get(:@http_client).should be_a RightScale::BlockingClient
+    end
+
+    it "creates non-blocking HTTP client if specified" do
+      client = RightScale::BalancedHttpClient.new(@urls, :non_blocking => true)
+      client.instance_variable_get(:@http_client).should be_a RightScale::NonBlockingClient
     end
 
     it "creates request balancer with health checker" do
@@ -73,24 +81,28 @@ describe RightScale::BalancedHttpClient do
   context :check_health do
     before(:each) do
       @urls = ["http://my0.com", @url]
-      @http_client = flexmock("http client")
-      flexmock(RightSupport::Net::HTTPClient).should_receive(:new).and_return(@http_client).by_default
       @client = RightScale::BalancedHttpClient.new(@urls)
+      @health_check_proc = @client.instance_variable_get(:@http_client).health_check_proc
     end
 
     it "calls health check proc using first URL" do
-      @http_client.should_receive(:get).with("http://my0.com/health-check", Hash).once
+      flexmock(@health_check_proc).should_receive(:call).with("http://my0.com").once
       @client.check_health
     end
 
     it "calls health check proc using specified URL" do
-      @http_client.should_receive(:get).with("http://my1.com/health-check", Hash).once
-      @client.check_health("http://my1.com")
+      flexmock(@health_check_proc).should_receive(:call).with("http://my1.com/api").once
+      @client.check_health("http://my1.com/api")
+    end
+
+    it "returns result of health check" do
+      flexmock(@health_check_proc).should_receive(:call).with("http://my0.com").and_return("ok").once
+      @client.check_health.should == "ok"
     end
 
     RightScale::BalancedHttpClient::RETRY_STATUS_CODES.each do |code|
       it "raises NotResponding for #{code}" do
-        @http_client.should_receive(:get).and_raise(RestExceptionMock.new(code))
+        flexmock(@health_check_proc).should_receive(:call).and_raise(RightScale::HttpExceptions.create(code))
         lambda { @client.check_health("http://my1.com") }.should raise_error(RightScale::BalancedHttpClient::NotResponding)
       end
     end
@@ -98,248 +110,244 @@ describe RightScale::BalancedHttpClient do
 
   context :request do
     before(:each) do
-      @result = {:out => 123}
-      @json_result = JSON.dump(@result)
-      @json_decoded_result = {"out" => 123}
-      @response = flexmock("response")
-      @response.should_receive(:code).and_return(200).by_default
-      @response.should_receive(:body).and_return(@json_result).by_default
-      @response.should_receive(:headers).and_return({:status => "200 OK"}).by_default
+      @params = {}
+      @result = {"out" => 123}
+      @body = JSON.dump({:out => 123})
+      @headers = {:status => "200 OK"}
+      @response = [@result, 200, @body, @headers]
+      @balancer.should_receive(:request).and_yield(@url).and_return(@response).by_default
       flexmock(RightSupport::Data::UUID).should_receive(:generate).and_return("random uuid").by_default
-      @balancer.should_receive(:request).and_yield(@url).by_default
-      flexmock(RightSupport::Net::RequestBalancer).should_receive(:new).and_return(@balancer).by_default
-      @http_client = flexmock("http client")
-      flexmock(RightSupport::Net::HTTPClient).should_receive(:new).and_return(@http_client).by_default
-      @client = RightScale::BalancedHttpClient.new(@urls)
+      @client = RightScale::BalancedHttpClient.new(@urls, :filter_params => [:secret])
+      @http_client = @client.instance_variable_get(:@http_client)
+      flexmock(@http_client).should_receive(:request).and_return(@response).by_default
     end
 
-    context "with no options" do
-      before(:each) do
-        @options = {}
-        @http_client.should_receive(:get).with("#{@url}#{@path}", on { |a| @options = a }).and_return(nil)
-        @client.send(:request, :get, @path).should be_nil
-      end
-
-      it "sets default open and request timeouts" do
-        @options[:open_timeout].should == RightScale::BalancedHttpClient::DEFAULT_OPEN_TIMEOUT
-        @options[:timeout].should == RightScale::BalancedHttpClient::DEFAULT_REQUEST_TIMEOUT
-      end
-
-      it "sets random request uuid in header" do
-        @options[:headers]["X-Request-Lineage-Uuid"] == "random uuid"
-      end
+    it "uses specified request UUID" do
+      @log.should_receive(:info).with("Requesting POST <my uuid> /foo/bar").once
+      @log.should_receive(:info).with("Completed <my uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes")
+      @client.request(:post, @path, @params, :request_uuid => "my uuid")
+    end
 
-      it "sets response to be JSON-encoded" do
-        @options[:headers][:accept] == "application/json"
-      end
+    it "generates request UUID if none specified" do
+      @log.should_receive(:info).with("Requesting POST <random uuid> /foo/bar").once
+      @log.should_receive(:info).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes").once
+      @client.request(:post, @path)
+    end
 
-      it "does not set API version" do
-        @options[:headers]["X-API-Version"].should be_nil
-      end
+    it "logs request before sending it" do
+      @log.should_receive(:info).with("Requesting POST <random uuid> /foo/bar").once
+      flexmock(@http_client).should_receive(:request).and_raise(RuntimeError).once
+      @client.request(:post, @path) rescue nil
     end
 
-    context "with options" do
-      before(:each) do
-        @options = {}
-        create_options = {
-          :request_uuid => "my uuid",
-          :api_version => "1.0" }
-        request_options = {
-          :open_timeout => 1,
-          :request_timeout => 2,
-          :headers => {"Authorization" => "Bearer <session>"} }
-        @client = RightScale::BalancedHttpClient.new(@urls, create_options)
-        @http_client.should_receive(:get).with("#{@url}#{@path}", on { |a| @options = a }).and_return(nil)
-        @client.send(:request, :get, @path, nil, request_options).should be_nil
-      end
+    it "logs using specified log level" do
+      @params = {:some => "data"}
+      @log.should_receive(:debug).with("Requesting POST <random uuid> /foo/bar {:some=>\"data\"}").once
+      @log.should_receive(:debug).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes | {\"out\"=>123}").once
+      @client.request(:post, @path, @params, :log_level => :debug)
+    end
 
-      it "sets open and request timeouts" do
-        @options[:open_timeout].should == 1
-        @options[:timeout].should == 2
-      end
+    it "logs using configured log level if none specified" do
+      @params = {:some => "data"}
+      @log.should_receive(:level).and_return(:debug)
+      @log.should_receive(:debug).with("Requesting POST <random uuid> /foo/bar {:some=>\"data\"}").once
+      @log.should_receive(:debug).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes | {\"out\"=>123}").once
+      @client.request(:post, @path, @params)
+    end
 
-      it "sets request uuid in header" do
-        @options[:headers]["X-Request-Lineage-Uuid"] == "my uuid"
-      end
+    it "appends specified filter parameters to list" do
+      @params = {:some => "data", :secret => "data", :other => "data"}
+      @log.should_receive(:debug).with("Requesting POST <random uuid> /foo/bar {:some=>\"data\", :secret=>\"<hidden>\", :other=>\"<hidden>\"}").once
+      @log.should_receive(:debug).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes | {\"out\"=>123}").once
+      @client.request(:post, @path, @params, :log_level => :debug, :filter_params => [:other])
+    end
 
-      it "sets API version in header" do
-        @options[:headers]["X-API-Version"].should == "1.0"
-      end
+    it "removes user and password from host when logging" do
+      flexmock(@client).should_receive(:log_success).with(@result, 200, @body, @headers, "http://my.com", @path, "random uuid", Time, :info).once
+      @balancer.should_receive(:request).and_yield("http://111:secret@my.com").and_return(@response).once
+      @client.request(:post, @path)
+    end
 
-      it "uses headers for setting authorization in header" do
-        @options[:headers]["Authorization"].should == "Bearer <session>"
-      end
+    it "logs successful completion of request" do
+      @log.should_receive(:info).with("Requesting POST <random uuid> /foo/bar")
+      @log.should_receive(:info).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes").once
+      @client.request(:post, @path, @params, @options)
     end
 
-    [:get, :delete].each do |verb|
-      context "with #{verb.inspect}" do
-        it "uses form-encoded query option for parameters" do
-          params = {:id => 10}
-          @http_client.should_receive(verb).with("#{@url}#{@path}", on { |a| a[:query] == params &&
-              a[:payload].nil? }).and_return(nil)
-          @client.send(:request, verb, @path, params).should be_nil
-        end
-      end
+    it "returns result of request" do
+      @client.request(:post, @path, @params, @options).should == @result
     end
 
-    [:post, :put].each do |verb|
-      context "with #{verb.inspect}" do
-        it "uses JSON-encoded payload options for parameters" do
-          payload = {:pay => "load"}
-          json_payload = JSON.dump(payload)
-          @http_client.should_receive(verb).with("#{@url}#{@path}", on { |a| a[:payload] == json_payload &&
-              a[:query].nil? }).and_return(nil).once
-          @client.send(:request, verb, @path, payload).should be_nil
-        end
-      end
+    it "handles poll requests separately" do
+      flexmock(@client).should_receive(:poll_request).and_return(@response).once
+      flexmock(@client).should_receive(:rest_request).and_return(@response).never
+      @client.request(:poll, @path, @params, @options).should == @result
     end
 
-    context "health check proc" do
-      it "removes user and password from URL when checking health" do
-        @url = "http://me:pass@my.com"
-        @client = RightScale::BalancedHttpClient.new(@url, :health_check_path => "/health-check")
-        @http_client.should_receive(:get).with("http://my.com/health-check", Hash).once
-        @client.instance_variable_get(:@health_check_proc).call(@url)
-      end
+    it "handles NoResult response from balancer" do
+      no_result = RightSupport::Net::NoResult.new("no result", {})
+      @log.should_receive(:error).with(/Failed <random uuid>.*#{@path}/).once
+      @balancer.should_receive(:request).and_yield(@url).once
+      flexmock(@http_client).should_receive(:request).and_raise(no_result).once
+      flexmock(@client).should_receive(:handle_no_result).with(RightSupport::Net::NoResult, @url, Proc).and_yield(no_result).once
+      @client.request(:get, @path)
+    end
 
-      it "uses default path if none specified" do
-        @client = RightScale::BalancedHttpClient.new(@url)
-        @http_client.should_receive(:get).with("http://my.com/health-check", Hash).once
-        @client.instance_variable_get(:@health_check_proc).call(@url)
-      end
+    it "converts RestClient exception to HttpException and raises it" do
+      bad_request = RestClient::Exceptions::EXCEPTIONS_MAP[400].new(nil, 400)
+      @log.should_receive(:error).with(/Failed <random uuid>.*#{@path}/).once
+      flexmock(@http_client).should_receive(:request).and_raise(bad_request).once
+      lambda { @client.request(:get, @path) }.should raise_error(RightScale::HttpExceptions::BadRequest)
+    end
 
-      it "appends health check path to any existing path" do
-        @url = "http://my.com/foo"
-        @client = RightScale::BalancedHttpClient.new(@url)
-        @http_client.should_receive(:get).with("http://my.com/foo/health-check", Hash).once
-        @client.instance_variable_get(:@health_check_proc).call(@url)
-      end
+    it "logs and re-raises unexpected exception" do
+      @log.should_receive(:error).with(/Failed <random uuid>.*#{@path}/).once
+      flexmock(@http_client).should_receive(:request).and_raise(RuntimeError).once
+      lambda { @client.request(:get, @path) }.should raise_error(RuntimeError)
+    end
 
-      it "uses fixed timeout values" do
-        @client = RightScale::BalancedHttpClient.new(@url, :open_timeout => 5, :request_timeout => 30)
-        @http_client.should_receive(:get).with(String, {:open_timeout => 2, :timeout => 5}).once
-        @client.instance_variable_get(:@health_check_proc).call(@url)
-      end
+    it "passes back host used with user and password removed" do
+      flexmock(@client).should_receive(:log_success).with(@result, 200, @body, @headers, "http://my.com", @path, "random uuid", Time, :info).once
+      @balancer.should_receive(:request).and_yield("http://111:secret@my.com").and_return(@response).once
+      @client.request(:post, @path)
+    end
+  end
 
-      it "sets API version if specified" do
-        @client = RightScale::BalancedHttpClient.new(@url, :api_version => "2.0")
-        @http_client.should_receive(:get).with(String, hsh(:headers => {"X-API-Version" => "2.0"})).once
-        @client.instance_variable_get(:@health_check_proc).call(@url)
-      end
+  context :request_headers do
+    before(:each) do
+      @request_uuid = "my uuid"
+      @client = RightScale::BalancedHttpClient.new(@urls)
     end
 
-    it "sends request using HTTPClient via balancer" do
-      @http_client.should_receive(:post).with("#{@url}#{@path}", Hash).and_return(nil).once
-      @balancer.should_receive(:request).and_yield(@url).once
-      @client.send(:request, :post, @path).should be_nil
+    it "sets request uuid in header" do
+      headers = @client.send(:request_headers, @request_uuid, @options)
+      headers["X-Request-Lineage-Uuid"].should == "my uuid"
     end
 
-    it "returns location header for 201 response" do
-      @response.should_receive(:code).and_return(201).once
-      @response.should_receive(:body).and_return("").once
-      @response.should_receive(:headers).and_return({:status => "201 Created", :location => "/href"}).once
-      @balancer.should_receive(:request).and_return(@response).once
-      @client.send(:request, :get, @path).should == "/href"
+    it "sets response to be JSON-encoded" do
+      headers = @client.send(:request_headers, @request_uuid, @options)
+      headers[:accept] == "application/json"
     end
 
-    it "returns JSON decoded response" do
-      @response.should_receive(:body).and_return(@json_result).once
-      @balancer.should_receive(:request).and_return(@response).once
-      @client.send(:request, :get, @path).should == @json_decoded_result
+    it "sets API version" do
+      @client = RightScale::BalancedHttpClient.new(@urls, :api_version => "2.0")
+      headers = @client.send(:request_headers, @request_uuid, @options)
+      headers["X-API-Version"].should == "2.0"
     end
 
-    it "returns nil if response is empty" do
-      @response.should_receive(:body).and_return("").once
-      @balancer.should_receive(:request).and_return(@response).once
-      @client.send(:request, :get, @path).should be_nil
+    it "does not set API version if not defined" do
+      headers = @client.send(:request_headers, @request_uuid, @options)
+      headers.has_key?("X-API-Version").should be false
     end
 
-    it "returns nil if response is nil" do
-      @balancer.should_receive(:request).and_return(nil).once
-      @client.send(:request, :get, @path).should be_nil
+    it "sets any optionally specified headers" do
+      @options = {:headers => {"Authorization" => "Bearer <session>"}}
+      headers = @client.send(:request_headers, @request_uuid, @options)
+      headers["Authorization"].should == "Bearer <session>"
     end
 
-    it "returns nil if response status indicates no content" do
-      @response.should_receive(:code).and_return(204).once
-      @balancer.should_receive(:request).and_return(@response).once
-      @client.send(:request, :get, @path).should be_nil
+    it "sets debug option if in debug mode" do
+      @log.should_receive(:level).and_return(:debug)
+      headers = @client.send(:request_headers, @request_uuid, @options)
+      headers["X-DEBUG"].should be true
     end
+  end
 
-    it "handles NoResult response from balancer" do
-      no_result = RightSupport::Net::NoResult.new("no result", {})
-      @log.should_receive(:error).with(/Failed <random uuid>.*#{@path}/).once
-      @balancer.should_receive(:request).and_yield(@url).once
-      flexmock(RightSupport::Net::HTTPClient).should_receive(:new).and_raise(no_result).once
-      flexmock(@client).should_receive(:handle_no_result).with(RightSupport::Net::NoResult, @url, Proc).and_yield(no_result).once
-      @client.send(:request, :get, @path)
+  context "requesting" do
+    before(:each) do
+      @tick = 1
+      @used = {}
+      @result = {"out" => 123}
+      @body = JSON.dump({:out => 123})
+      @headers = {:status => "200 OK"}
+      @response = [@result, 200, @body, @headers]
+      @balancer.should_receive(:request).and_yield(@url).and_return(@response).by_default
+      @client = RightScale::BalancedHttpClient.new(@urls)
+      @http_client = @client.instance_variable_get(:@http_client)
+      flexmock(@http_client).should_receive(:request).and_return(@response).by_default
+      @connect_options = {}
+      @request_options = {:open_timeout => 2, :request_timeout => 5}
     end
 
-    it "reports and re-raises unexpected exception" do
-      @log.should_receive(:error).with(/Failed <random uuid>.*#{@path}/).once
-      @balancer.should_receive(:request).and_raise(RuntimeError).once
-      lambda { @client.send(:request, :get, @path) }.should raise_error(RuntimeError)
+    context :rest_request do
+      it "make requests using request balancer" do
+        flexmock(@http_client).should_receive(:request).with(:post, @path, @url, Hash, Hash).and_return(@response).once
+        @balancer.should_receive(:request).and_yield(@url).once
+        @client.send(:rest_request, :post, @path, @connect_options, @request_options, @used)
+      end
+
+      it "uses request balancer to make request" do
+        flexmock(@http_client).should_receive(:request).with(:post, @path, @url, Hash, Hash).and_return(@response).once
+        @balancer.should_receive(:request).and_yield(@url).and_return(@response).once
+        @client.send(:rest_request, :post, @path, @connect_options, @request_options, @used)
+      end
+
+      it "passes back host used with user and password removed" do
+        @balancer.should_receive(:request).and_yield("http://111:secret@my.com").and_return(@response).once
+        @client.send(:rest_request, :post, @path, @connect_options, @request_options, @used)
+        @used[:host].should == "http://my.com"
+      end
+
+      it "returns result and response info" do
+        @balancer.should_receive(:request).and_yield(@url).and_return(@response).once
+        @client.send(:rest_request, :post, @path, @connect_options, @request_options, @used).should == @response
+      end
     end
 
-    context "when logging" do
+    context :poll_request do
       before(:each) do
-        now = Time.now
-        flexmock(Time).should_receive(:now).and_return(now, now + 0.01)
+        @started_at = @now
+        @request_timeout = 30
+        @stop_at = @started_at + @request_timeout
+        @connection = {:host => @host, :expires_at => @later + 5}
       end
 
-      it "logs request and response" do
-        @http_client.should_receive(:post).and_return(@response)
-        @log.should_receive(:info).with("Requesting POST <random uuid> /foo/bar").once
-        @log.should_receive(:info).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes").once
-        @client.send(:request, :post, @path)
+      it "makes get request if there is no existing connection" do
+        @http_client.instance_variable_get(:@connections)[@path].should be nil
+        flexmock(@http_client).should_receive(:request).with(:get, @path, @url, Hash, Hash).and_return(@response).once
+        @client.send(:poll_request, @path, @connect_options, @request_options, @request_timeout, @started_at, @used)
       end
 
-      it "logs using specified log level" do
-        @http_client.should_receive(:post).and_return(@response)
-        @log.should_receive(:debug).with("Requesting POST <random uuid> /foo/bar").once
-        @log.should_receive(:debug).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes").once
-        @client.send(:request, :post, @path, {}, :log_level => :debug)
+      it "makes get request if there is connection but it has expired" do
+        @connection = {:host => @host, :expires_at => @now}
+        @http_client.instance_variable_get(:@connections)[@path] = @connection
+        flexmock(@http_client).should_receive(:request).with(:get, @path, @url, Hash, Hash).and_return(@response).once
+        @client.send(:poll_request, @path, @connect_options, @request_options, @request_timeout, @started_at, @used)
       end
 
-      it "logs response length using header :content_length if available" do
-        @response.should_receive(:headers).and_return({:status => "200 OK", :content_length => 99})
-        @http_client.should_receive(:post).and_return(@response)
-        @log.should_receive(:info).with("Requesting POST <random uuid> /foo/bar").once
-        @log.should_receive(:info).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 99 bytes").once
-        @client.send(:request, :post, @path)
+      it "returns after get request if request has timed out" do
+        @response = [nil, 200, nil, @headers]
+        @balancer.should_receive(:request).and_return(@response).once
+        @client.send(:poll_request, @path, @connect_options, @request_options, 1, @started_at, @used).should == @response
       end
 
-      it "omits user and password from logged host" do
-        url = "http://111:secret@my.com"
-        @client = RightScale::BalancedHttpClient.new([url])
-        @log.should_receive(:info).with("Requesting POST <random uuid> /foo/bar").once
-        @log.should_receive(:info).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes").once
-        @http_client.should_receive(:post).with("#{url}#{@path}", Hash).and_return(@response).once
-        @balancer.should_receive(:request).and_yield(url).once
-        @client.send(:request, :post, @path)
+      it "makes poll request after get request if result is nil and there is an unexpired connection" do
+        get_response = [nil, 200, nil, @headers]
+        @balancer.should_receive(:request).and_return do
+          @http_client.instance_variable_get(:@connections)[@path] = @connection
+          get_response
+        end.once
+        flexmock(@http_client).should_receive(:poll).with(@connection, @request_options, @stop_at).and_return(@response).once
+        @client.send(:poll_request, @path, @connect_options, @request_options, @request_timeout, @started_at, @used).should == @response
       end
 
-      context "when filtering" do
-        before(:each) do
-          @client = RightScale::BalancedHttpClient.new(@urls, :filter_params => [:secret])
-          @params = {:public => "data", "secret" => "data", "very secret" => "data"}
-          @options = {:filter_params => ["very secret"]}
-          @filtered_params = "{:public=>\"data\", \"secret\"=>\"<hidden>\", \"very secret\"=>\"<hidden>\"}"
-        end
+      it "makes poll request without get request if there is an unexpired connection" do
+        @connection[:expires_at] = @later + 10
+        @http_client.instance_variable_get(:@connections)[@path] = @connection
+        flexmock(@http_client).should_receive(:poll).with(@connection, @request_options, @stop_at).and_return(@response).once
+        flexmock(@http_client).should_receive(:request).never
+        @client.send(:poll_request, @path, @connect_options, @request_options, @request_timeout, @started_at, @used).should == @response
+      end
 
-        it "logs request with filtered params if in debug mode" do
-          @log.should_receive(:level).and_return(:debug)
-          @http_client.should_receive(:post).and_return(@response)
-          @log.should_receive(:info).with("Requesting POST <random uuid> /foo/bar #{@filtered_params}").once
-          @log.should_receive(:info).with("Completed <random uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes | {\"out\"=>123}").once
-          @client.send(:request, :post, @path, @params, @options)
-        end
+      it "returns result and response info" do
+        @balancer.should_receive(:request).and_yield(@url).and_return(@response).once
+        @client.send(:poll_request, @path, @connect_options, @request_options, @request_timeout, @started_at, @used).should == @response
+      end
 
-        it "logs response failure including filtered params" do
-          @http_client.should_receive(:post).and_raise(RestExceptionMock.new(400, "bad data")).once
-          @log.should_receive(:info).with("Requesting POST <random uuid> /foo/bar").once
-          @log.should_receive(:error).with("Failed <random uuid> in 10ms | 400 [http://my.com/foo/bar #{@filtered_params}] | 400 Bad Request: bad data").once
-          lambda { @client.send(:request, :post, @path, @params, @options) }.should raise_error(RuntimeError)
-        end
+      it "passes back host used" do
+        @balancer.should_receive(:request).and_yield("http://111:secret@my.com").and_return(@response).once
+        @client.send(:poll_request, @path, @connect_options, @request_options, @request_timeout, @started_at, @used)
+        @used[:host].should == "http://my.com"
       end
     end
   end
@@ -364,8 +372,8 @@ describe RightScale::BalancedHttpClient do
     end
 
     it "uses last exception stored in NoResult details" do
-      gateway_timeout = RestExceptionMock.new(504, "server timeout")
-      bad_request = RestExceptionMock.new(400, "bad data")
+      gateway_timeout = RightScale::HttpExceptions.create(504, "server timeout")
+      bad_request = RightScale::HttpExceptions.create(400, "bad data")
       @no_result = RightSupport::Net::NoResult.new("no result", {@url => gateway_timeout, @url => bad_request})
       lambda { @client.send(:handle_no_result, @no_result, @url, &@proc) }.should raise_error(bad_request)
       @yielded.should == bad_request
@@ -378,7 +386,7 @@ describe RightScale::BalancedHttpClient do
     end
 
     it "uses http_body in raised NotResponding exception if status code is 504" do
-      gateway_timeout = RestExceptionMock.new(504, "server timeout")
+      gateway_timeout = RightScale::HttpExceptions.create(504, "server timeout")
       @no_result = RightSupport::Net::NoResult.new("no result", {@url => gateway_timeout})
       lambda { @client.send(:handle_no_result, @no_result, @url, &@proc) }.should \
         raise_error(RightScale::BalancedHttpClient::NotResponding, "server timeout")
@@ -386,7 +394,7 @@ describe RightScale::BalancedHttpClient do
     end
 
     it "uses raise NotResponding if status code is 504 and http_body is nil or empty" do
-      gateway_timeout = RestExceptionMock.new(504, "")
+      gateway_timeout = RightScale::HttpExceptions.create(504, "")
       @no_result = RightSupport::Net::NoResult.new("no result", {@url => gateway_timeout})
       lambda { @client.send(:handle_no_result, @no_result, @url, &@proc) }.should \
         raise_error(RightScale::BalancedHttpClient::NotResponding, "http://my.com not responding")
@@ -395,7 +403,7 @@ describe RightScale::BalancedHttpClient do
 
     [502, 503].each do |code|
       it "uses server name in NotResponding exception if status code is #{code}" do
-        e = RestExceptionMock.new(code)
+        e = RightScale::HttpExceptions.create(code)
         @no_result = RightSupport::Net::NoResult.new("no result", {@url => e})
         lambda { @client.send(:handle_no_result, @no_result, @url, &@proc) }.should \
           raise_error(RightScale::BalancedHttpClient::NotResponding, "http://my.com not responding")
@@ -403,34 +411,63 @@ describe RightScale::BalancedHttpClient do
     end
 
     it "raises last exception in details if not retryable" do
-      bad_request = RestExceptionMock.new(400, "bad data")
+      bad_request = RightScale::HttpExceptions.create(400, "bad data")
       @no_result = RightSupport::Net::NoResult.new("no result", {@url => bad_request})
       lambda { @client.send(:handle_no_result, @no_result, @url, &@proc) }.should raise_error(bad_request)
       @yielded.should == bad_request
     end
   end
 
-  context :report_failure do
+  context :log_success do
+    before(:each) do
+      @result = {"out" => 123}
+      @body = JSON.dump({:out => 123})
+      @headers = {:status => "200 OK"}
+      @response = [@result, 200, @body, @headers]
+      @client = RightScale::BalancedHttpClient.new(@urls)
+    end
+
+    it "logs response length using header :content_length if available" do
+      @headers.merge!(:content_length => 99)
+      @log.should_receive(:info).with("Completed <uuid> in 10ms | 200 [http://my.com/foo/bar] | 99 bytes").once
+      @client.send(:log_success, @result, 200, @body, @headers, "http://my.com", @path, "uuid", @now, :info).should be true
+    end
+
+    it "logs response length using response body size of :content_lenght not available" do
+      @log.should_receive(:info).with("Completed <uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes").once
+      @client.send(:log_success, @result, 200, @body, @headers, "http://my.com", @path, "uuid", @now, :info).should be true
+    end
+
+    it "logs duration based on request start time" do
+      @log.should_receive(:info).with("Completed <uuid> in 20ms | 200 [http://my.com/foo/bar] | 11 bytes").once
+      @client.send(:log_success, @result, 200, @body, @headers, "http://my.com", @path, "uuid", @now - 0.01, :info).should be true
+    end
+
+    it "log result if log level option set to :debug" do
+      @log.should_receive(:debug).with("Completed <uuid> in 10ms | 200 [http://my.com/foo/bar] | 11 bytes | {\"out\"=>123}").once
+      @client.send(:log_success, @result, 200, @body, @headers, "http://my.com", @path, "uuid", @now, :debug).should be true
+    end
+  end
+
+  context :log_failure do
     before(:each) do
-      @started_at = Time.now
-      flexmock(Time).should_receive(:now).and_return(@started_at + 0.01)
       @client = RightScale::BalancedHttpClient.new(@urls)
     end
 
     it "logs exception" do
-      exception = RestExceptionMock.new(400, "bad data")
+      exception = RightScale::HttpExceptions.create(400, "bad data")
       @log.should_receive(:error).with("Failed <uuid> in 10ms | 400 [http://my.com/foo/bar \"params\"] | 400 Bad Request: bad data").once
-      @client.send(:report_failure, @url, @path, "params", [], "uuid", @started_at, exception)
+      @client.send(:log_failure, @url, @path, "params", [], "uuid", @now, :info, exception).should be true
     end
 
     it "logs error string" do
       @log.should_receive(:error).with("Failed <uuid> in 10ms | nil [http://my.com/foo/bar \"params\"] | bad data").once
-      @client.send(:report_failure, @url, @path, "params", [], "uuid", @started_at, "bad data")
+      @client.send(:log_failure, @url, @path, "params", [], "uuid", @now, :info, "bad data").should be true
     end
 
-    it "filters params" do
+    it "filters parameters" do
       @log.should_receive(:error).with("Failed <uuid> in 10ms | nil [http://my.com/foo/bar {:secret=>\"<hidden>\"}] | bad data").once
-      @client.send(:report_failure, @url, @path, {:secret => "data"}, ["secret"], "uuid", @started_at, "bad data")
+      @client.send(:log_failure, @url, @path, {:secret => "data"}, ["secret"], "uuid", @now, :info, "bad data").should be true
     end
   end
 
@@ -443,22 +480,21 @@ describe RightScale::BalancedHttpClient do
 
       context "in info mode with no host" do
         it "generates text containing path" do
-          text = @client.send(:log_text, @path, {:value => 123}, [])
+          text = @client.send(:log_text, @path, {:value => 123}, [], :info)
           text.should == "/foo/bar"
         end
       end
 
       context "in info mode with host" do
         it "generates text containing host and path" do
-          text = @client.send(:log_text, @path, {:value => 123}, [], @url)
+          text = @client.send(:log_text, @path, {:value => 123}, [], :info, @url)
           text.should == "[http://my.com/foo/bar]"
         end
       end
 
       context "and in debug mode" do
-        it "generates text containing containing host, path, and filtered params" do
-          @log.should_receive(:level).and_return(:debug)
-          text = @client.send(:log_text, @path, {:some => "data", :secret => "data"}, ["secret"], @url)
+        it "generates text containing containing host, path, and filtered parameters" do
+          text = @client.send(:log_text, @path, {:some => "data", :secret => "data"}, ["secret"], :debug, @url)
           text.should == "[http://my.com/foo/bar {:some=>\"data\", :secret=>\"<hidden>\"}]"
         end
       end
@@ -466,13 +502,13 @@ describe RightScale::BalancedHttpClient do
 
     context "when exception" do
       it "includes params regardless of mode" do
-        text = @client.send(:log_text, @path, {:some => "data", :secret => "data"}, ["secret"], @url, "failed")
+        text = @client.send(:log_text, @path, {:some => "data", :secret => "data"}, ["secret"], :info, @url, "failed")
         text.should == "[http://my.com/foo/bar {:some=>\"data\", :secret=>\"<hidden>\"}] | failed"
       end
 
       it "includes exception text" do
-        exception = RestExceptionMock.new(400, "bad data")
-        text = @client.send(:log_text, @path, {:some => "data", :secret => "data"}, ["secret"], @url, exception)
+        exception = RightScale::HttpExceptions.create(400, "bad data")
+        text = @client.send(:log_text, @path, {:some => "data", :secret => "data"}, ["secret"], :info, @url, exception)
         text.should == "[http://my.com/foo/bar {:some=>\"data\", :secret=>\"<hidden>\"}] | 400 Bad Request: bad data"
       end
     end
@@ -489,7 +525,7 @@ describe RightScale::BalancedHttpClient do
       @client.send(:filter, params, filter).should == {:some => 1, "secret" => "<hidden>"}
     end
 
-    it "converts param names to string before comparing to filter list" do
+    it "converts parameter names to string before comparing to filter list" do
       filter = ["secret", "very secret"]
       params = {:some => 1, :secret => "data", "very secret" => "data"}
       @client.send(:filter, params, filter).should == {:some => 1, :secret => "<hidden>", "very secret" => "<hidden>"}
@@ -527,41 +563,110 @@ describe RightScale::BalancedHttpClient do
     end
   end
 
+  context :format do
+    before(:each) do
+      @client = RightScale::BalancedHttpClient
+    end
+
+    it "converts param to key=value format" do
+      @client.format({:foo => "bar"}).should == "foo=bar"
+    end
+
+    it "escapes param value" do
+      @client.format({:path => "/foo/bar"}).should == "path=%2Ffoo%2Fbar"
+    end
+
+    it "breaks arrays into multiple params" do
+      params = {:paths => ["/foo/bar", "/foo/bar2"]}
+      @client.format(params).should == "paths[]=%2Ffoo%2Fbar&paths[]=%2Ffoo%2Fbar2"
+    end
+
+    it "separates params with '&'" do
+      params = {:foo => 111, :paths => ["/foo/bar", "/foo/bar2"], :bar => 999}
+      response = @client.format(params)
+      response.split("&").sort.should == ["bar=999", "foo=111", "paths[]=%2Ffoo%2Fbar", "paths[]=%2Ffoo%2Fbar2"]
+    end
+  end
+
+  context :process_response do
+    before(:each) do
+      @result = {"out" => 123}
+      @body = JSON.dump({:out => 123})
+      @client = RightScale::BalancedHttpClient
+    end
+
+    it "returns location header for 201 response" do
+      @client.response(201, "", {:status => "201 Created", :location => "/href"}, false).should == "/href"
+    end
+
+    it "returns body without decoding by default" do
+      @client.response(200, @body, {:status => "200 OK"}, false).should == @body
+    end
+
+    it "returns JSON decoded response if decoding requested" do
+      @client.response(200, @body, {:status => "200 OK"}, true).should == @result
+    end
+
+    it "returns nil if JSON decoded response is empty" do
+      @client.response(200, "null", {:status => "200 OK"}, true).should be nil
+    end
+
+    it "returns nil if response is empty" do
+      @client.response(204, nil, {:status => "204 No Content"}, false).should be nil
+    end
+
+    it "returns nil if response is nil" do
+      @client.response(200, nil, {:status => "200 OK"}, false).should be nil
+    end
+
+    it "returns nil if body is empty" do
+      @client.response(200, "", {:status => "200 OK"}, false).should be nil
+    end
+
+    it "raises exception if response code indicates failure" do
+      lambda { @client.response(400, nil, {:status => "400 Bad Request"}, false) }.should raise_error(RightScale::HttpExceptions::BadRequest)
+    end
+  end
+
   context :exception_text do
+    before(:each) do
+      @client = RightScale::BalancedHttpClient
+    end
+
     context "when string exception" do
       it "adds exception text" do
-        RightScale::BalancedHttpClient.exception_text("failed").should == "failed"
+        @client.exception_text("failed").should == "failed"
       end
     end
 
     context "when REST exception" do
       it "adds exception code/type and any http_body" do
-        exception = RestExceptionMock.new(400, "bad data")
-        RightScale::BalancedHttpClient.exception_text(exception).should == "400 Bad Request: bad data"
+        exception = RightScale::HttpExceptions.create(400, "bad data")
+        @client.exception_text(exception).should == "400 Bad Request: bad data"
       end
 
       it "adds exception code/type but omits http_body if it is html" do
-        exception = RestExceptionMock.new(400, "<html> bad </html>")
-        RightScale::BalancedHttpClient.exception_text(exception).should == "400 Bad Request"
+        exception = RightScale::HttpExceptions.create(400, "<html> bad </html>")
+        @client.exception_text(exception).should == "400 Bad Request"
       end
 
       it "adds exception code/type and omits http_body if it is blank" do
-        exception = RestExceptionMock.new(400)
-        RightScale::BalancedHttpClient.exception_text(exception).should == "400 Bad Request"
+        exception = RightScale::HttpExceptions.create(400, "")
+        @client.exception_text(exception).should == "400 Bad Request"
       end
     end
 
     context "when NoResult exception" do
       it "adds exception class and message" do
         exception = RightSupport::Net::NoResult.new("no result")
-        RightScale::BalancedHttpClient.exception_text(exception).should == "RightSupport::Net::NoResult: no result"
+        @client.exception_text(exception).should == "RightSupport::Net::NoResult: no result"
       end
     end
 
     context "when non-REST, non-NoResult exception" do
       it "adds exception class and message" do
         exception = ArgumentError.new("bad arg")
-        RightScale::BalancedHttpClient.exception_text(exception).should == "ArgumentError: bad arg"
+        @client.exception_text(exception).should == "ArgumentError: bad arg"
       end
     end
 
@@ -569,7 +674,7 @@ describe RightScale::BalancedHttpClient do
       it "adds exception class, message, and backtrace" do
         exception = ArgumentError.new("bad arg")
         flexmock(exception).should_receive(:backtrace).and_return(["line 1", "line 2"])
-        RightScale::BalancedHttpClient.exception_text(exception).should == "ArgumentError: bad arg in\nline 1\nline 2"
+        @client.exception_text(exception).should == "ArgumentError: bad arg in\nline 1\nline 2"
       end
     end
   end