em-voldemort 0.1.5

data/lib/em-voldemort/errors.rb

@@ -0,0 +1,13 @@
+module EM::Voldemort
+  # Like HTTP 400 series responses, this exception is used for errors that are the client's fault.
+  # The request generally should not be retried.
+  class ClientError < RuntimeError; end
+
+  # Exception to indicate that the requested key does not exist in the Voldemort store.
+  class KeyNotFound < ClientError; end
+
+  # Like HTTP 500 series responses, this exception is used for errors on the server side or in the
+  # network. They are generally transient, so it makes sense to retry failed requests a limited
+  # number of times.
+  class ServerError < RuntimeError; end
+end

data/lib/em-voldemort/protobuf.rb

@@ -0,0 +1,105 @@
+module EM::Voldemort
+  # https://github.com/voldemort/voldemort/blob/master/src/proto/voldemort-client.proto
+  class Protobuf
+    class ClockEntry
+      include Beefcake::Message
+      required :node_id, :int32, 1
+      required :version, :int64, 2
+    end
+
+    class VectorClock
+      include Beefcake::Message
+      repeated :entries, ClockEntry, 1
+      optional :timestamp, :int64, 2
+    end
+
+    class Versioned
+      include Beefcake::Message
+      required :value, :bytes, 1
+      required :version, VectorClock, 2
+    end
+
+    class Error
+      include Beefcake::Message
+      required :error_code, :int32, 1
+      required :error_message, :string, 2
+    end
+
+    class KeyedVersions
+      include Beefcake::Message
+      required :key, :bytes, 1
+      repeated :versions, Versioned, 2
+    end
+
+    class GetRequest
+      include Beefcake::Message
+      optional :key, :bytes, 1
+    end
+
+    class GetResponse
+      include Beefcake::Message
+      repeated :versioned, Versioned, 1
+      optional :error, Error, 2
+    end
+
+    class GetVersionResponse
+      include Beefcake::Message
+      repeated :versions, VectorClock, 1
+      optional :error, Error, 2
+    end
+
+    class GetAllRequest
+      include Beefcake::Message
+      repeated :keys, :bytes, 1
+    end
+
+    class GetAllResponse
+      include Beefcake::Message
+      repeated :values, KeyedVersions, 1
+      optional :error, Error, 2
+    end
+
+    class PutRequest
+      include Beefcake::Message
+      required :key, :bytes, 1
+      required :versioned, Versioned, 2
+    end
+
+    class PutResponse
+      include Beefcake::Message
+      optional :error, Error, 1
+    end
+
+    class DeleteRequest
+      include Beefcake::Message
+      required :key, :bytes, 1
+      required :version, VectorClock, 2
+    end
+
+    class DeleteResponse
+      include Beefcake::Message
+      required :success, :bool, 1
+      optional :error, Error, 2
+    end
+
+    module RequestType
+      GET = 0
+      GET_ALL = 1
+      PUT = 2
+      DELETE = 3
+      GET_VERSION = 4
+    end
+
+    class Request
+      include Beefcake::Message
+      required :type, RequestType, 1
+      required :should_route, :bool, 2, :default => false
+      required :store, :string, 3
+      optional :get, GetRequest, 4
+      optional :getAll, GetAllRequest, 5
+      optional :put, PutRequest, 6
+      optional :delete, DeleteRequest, 7
+      optional :requestRouteType, :int32, 8
+    end
+  end
+end

data/lib/em-voldemort/protocol.rb

@@ -0,0 +1,23 @@
+module EM::Voldemort
+  # Implementation of Voldemort's pb0 (protocol buffers) protocol.
+  # Very incomplete -- currently only supports the get command.
+  module Protocol
+    def get_request(store, key)
+      Protobuf::Request.new(
+        :type => Protobuf::RequestType::GET,
+        :should_route => false,
+        :store => store.to_s,
+        :get => Protobuf::GetRequest.new(:key => key.to_s)
+      ).encode.to_s
+    end
+
+    def get_response(bytes)
+      response = Protobuf::GetResponse.decode(bytes.dup)
+      if response.error
+        raise ClientError, "GetResponse error #{response.error.error_code}: #{response.error.error_message}"
+      end
+      raise KeyNotFound if response.versioned.nil? || response.versioned.empty?
+      response.versioned.max{|a, b| a.version.timestamp <=> b.version.timestamp }.value
+    end
+  end
+end

data/lib/em-voldemort/router.rb

@@ -0,0 +1,62 @@
+module EM::Voldemort
+  # For a given request, the router figures out which partition on which node in the cluster it
+  # should be sent to. Ruby port of voldemort.routing.ConsistentRoutingStrategy.
+  class Router
+
+    def initialize(type, replicas)
+      raise ClientError, "unsupported routing strategy: #{type}" if !type && type != 'consistent-routing'
+      raise ClientError, "bad number of replicas: #{replicas.inspect}" if !replicas || replicas <= 0
+      @replicas = replicas
+    end
+
+    # Returns a list of partitions on which a particular key can be found.
+    #
+    # @param key A binary string
+    # @param partitions Hash of partition number to node
+    # @returns Array of partitions (numbers between 0 and partitions.size - 1)
+    def partitions(key, partitions)
+      master = fnv_hash(key) % partitions.size
+      selected = [master]
+      nodes = [partitions[master]]
+      current = (master + 1) % partitions.size
+
+      # Walk clockwise around the ring of partitions, starting from the master partition.
+      # The next few unique nodes in ring order are the replicas.
+      while current != master && selected.size < @replicas
+        if !nodes.include? partitions[current]
+          nodes << partitions[current]
+          selected << current
+        end
+        current = (current + 1) % partitions.size
+      end
+
+      selected
+    end
+
+
+    private
+
+    FNV_BASIS = 0x811c9dc5
+    FNV_PRIME = (1 << 24) + 0x193
+
+    # Port of voldemort.utils.FnvHashFunction. See also http://www.isthe.com/chongo/tech/comp/fnv
+    # Returns a number between 0 and 2**31 - 1.
+    def fnv_hash(bytes)
+      hash = FNV_BASIS
+      bytes.each_byte do |byte|
+        hash = (hash ^ byte) * FNV_PRIME % 2**64
+        hash -= 2**64 if hash >= 2**63 # simulate overflow of signed long
+      end
+
+      # cast signed long to signed int
+      hash = hash % 2**32
+      hash -= 2**32 if hash >= 2**31
+
+      # modified absolute value, as per voldemort.routing.ConsistentRoutingStrategy.abs(int)
+      hash = 2**31 - 1 if hash == -2**31
+      hash = -hash if hash < 0
+      hash
+    end
+
+  end
+end

data/lib/em-voldemort/store.rb

@@ -0,0 +1,108 @@
+module EM::Voldemort
+  # Provides access to one particular store on a Voldemort cluster. Deals with encoding of keys and
+  # values.
+  class Store
+
+    attr_reader :cluster, :store_name
+
+    # Creates a new store client from a URL like voldemort://node0.example.com:6666/store-name
+    def self.from_url(url, options={})
+      url = URI.parse(url) if url.is_a?(String)
+      cluster = Cluster.new({:host => url.host, :port => url.port || 6666}.merge(options))
+      cluster.store(url.path.sub(%r{\A/}, ''))
+    end
+
+    # Internal -- don't call this from application code. Use {Cluster#store} or {Store.from_url}
+    # instead.
+    def initialize(cluster, store_name)
+      @cluster = cluster
+      @store_name = store_name.to_s
+    end
+
+    # Internal
+    def load_config(xml)
+      @persistence = xml.xpath('persistence').text
+      @router = Router.new(xml.xpath('routing-strategy').text, xml.xpath('replication-factor').text.to_i)
+      @key_serializer = make_serializer(xml.xpath('key-serializer').first)
+      @key_compressor = Compressor.new(xml.xpath('key-serializer/compression').first)
+      @value_serializer = make_serializer(xml.xpath('value-serializer').first)
+      @value_compressor = Compressor.new(xml.xpath('value-serializer/compression').first)
+    end
+
+    # Fetches the value associated with a particular key in this store. Returns a deferrable that
+    # succeeds with the value, or fails with an exception object. If a serializer is configured for
+    # the store, the key is automatically serialized and the value automatically unserialized.
+    def get(key)
+      EM::DefaultDeferrable.new.tap do |request|
+        if @persistence
+          get_after_bootstrap(key, request)
+        else
+          bootstrap = @cluster.connect
+          if bootstrap
+            bootstrap.callback { get_after_bootstrap(key, request) }
+            bootstrap.errback do
+              request.fail(ServerError.new('Could not bootstrap Voldemort cluster'))
+            end
+          else
+            request.fail(ClientError.new("Store #{store_name} is not configured on the cluster"))
+          end
+        end
+      end
+    end
+
+    private
+
+    def get_after_bootstrap(key, deferrable)
+      if @persistence.nil?
+        deferrable.fail(ClientError.new("Store #{store_name} is not configured on the cluster"))
+      elsif @persistence != 'read-only'
+        deferrable.fail(ClientError.new("Sorry, accessing #{persistence} stores is not yet supported"))
+      else
+        begin
+          encoded_key = encode_key(key)
+        rescue => error
+          deferrable.fail(error)
+        else
+          request = @cluster.get(store_name, encoded_key, @router)
+          request.errback {|error| deferrable.fail(error) }
+
+          request.callback do |response|
+            begin
+              value = decode_value(response)
+            rescue => error
+              deferrable.fail(error)
+            else
+              deferrable.succeed(value)
+            end
+          end
+        end
+      end
+    end
+
+    def encode_key(key)
+      @key_compressor.encode(@key_serializer.encode(key))
+    end
+
+    def decode_value(value)
+      @value_serializer.decode(@value_compressor.decode(value))
+    end
+
+    def make_serializer(xml)
+      if xml.xpath('type').text == 'json'
+        has_version_tag = true
+        schemas = xml.xpath('schema-info').each_with_object({}) do |schema, hash|
+          has_version_tag = false if schema['version'] == 'none'
+          hash[schema['version'].to_i] = schema.text
+        end
+        BinaryJson.new(schemas, has_version_tag)
+      else
+        NullSerializer.new
+      end
+    end
+
+    class NullSerializer
+      def encode(data); data; end
+      def decode(data); data; end
+    end
+  end
+end

data/spec/em-voldemort/binary_json_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe EM::Voldemort::BinaryJson do
+  describe 'encoding strings' do
+    before do
+      @codec = EM::Voldemort::BinaryJson.new(0 => '"string"')
+    end
+
+    it 'should encode short strings' do
+      @codec.encode('hello').should == "\x00\x00\x05hello"
+    end
+
+    it 'should decode short strings' do
+      @codec.decode("\x00\x00\x05hello").should == 'hello'
+    end
+
+    it 'should encode strings between 16kB and 32kB in length' do
+      @codec.encode('hellohello' * 1700).should == "\x00\x42\x68" + 'hellohello' * 1700
+    end
+
+    it 'should decode strings between 16kB and 32kB in length' do
+      @codec.decode("\x00\x42\x68" + 'hellohello' * 1700).should == 'hellohello' * 1700
+    end
+
+    it 'should encode strings above 32kB in length' do
+      @codec.encode('hellohello' * 3400).should == "\x00\xC0\x00\x84\xd0" + 'hellohello' * 3400
+    end
+
+    it 'should decode strings above 32kB in length' do
+      @codec.decode("\x00\xC0\x00\x84\xd0" + 'hellohello' * 3400).should == 'hellohello' * 3400
+    end
+  end
+end

data/spec/em-voldemort/cluster_spec.rb

@@ -0,0 +1,363 @@
+require 'spec_helper'
+
+describe EM::Voldemort::Cluster do
+  before do
+    @timer_double = double('timer', :cancel => nil)
+    EM::Voldemort::Cluster.any_instance.stub(:setup_bootstrap_timer) do |&timer|
+      @bootstrap_timer = timer
+      @timer_double
+    end
+
+    @logger = Logger.new($stdout)
+    @logger.level = Logger::ERROR
+    @cluster = EM::Voldemort::Cluster.new :host => 'localhost', :port => 6666, :logger => @logger
+
+    Timecop.freeze
+  end
+
+  def request(store, key)
+    EM::Voldemort::Protobuf::Request.new(
+      :type => EM::Voldemort::Protobuf::RequestType::GET,
+      :should_route => false,
+      :store => store.to_s,
+      :get => EM::Voldemort::Protobuf::GetRequest.new(:key => key.to_s)
+    ).encode.to_s
+  end
+
+  def success_response(value)
+    EM::Voldemort::Protobuf::GetResponse.new(
+      :versioned => [EM::Voldemort::Protobuf::Versioned.new(
+        :value => value,
+        :version => EM::Voldemort::Protobuf::VectorClock.new(
+          :entries => [], # read-only stores leave this empty
+          :timestamp => (Time.now.to_r * 1000).to_i
+        )
+      )]
+    ).encode.to_s
+  end
+
+  def error_response(code, message)
+    EM::Voldemort::Protobuf::GetResponse.new(
+      :error => EM::Voldemort::Protobuf::Error.new(:error_code => code, :error_message => message)
+    ).encode.to_s
+  end
+
+  def cluster_xml(partitions_by_host={})
+    servers = ''
+    partitions_by_host.each_with_index do |(host, partitions), index|
+      servers << '<server>'
+      servers << "<id>#{index}</id>"
+      servers << "<host>#{host}</host>"
+      servers << '<http-port>8081</http-port><socket-port>6666</socket-port><admin-port>6667</admin-port>'
+      servers << "<partitions>#{partitions.join(', ')}</partitions>"
+      servers << '</server>'
+    end
+    "<cluster><name>example-cluster</name>#{servers}</cluster>"
+  end
+
+  def stores_xml(properties_by_name={})
+    stores = ''
+    properties_by_name.each_pair do |name, properties|
+      stores << '<store>'
+      stores << "<name>#{name}</name>"
+      stores << '<persistence>read-only</persistence>'
+      stores << '<routing-strategy>consistent-routing</routing-strategy>'
+      stores << '<routing>client</routing>'
+      stores << '<replication-factor>2</replication-factor>'
+      stores << '<required-reads>1</required-reads>'
+      stores << '<required-writes>1</required-writes>'
+      stores << '<key-serializer>'
+      stores << "<type>#{properties[:key_type]}</type>"
+      stores << "<schema-info version=\"0\">#{properties[:key_schema]}</schema-info>"
+      stores << '</key-serializer>'
+      stores << '<value-serializer>'
+      stores << "<type>#{properties[:value_type]}</type>"
+      properties[:value_schemas].each_pair do |version, schema|
+        stores << "<schema-info version=\"#{version}\">#{schema}</schema-info>"
+      end
+      stores << "<compression><type>#{properties[:compression]}</type></compression>"
+      stores << '</value-serializer>'
+      stores << '</store>'
+    end
+    "<stores>#{stores}</stores>"
+  end
+
+  def expect_bootstrap(cluster_info={}, stores_info={})
+    @bootstrap_connection = double('bootstrap connection')
+    EM::Voldemort::Connection.should_receive(:new).
+      with(:host => 'localhost', :port => 6666, :logger => @logger).
+      and_return(@bootstrap_connection)
+
+    cluster_request = EM::DefaultDeferrable.new
+    @bootstrap_connection.should_receive(:send_request).with(request('metadata', 'cluster.xml')) do
+      EM.next_tick do
+        stores_request = EM::DefaultDeferrable.new
+        @bootstrap_connection.should_receive(:send_request).with(request('metadata', 'stores.xml')) do
+          EM.next_tick do
+            stores_request.succeed(success_response(stores_xml(stores_info)))
+          end
+          stores_request
+        end
+        cluster_request.succeed(success_response(cluster_xml(cluster_info)))
+      end
+      cluster_request
+    end
+
+    @bootstrap_connection.should_receive(:close) { yield if block_given? }
+  end
+
+
+  it 'should request cluster.xml and stores.xml when bootstrapping' do
+    expect_bootstrap('voldemort0.example.com' => [0])
+    EM::Voldemort::Connection.should_receive(:new).
+      with(:host => 'voldemort0.example.com', :port => 6666, :node_id => 0, :logger => @logger).
+      and_return(double('Connection 0'))
+    EM.run { @cluster.connect.callback { EM.stop } }
+  end
+
+  it 'should make a connection to each node in the cluster' do
+    expect_bootstrap('voldemort0.example.com' => [0, 1, 2, 3], 'voldemort1.example.com' => [4, 5, 6, 7])
+    EM::Voldemort::Connection.should_receive(:new).
+      with(:host => 'voldemort0.example.com', :port => 6666, :node_id => 0, :logger => @logger).
+      and_return(double('Connection 0'))
+    EM::Voldemort::Connection.should_receive(:new).
+      with(:host => 'voldemort1.example.com', :port => 6666, :node_id => 1, :logger => @logger).
+      and_return(double('Connection 1'))
+    EM.run { @cluster.connect.callback { EM.stop } }
+  end
+
+  it 'should retry bootstrapping if it fails' do
+    EM.run do
+      request1 = EM::DefaultDeferrable.new
+      connection1 = double('connection attempt 1', :send_request => request1, :close => nil)
+      EM::Voldemort::Connection.should_receive(:new).and_return(connection1)
+      @cluster.connect
+      @bootstrap_timer.call
+      EM.next_tick do
+        request1.fail(EM::Voldemort::ServerError.new('connection refused'))
+        EM.next_tick do
+          request2 = EM::DefaultDeferrable.new
+          connection2 = double('connection attempt 2', :send_request => request2, :close => nil)
+          EM::Voldemort::Connection.should_receive(:new).and_return(connection2)
+          @bootstrap_timer.call
+          EM.next_tick do
+            request2.fail(EM::Voldemort::ServerError.new('connection refused'))
+            EM.next_tick do
+              EM::Voldemort::Connection.should_receive(:new).
+                with(:host => 'voldemort0.example.com', :port => 6666, :node_id => 0, :logger => @logger).
+                and_return(double('Connection 0'))
+              expect_bootstrap('voldemort0.example.com' => [0]) { EM.stop }
+              @timer_double.should_receive(:cancel)
+              @bootstrap_timer.call
+            end
+          end
+        end
+      end
+    end
+  end
+
+  it 'should delay requests until bootstrapping is complete' do
+    metadata_request = EM::DefaultDeferrable.new
+    bootstrap = double('bootstrap connection', :send_request => metadata_request, :close => nil)
+    EM::Voldemort::Connection.should_receive(:new).and_return(bootstrap)
+    connection = double('connection', :health => :good)
+    EM::Voldemort::Connection.should_receive(:new).and_return(connection)
+    EM.run do
+      @cluster.get('store1', 'request1').callback {|response| @response1 = response }
+      @cluster.get('store1', 'request2').callback do |response|
+        @response1.should == 'response1'
+        response.should == 'response2'
+        EM.stop
+      end
+      EM.next_tick do
+        connection.should_receive(:send_request).with(request('store1', 'request1')) do
+          EM::DefaultDeferrable.new.tap do |deferrable|
+            EM.next_tick { deferrable.succeed(success_response('response1')) }
+          end
+        end
+        connection.should_receive(:send_request).with(request('store1', 'request2')) do
+          EM::DefaultDeferrable.new.tap do |deferrable|
+            EM.next_tick { deferrable.succeed(success_response('response2')) }
+          end
+        end
+        metadata_request.succeed(success_response(cluster_xml('voldemort0.example.com' => [0, 1, 2, 3])))
+      end
+    end
+  end
+
+  it 'should fail requests if bootstrapping failed' do
+    EM.run do
+      metadata_request = EM::DefaultDeferrable.new
+      bootstrap = double('bootstrap connection', :send_request => metadata_request, :close => nil)
+      EM::Voldemort::Connection.should_receive(:new).and_return(bootstrap)
+      @cluster.get('store1', 'request1').errback {|error| @error = error }
+      EM.next_tick do
+        @error.should be_nil
+        metadata_request.fail(EM::Voldemort::ServerError.new('connection refused'))
+        @error.should be_a(EM::Voldemort::ServerError)
+        EM.stop
+      end
+    end
+  end
+
+  it 'should handle invalid XML responses' do
+    bootstrap = double('bootstrap connection', :close => nil)
+    EM::Voldemort::Connection.should_receive(:new).and_return(bootstrap)
+    cluster_request = EM::DefaultDeferrable.new
+    bootstrap.should_receive(:send_request).with(request('metadata', 'cluster.xml')) do
+      EM.next_tick { cluster_request.succeed(success_response("<xml>Ceci n'est pas XML.</xml>")) }
+      cluster_request
+    end
+    EM.run { @cluster.connect.errback { EM.stop } }
+  end
+
+
+  describe 'handling unavailable nodes' do
+    before do
+      expect_bootstrap('node0' => [0, 1, 2, 3], 'node1' => [4, 5, 6, 7])
+      @conn0 = double('connection 0')
+      @conn1 = double('connection 1')
+      EM::Voldemort::Connection.should_receive(:new).with(hash_including(:host => 'node0')).and_return(@conn0)
+      EM::Voldemort::Connection.should_receive(:new).with(hash_including(:host => 'node1')).and_return(@conn1)
+    end
+
+    it 'should only make a request to one connection if it is healthy' do
+      EM.run do
+        @conn0.should_receive(:health).and_return(:good)
+        @conn0.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request1|
+            EM.next_tick { request1.succeed(success_response('response1')) }
+          end
+        end
+        @cluster.get('store', 'request', double('router', :partitions => [2, 4])).callback do |response|
+          response.should == 'response1'
+          EM.stop
+        end
+      end
+    end
+
+    it 'should retry a request on another connection if the first request failed' do
+      EM.run do
+        @conn0.should_receive(:health).and_return(:good)
+        @conn0.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request1|
+            EM.next_tick do
+              @conn1.should_receive(:send_request).with(request('store', 'request')) do
+                EM::DefaultDeferrable.new.tap do |request2|
+                  EM.next_tick { request2.succeed(success_response('response2')) }
+                end
+              end
+              request1.fail(EM::Voldemort::ServerError.new('connection closed'))
+            end
+          end
+        end
+        @cluster.get('store', 'request', double('router', :partitions => [2, 4])).callback do |response|
+          response.should == 'response2'
+          EM.stop
+        end
+      end
+    end
+
+    it 'should fail the request if all attempts fail' do
+      EM.run do
+        @conn0.should_receive(:health).and_return(:good)
+        @conn0.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request1|
+            EM.next_tick do
+              @conn1.should_receive(:send_request).with(request('store', 'request')) do
+                EM::DefaultDeferrable.new.tap do |request2|
+                  EM.next_tick { request2.fail(EM::Voldemort::ServerError.new('no route to host')) }
+                end
+              end
+              request1.fail(EM::Voldemort::ServerError.new('connection timed out'))
+            end
+          end
+        end
+        @cluster.get('store', 'request', double('router', :partitions => [2, 4])).errback do |error|
+          error.should be_a(EM::Voldemort::ServerError)
+          error.message.should == 'no route to host'
+          EM.stop
+        end
+      end
+    end
+
+    it 'should not retry requests that failed due to client error' do
+      EM.run do
+        @conn0.should_receive(:health).and_return(:good)
+        @conn0.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request1|
+            EM.next_tick { request1.succeed('') } # empty response = no value for that key
+          end
+        end
+        @cluster.get('store', 'request', double('router', :partitions => [2, 4])).errback do |error|
+          error.should be_a(EM::Voldemort::KeyNotFound)
+          EM.stop
+        end
+      end
+    end
+
+    it 'should retry a request on another connection if parsing the first response failed' do
+      EM.run do
+        @conn0.should_receive(:health).and_return(:good)
+        @conn0.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request1|
+            EM.next_tick do
+              @conn1.should_receive(:send_request).with(request('store', 'request')) do
+                EM::DefaultDeferrable.new.tap do |request2|
+                  EM.next_tick { request2.succeed(success_response('response2')) }
+                end
+              end
+              @logger.should_receive(:error).with(/protocol error/)
+              request1.succeed("\x00") # not valid protobuf
+            end
+          end
+        end
+        @cluster.get('store', 'request', double('router', :partitions => [2, 4])).callback do |response|
+          response.should == 'response2'
+          EM.stop
+        end
+      end
+    end
+
+    it 'should keep trying to make requests to a node that is down' do
+      EM.run do
+        @conn0.should_receive(:health).and_return(:bad)
+        @conn0.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request1|
+            EM.next_tick { request1.fail(EM::Voldemort::ServerError.new('not connected')) }
+          end
+        end
+        @conn1.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request2|
+            EM.next_tick { request2.succeed(success_response('response2')) }
+          end
+        end
+        @cluster.get('store', 'request', double('router', :partitions => [2, 4])).callback do |response|
+          response.should == 'response2'
+          EM.stop
+        end
+      end
+    end
+
+    it 'should fail the request if all nodes are down' do
+      EM.run do
+        @conn0.should_receive(:health).and_return(:bad)
+        @conn0.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request1|
+            EM.next_tick { request1.fail(EM::Voldemort::ServerError.new('not connected')) }
+          end
+        end
+        @conn1.should_receive(:send_request).with(request('store', 'request')) do
+          EM::DefaultDeferrable.new.tap do |request2|
+            EM.next_tick { request2.fail(EM::Voldemort::ServerError.new('not connected')) }
+          end
+        end
+        @cluster.get('store', 'request', double('router', :partitions => [2, 4])).errback do |error|
+          error.should be_a(EM::Voldemort::ServerError)
+          error.message.should == 'not connected'
+          EM.stop
+        end
+      end
+    end
+  end
+end