mogilefs-client 2.2.0 → 3.0.0.rc1

data/test/test_backend.rb

@@ -1,11 +1,9 @@
 # -*- encoding: binary -*-
 require 'test/unit'
-require 'test/setup'
+require './test/setup'
 
 $TESTING = true
 
-require 'mogilefs/backend'
-
 class MogileFS::Backend
 
   attr_accessor :hosts
@@ -36,37 +34,18 @@ class TestBackend < Test::Unit::TestCase
   end
 
   def test_do_request
-    received = Tempfile.new('received')
-    tmp = TempServer.new(Proc.new do |serv, port|
-      client, client_addr = serv.accept
-      client.sync = true
-      received.syswrite(client.recv(4096))
-      client.send "OK 1 you=win\r\n", 0
-    end)
-
-    @backend.hosts = [ "127.0.0.1:#{tmp.port}" ]
-
+    srv = TCPServer.new("127.0.0.1", 0)
+    port = srv.addr[1]
+    accepted = Thread.new do
+      client = srv.accept
+      client.write("OK 1 you=win\r\n")
+      client
+    end
+    @backend.hosts = [ "127.0.0.1:#{port}" ]
     assert_equal({'you' => 'win'},
                  @backend.do_request('go!', { 'fight' => 'team fight!' }))
-    received.sysseek(0)
-    assert_equal "go! fight=team+fight%21\r\n", received.sysread(4096)
-    ensure
-      TempServer.destroy_all!
-  end
-
-  def test_do_request_send_error
-    socket_request = ''
-    socket = Object.new
-    def socket.closed?() false end
-    def socket.write(request) raise SystemCallError, 'dummy' end
-
-    @backend.instance_variable_set '@socket', socket
-
-    assert_raises MogileFS::UnreachableBackendError do
-      @backend.do_request 'go!', { 'fight' => 'team fight!' }
-    end
-
-    assert_equal nil, @backend.instance_variable_get('@socket')
+    accepted = accepted.value
+    assert_equal "go! fight=team+fight%21\r\n", accepted.readpartial(4096)
   end
 
   def test_automatic_exception
@@ -89,19 +68,6 @@ class TestBackend < Test::Unit::TestCase
     assert MogileFS::Backend.const_defined?('SizeVerifyError')
   end
 
-  def test_do_request_truncated
-    socket_request = ''
-    socket = Object.new
-    def socket.closed?() false end
-    def socket.write(request) return request.length - 1 end
-
-    @backend.instance_variable_set '@socket', socket
-
-    assert_raises MogileFS::RequestTruncatedError do
-      @backend.do_request 'go!', { 'fight' => 'team fight!' }
-    end
-  end
-
   def test_make_request
     assert_equal "go! fight=team+fight%21\r\n",
                  @backend.make_request('go!', { 'fight' => 'team fight!' })
@@ -109,11 +75,11 @@ class TestBackend < Test::Unit::TestCase
 
   def test_parse_response
     assert_equal({'foo' => 'bar', 'baz' => 'hoge'},
-                 @backend.parse_response('OK 1 foo=bar&baz=hoge'))
+                 @backend.parse_response("OK 1 foo=bar&baz=hoge\r\n"))
 
     err = nil
     begin
-      @backend.parse_response('ERR you totally suck')
+      @backend.parse_response("ERR you totally suck\r\n")
     rescue MogileFS::Error => err
       assert_equal 'MogileFS::Backend::YouError', err.class.to_s
       assert_equal 'totally suck', err.message
@@ -126,6 +92,9 @@ class TestBackend < Test::Unit::TestCase
     assert_raises MogileFS::InvalidResponseError do
       @backend.parse_response 'garbage'
     end
+    assert_raises MogileFS::InvalidResponseError do
+      @backend.parse_response("OK 1 foo=bar&baz=hoge")
+    end
   end
 
   def test_parse_response_newline
@@ -141,89 +110,64 @@ class TestBackend < Test::Unit::TestCase
   end
 
   def test_readable_eh_not_readable
-    tmp = TempServer.new(Proc.new { |serv,port| serv.accept; sleep })
-    @backend = MogileFS::Backend.new(:hosts => [ "127.0.0.1:#{tmp.port}" ],
+    srv = TCPServer.new("127.0.0.1", 0)
+    port = srv.addr[1]
+    @backend = MogileFS::Backend.new(:hosts => [ "127.0.0.1:#{port}" ],
                                      :timeout => 0.5)
     begin
       @backend.do_request 'foo', {}
     rescue MogileFS::UnreadableSocketError => e
-      assert_equal "127.0.0.1:#{tmp.port} never became readable", e.message
+      assert_equal "127.0.0.1:#{port} never became readable", e.message
     rescue Exception => err
       flunk "MogileFS::UnreadableSocketError not raised #{err} #{err.backtrace}"
     else
       flunk "MogileFS::UnreadableSocketError not raised"
-    ensure
-      TempServer.destroy_all!
     end
   end
 
-  def test_socket
+  def test_socket_dead
     assert_equal({}, @backend.dead)
-    assert_raises MogileFS::UnreachableBackendError do @backend.socket end
+    assert_raises(MogileFS::UnreachableBackendError) do
+      @backend.do_request('test', {})
+    end
     assert_equal(['localhost:1'], @backend.dead.keys)
   end
 
-  def test_socket_robust
-    bad_accept = Tempfile.new('bad_accept')
-    accept = Tempfile.new('accept')
-    bad = Proc.new do |serv,port|
-      client, client_addr = serv.accept
-      bad_accept.syswrite('!')
-    end
-    good = Proc.new do |serv,port|
-      client, client_addr = serv.accept
-      accept.syswrite('.')
-      client.syswrite('.')
-      client.close
-      sleep
-    end
-    nr = 10
-
-    nr.times do
-      begin
-        t1 = TempServer.new(bad, ENV['TEST_DEAD_PORT'])
-        t2 = TempServer.new(good)
-        hosts = ["127.0.0.1:#{t1.port}", "127.0.0.1:#{t2.port}"]
-        @backend = MogileFS::Backend.new(:hosts => hosts.dup)
-        assert_equal({}, @backend.dead)
-        old_chld_handler = trap('CHLD', 'DEFAULT')
-        t1.destroy!
-        Process.waitpid(t1.pid)
-        trap('CHLD', old_chld_handler)
-        sock = @backend.socket
-        assert_equal Socket, sock.class
-        port = Socket.unpack_sockaddr_in(sock.getpeername).first
-        # p [ 'ports', "port=#{port}", "t1=#{t1.port}", "t2=#{t2.port}" ]
-        assert_equal t2.port, port
-        IO.select([sock])
-        assert_equal '.', sock.sysread(1)
-      ensure
-        TempServer.destroy_all!
+  def test_socket_robust_on_dead_server
+    10.times do
+      t1 = TCPServer.new("127.0.0.1", 0)
+      t2 = TCPServer.new("127.0.0.1", 0)
+      hosts = ["127.0.0.1:#{t1.addr[1]}", "127.0.0.1:#{t2.addr[1]}"]
+      @backend = MogileFS::Backend.new(:hosts => hosts.dup)
+      assert_equal({}, @backend.dead)
+      t1.close
+      thr = Thread.new do
+        client = t2.accept
+        client.write("OK 1 foo=bar\r\n")
+        client
       end
-    end # nr.times
-    assert_equal 0, bad_accept.stat.size
-    assert_equal nr, accept.stat.size
+      rv = nil
+      assert_nothing_raised do
+        rv = @backend.do_request('test', { "all" => "ALL" })
+      end
+      accepted = thr.value
+      assert_equal "test all=ALL\r\n", accepted.readpartial(666)
+      assert_equal({"foo"=>"bar"}, rv)
+    end
   end
 
   def test_shutdown
-    accept_nr = 0
-    tmp = TempServer.new(Proc.new do |serv,port|
-      client, client_addr = serv.accept
-      accept_nr += 1
-      r = IO.select([client], [client])
-      client.syswrite(accept_nr.to_s)
-      sleep
-    end)
-    @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{tmp.port}" ]
+    srv = TCPServer.new('127.0.0.1', 0)
+    port = srv.addr[1]
+    @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{port}" ]
     assert @backend.socket
     assert ! @backend.socket.closed?
-    IO.select([@backend.socket])
-    resp = @backend.socket.sysread(4096)
+    client = srv.accept
+    client.write '1'
+    resp = @backend.socket.read(1)
     @backend.shutdown
     assert_equal nil, @backend.instance_variable_get(:@socket)
     assert_equal 1, resp.to_i
-    ensure
-      TempServer.destroy_all!
   end
 
   def test_url_decode
@@ -273,7 +217,6 @@ class TestBackend < Test::Unit::TestCase
     actual = input.map { |c| @backend.url_unescape c }.join "\n"
 
     expected = (0..255).map { |c| c.chr }.join "\n"
-    expected.sub! '+', ' '
 
     assert_equal expected, actual
   end

data/test/test_bigfile.rb

@@ -1,5 +1,5 @@
 # -*- encoding: binary -*-
-require 'test/setup'
+require './test/setup'
 require 'mogilefs/bigfile'
 
 class TestMogileFS__Bigfile < TestMogileFS
@@ -41,7 +41,7 @@ size 18
 part 1 bytes=12 md5=d3b4d15c294b24d9f853e26095dfe3d0 paths: http://foo1:7500/dev2/0/000/144/0000144411.fid, http://foo2:7500/dev1/0/000/144/0000144411.fid
 part 2 bytes=6 md5=d3b4d15c294b24d9f853e26095dfe3d0 paths: http://foo4:7500/dev2/0/000/144/0000144411.fid, http://foo3:7500/dev1/0/000/144/0000144411.fid
 EOS
-    i = parse_info(s)
+    i = bigfile_parse_info(s)
     assert_equal expect, i
   end
 

data/test/test_db_backend.rb

@@ -1,6 +1,5 @@
 # -*- encoding: binary -*-
-require 'test/setup'
-require 'mogilefs/mysql'
+require './test/setup'
 
 class TestMogileFS__DbBackend < Test::Unit::TestCase
   def setup

data/test/test_fresh.rb

@@ -0,0 +1,8 @@
+# -*- encoding: binary -*-
+require "./test/fresh"
+
+class TestMogFresh < Test::Unit::TestCase
+  include TestFreshSetup
+  alias setup setup_mogilefs
+  alias teardown teardown_mogilefs
+end

data/test/test_http_reader.rb

@@ -0,0 +1,34 @@
+# -*- encoding: binary -*-
+require "./test/setup"
+
+class TestHTTPReader < Test::Unit::TestCase
+  def rsock
+    host = "127.0.0.1"
+    s = TCPServer.new(host, 0)
+    th = Thread.new do
+      c = s.accept
+      c.readpartial(0x666)
+      c.write("HTTP/1.0 200 OK\r\nContent-Length: 666\r\n\r\n666")
+      c.close
+    end
+    path = "http://#{host}:#{s.addr[1]}/"
+    r = MogileFS::HTTPReader.try(path, 666, nil)
+    assert_nil th.value
+    assert_kind_of IO, r
+    r
+    ensure
+      s.close
+  end
+
+  def test_short_to_s
+    r = rsock
+    assert_raises(MogileFS::SizeMismatchError) { r.to_s }
+    r.close
+  end
+
+  def test_short_stream_to
+    r = rsock
+    assert_raises(MogileFS::SizeMismatchError) { r.stream_to("/dev/null") }
+    r.close
+  end
+end

data/test/test_mogilefs.rb

@@ -1,12 +1,10 @@
 # -*- encoding: binary -*-
-require 'test/setup'
+require './test/setup'
 require 'stringio'
 require 'tempfile'
 require 'fileutils'
 
 class TestMogileFS__MogileFS < TestMogileFS
-  include MogileFS::Util
-
   def setup
     @klass = MogileFS::MogileFS
     super
@@ -24,7 +22,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     tmp = Tempfile.new('accept')
     accept = File.open(tmp.path, "ab")
     svr = Proc.new do |serv, port|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
       readed = client.recv(4096, 0)
       assert(readed =~ \
@@ -50,7 +48,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     tmp = Tempfile.new('accept')
     accept = File.open(tmp.path, 'ab')
     svr1 = Proc.new do |serv, port|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
       readed = client.recv(4096, 0)
       assert(readed =~ \
@@ -61,7 +59,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     end
 
     svr2 = Proc.new do |serv, port|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
       readed = client.recv(4096, 0)
       assert(readed =~ \
@@ -97,13 +95,13 @@ class TestMogileFS__MogileFS < TestMogileFS
 
     accept = Tempfile.new('accept')
     svr = Proc.new do |serv, port|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
       accept.syswrite('.')
       readed = client.recv(4096, 0)
       assert(readed =~ \
             %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
-      sysrwloop(tmpfp, client)
+      MogileFS.io.copy_stream(tmpfp, client)
       client.close
       exit 0
     end
@@ -210,40 +208,6 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal 'new_key_2', next_after
   end
 
-  def test_list_keys_block
-    @backend.list_keys = { 'key_count' => '2', 'next_after' => 'new_key_2',
-                           'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
-    http_resp = "HTTP/1.0 200 OK\r\nContent-Length: %u\r\n"
-    srv = Proc.new do |serv, port, size|
-      client, client_addr = serv.accept
-      client.sync = true
-      readed = client.readpartial(4096)
-      assert %r{\AHEAD } =~ readed
-      client.send(http_resp % size, 0)
-      client.close
-    end
-    t1 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
-    t2 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
-    t3 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 10) })
-    @backend.get_paths = { 'paths' => '2',
-                           'path1' => "http://127.0.0.1:#{t1.port}/",
-                           'path2' => "http://127.0.0.1:#{t2.port}/" }
-    @backend.get_paths = { 'paths' => '1',
-                           'path1' => "http://127.0.0.1:#{t3.port}/" }
-
-    res = []
-    keys, next_after = @client.list_keys('new') do |key,length,devcount|
-      res << [ key, length, devcount ]
-    end
-
-    expect_res = [ [ 'new_key_1', 5, 2 ], [ 'new_key_2', 10, 1 ] ]
-    assert_equal expect_res, res
-    assert_equal ['new_key_1', 'new_key_2'], keys.sort
-    assert_equal 'new_key_2', next_after
-    ensure
-      TempServer.destroy_all!
-  end
-
   def test_new_file_http
     @client.readonly = true
     assert_raises MogileFS::ReadOnlyError do
@@ -258,44 +222,6 @@ class TestMogileFS__MogileFS < TestMogileFS
     end
   end
 
-  def test_size_http
-    accept = Tempfile.new('accept')
-    t = TempServer.new(Proc.new do |serv,port|
-      client, client_addr = serv.accept
-      client.sync = true
-      readed = client.recv(4096, 0) rescue nil
-      accept.syswrite('.')
-      assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
-      client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\n", 0)
-      client.close
-    end)
-
-    path = "http://127.0.0.1:#{t.port}/path"
-    @backend.get_paths = { 'paths' => 1, 'path1' => path }
-
-    assert_equal 5, @client.size('key')
-    assert_equal 1, accept.stat.size
-  end
-
-  def test_bad_size_http
-    tmp = Tempfile.new('accept')
-    t = TempServer.new(Proc.new do |serv,port|
-      client, client_addr = serv.accept
-      client.sync = true
-      readed = client.recv(4096, 0) rescue nil
-      assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
-      tmp.syswrite('.')
-      client.send("HTTP/1.0 404 Not Found\r\nContent-Length: 5\r\n\r\n", 0)
-      client.close
-    end)
-
-    path = "http://127.0.0.1:#{t.port}/path"
-    @backend.get_paths = { 'paths' => 1, 'path1' => path }
-
-    assert_nil @client.size('key')
-    assert_equal 1, tmp.stat.size
-  end
-
   def test_store_file_small_http
     received = Tempfile.new('received')
     to_store = Tempfile.new('small')
@@ -303,9 +229,9 @@ class TestMogileFS__MogileFS < TestMogileFS
 
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
     t = TempServer.new(Proc.new do |serv, accept|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
-      received.syswrite(client.recv(4096, 0))
+      received.syswrite(client.read(expected.bytesize))
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
     end)
@@ -327,9 +253,14 @@ class TestMogileFS__MogileFS < TestMogileFS
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
 
     t = TempServer.new(Proc.new do |serv, accept|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
-      received.syswrite(client.recv(4096, 0))
+      seen = ""
+      while seen !~ /\r\n\r\ndata/
+        buf = client.readpartial(4096)
+        seen << buf
+        received.syswrite(buf)
+      end
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
     end)
@@ -357,7 +288,7 @@ class TestMogileFS__MogileFS < TestMogileFS
       expected += "data"
     end
     t = TempServer.new(Proc.new do |serv, accept|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
       nr = 0
       loop do
@@ -391,23 +322,35 @@ class TestMogileFS__MogileFS < TestMogileFS
       TempServer.destroy_all!
   end
 
-  def test_store_content_multi_dest_failover
+  def test_store_content_multi_dest_failover_path
+    test_store_content_multi_dest_failover(true)
+  end
+
+  def test_store_content_multi_dest_failover(big_io = false)
     received1 = Tempfile.new('received')
     received2 = Tempfile.new('received')
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
 
     t1 = TempServer.new(Proc.new do |serv, accept|
-      client, client_addr = serv.accept
-      client.sync = true
-      received1.syswrite(client.recv(4096, 0))
+      client, _ = serv.accept
+      seen = ""
+      while seen !~ /\r\n\r\ndata/
+        buf = client.readpartial(4096)
+        seen << buf
+        received1.syswrite(buf)
+      end
       client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
       client.close
     end)
 
     t2 = TempServer.new(Proc.new do |serv, accept|
-      client, client_addr = serv.accept
-      client.sync = true
-      received2.syswrite(client.recv(4096, 0))
+      client, _ = serv.accept
+      seen = ""
+      while seen !~ /\r\n\r\ndata/
+        buf = client.readpartial(4096)
+        seen << buf
+        received2.syswrite(buf)
+      end
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
     end)
@@ -420,7 +363,15 @@ class TestMogileFS__MogileFS < TestMogileFS
       'path_2' => "http://127.0.0.1:#{t2.port}/path",
     }
 
-    nr = @client.store_content 'new_key', 'test', 'data'
+    if big_io
+      tmp = Tempfile.new('data')
+      tmp.sync = true
+      tmp.write 'data'
+      nr = @client.store_file('new_key', 'test', tmp.path)
+      tmp.close!
+    else
+      nr = @client.store_content 'new_key', 'test', 'data'
+    end
     assert_equal 4, nr
     received1.sysseek(0)
     received2.sysseek(0)
@@ -432,7 +383,7 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_store_content_http_fail
     t = TempServer.new(Proc.new do |serv, accept|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
       client.recv(4096, 0)
       client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
@@ -444,7 +395,7 @@ class TestMogileFS__MogileFS < TestMogileFS
       'path' => "http://127.0.0.1:#{t.port}/path",
     }
 
-    assert_raises MogileFS::HTTPFile::BadResponseError do
+    assert_raises MogileFS::HTTPFile::NoStorageNodesError do
       @client.store_content 'new_key', 'test', 'data'
     end
   end
@@ -453,7 +404,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     received = Tempfile.new('received')
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
     t = TempServer.new(Proc.new do |serv, accept|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
       received.syswrite(client.recv(4096, 0))
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
@@ -503,7 +454,7 @@ class TestMogileFS__MogileFS < TestMogileFS
 
     readed = Tempfile.new('readed')
     t = TempServer.new(Proc.new do |serv, accept|
-      client, client_addr = serv.accept
+      client, _ = serv.accept
       client.sync = true
       nr = 0
       loop do
@@ -525,7 +476,7 @@ class TestMogileFS__MogileFS < TestMogileFS
 
     orig_size = to_put.size
     nr = @client.store_file('new_key', 'test', to_put.path)
-    assert nr
+    assert nr, nr.inspect
     assert_equal orig_size, nr
     assert_equal orig_size, to_put.size
     readed.sysseek(0)
@@ -592,6 +543,235 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal 'readonly mogilefs', e.message
   end
 
+  def assert_get_paths_args(expect, *args)
+    sock = TCPServer.new("127.0.0.1", 0)
+    nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
+    c = MogileFS::MogileFS.new(nargs)
+    received = []
+    th = Thread.new do
+      a = sock.accept
+      line = a.gets
+      received << line
+      a.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
+      a.close
+    end
+    paths_expect = %w(http://0/a http://0/b)
+    assert_equal paths_expect, c.get_paths("f", *args)
+    th.join
+    assert_equal 1, received.size
+    tmp = c.backend.url_decode(received[0].split(/\s+/)[1])
+    assert_equal "f", tmp.delete("key")
+    assert_equal "foo", tmp.delete("domain")
+    assert_equal expect, tmp
+    c.backend.shutdown
+    ensure
+      sock.close
+  end
+
+  def test_get_paths_args
+    assert_get_paths_args({"noverify"=>"1", "zone"=>""})
+    assert_get_paths_args({"noverify"=>"0", "zone"=>""}, false)
+    assert_get_paths_args({"noverify"=>"0", "zone"=>""}, :noverify=>false)
+    assert_get_paths_args({"noverify"=>"1", "zone"=>"alt"}, true, "alt")
+    assert_get_paths_args({"noverify"=>"1", "zone"=>"alt"},
+                          {:noverify => true, :zone => "alt"})
+    assert_get_paths_args({"noverify"=>"1", "zone"=>"alt","pathcount"=>"666"},
+                          {:noverify => true, :zone => "alt", :pathcount=>666})
+  end
+
+  def test_idempotent_command_eof
+    ip = "127.0.0.1"
+    a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
+    hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
+    args = { :hosts => hosts, :domain => "foo" }
+    c = MogileFS::MogileFS.new(args)
+    received = []
+    th = Thread.new do
+      r = IO.select([a, b])
+      x = r[0][0].accept
+      received << x.gets
+      x.close
+
+      r = IO.select([a, b])
+      x = r[0][0].accept
+      received << x.gets
+      x.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
+      x.close
+    end
+    expect = %w(http://0/a http://0/b)
+    assert_equal expect, c.get_paths("f")
+    th.join
+    assert_equal 2, received.size
+    assert_equal received[0], received[1]
+  end
+
+  def test_idempotent_command_response_truncated
+    ip = "127.0.0.1"
+    a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
+    hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
+    args = { :hosts => hosts, :domain => "foo" }
+    c = MogileFS::MogileFS.new(args)
+    received = []
+    th = Thread.new do
+      r = IO.select([a, b])
+      x = r[0][0].accept
+      received << x.gets
+      x.write("OK paths=2&path1=http://0/a&path2=http://0/")
+      x.close
+
+      r = IO.select([a, b])
+      x = r[0][0].accept
+      received << x.gets
+      x.write("OK paths=2&path1=http://0/a&path2=http://0/b\r\n")
+      x.close
+    end
+    expect = %w(http://0/a http://0/b)
+    assert_equal expect, c.get_paths("f")
+    th.join
+    assert_equal 2, received.size
+    assert_equal received[0], received[1]
+  end
+
+  def test_non_idempotent_command_eof
+    ip = "127.0.0.1"
+    a, b = TCPServer.new(ip, 0), TCPServer.new(ip, 0)
+    hosts = [ "#{ip}:#{a.addr[1]}", "#{ip}:#{b.addr[1]}" ]
+    args = { :hosts => hosts, :domain => "foo" }
+    c = MogileFS::MogileFS.new(args)
+    received = []
+    th = Thread.new do
+      r = IO.select([a, b])
+      x = r[0][0].accept
+      received << x.gets
+      x.close
+    end
+    assert_raises(EOFError) { c.rename("a", "b") }
+    th.join
+    assert_equal 1, received.size
+  end
+
+  def test_list_keys_verbose_ordering # implementation detail
+    received = []
+    sock = TCPServer.new("127.0.0.1", 0)
+    nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
+    c = MogileFS::MogileFS.new(nargs)
+    th = Thread.new do
+      a = sock.accept
+      %w(a b c d e).each do |key|
+        line = a.gets
+        cmd, args = line.split(/\s+/, 2)
+        args = c.backend.url_decode(args.strip)
+        assert_equal "file_info", cmd
+        assert_equal key, args["key"]
+      end
+      out = { "length" => 3, "devcount" => 6 }
+      %w(a b c d e).shuffle.each do |key|
+        out["key"] = key
+        a.write "OK #{c.backend.url_encode(out)}\r\n"
+      end
+      a.close
+    end
+
+    blk = lambda do |key, length, devcount|
+      received << [ key, length, devcount ]
+    end
+    c.list_keys_verbose(%w(a b c d e), blk)
+    th.join
+    received.map! { |(key,_,_)| key }
+    assert_equal %w(a b c d e), received
+    ensure
+      sock.close
+  end
+
+  def test_list_keys_verbose_retry_eof # implementation detail
+    received = []
+    sock = TCPServer.new("127.0.0.1", 0)
+    nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
+    c = MogileFS::MogileFS.new(nargs)
+    th = Thread.new do
+      a = sock.accept
+      %w(a b c d e).each do |key|
+        line = a.gets
+        cmd, args = line.split(/\s+/, 2)
+        args = c.backend.url_decode(args.strip)
+        assert_equal "file_info", cmd
+        assert_equal key, args["key"]
+      end
+      out = { "length" => 3, "devcount" => 6 }
+      %w(d e).each do |key|
+        out["key"] = key
+        a.write "OK #{c.backend.url_encode(out)}\r\n"
+      end
+      a.close # trigger EOF
+      a = sock.accept # client will retry
+      %w(a b c).each do |key|
+        line = a.gets
+        cmd, args = line.split(/\s+/, 2)
+        args = c.backend.url_decode(args.strip)
+        assert_equal "file_info", cmd
+        assert_equal key, args["key"]
+        out["key"] = key
+        a.write "OK #{c.backend.url_encode(out)}\r\n"
+      end
+      a.close
+    end
+
+    blk = lambda do |key, length, devcount|
+      received << [ key, length, devcount ]
+    end
+    c.list_keys_verbose(%w(a b c d e), blk)
+    th.join
+    received.map! { |(key,_,_)| key }
+    assert_equal %w(a b c d e), received
+    ensure
+      sock.close
+  end
+
+  def test_list_keys_verbose_retry_truncated # implementation detail
+    received = []
+    sock = TCPServer.new("127.0.0.1", 0)
+    nargs = { :hosts => [ "127.0.0.1:#{sock.addr[1]}" ], :domain => "foo" }
+    c = MogileFS::MogileFS.new(nargs)
+    th = Thread.new do
+      a = sock.accept
+      %w(a b c d e).each do |key|
+        line = a.gets
+        cmd, args = line.split(/\s+/, 2)
+        args = c.backend.url_decode(args.strip)
+        assert_equal "file_info", cmd
+        assert_equal key, args["key"]
+      end
+      out = { "length" => 3, "devcount" => 6 }
+      out["key"] = "a"
+      a.write "OK #{c.backend.url_encode(out)}\r\n"
+      out["key"] = "b"
+      a.write "OK #{c.backend.url_encode(out)}"
+      a.close # trigger EOF
+
+      a = sock.accept # client will retry
+      %w(b c d e).each do |key|
+        line = a.gets
+        cmd, args = line.split(/\s+/, 2)
+        args = c.backend.url_decode(args.strip)
+        assert_equal "file_info", cmd
+        assert_equal key, args["key"]
+        out["key"] = key
+        a.write "OK #{c.backend.url_encode(out)}\r\n"
+      end
+      a.close
+    end
+
+    blk = lambda do |key, length, devcount|
+      received << [ key, length, devcount ]
+    end
+    c.list_keys_verbose(%w(a b c d e), blk)
+    th.join
+    received.map! { |(key,_,_)| key }
+    assert_equal %w(a b c d e), received
+    ensure
+      sock.close
+  end
+
   def test_sleep
     @backend.sleep = {}
     assert_nothing_raised do