mogilefs-client 1.3.1 → 2.0.0

data/test/test_admin.rb

@@ -19,7 +19,7 @@ class TestMogileFS__Admin < TestMogileFS
            "host1_status"=>"alive",
            "host1_altmask"=>""}
     actual = @client.clean 'hosts', 'host', res
-  
+
     expected = [{"status"=>"alive",
                  "http_get_port"=>"",
                  "http_port"=>"",
@@ -119,7 +119,7 @@ class TestMogileFS__Admin < TestMogileFS
       'file' => [{ 'files' => '2', 'class' => 'normal', 'domain' => 'test' }]
     }
 
-    assert_equal 
+    assert_equal
   end
 
   def test_get_stats_fids

data/test/test_backend.rb

@@ -16,8 +16,6 @@ end
 class TestBackend < Test::Unit::TestCase
 
   def setup
-    TCPSocket.connections = []
-    TCPSocket.sockets = []
     @backend = MogileFS::Backend.new :hosts => ['localhost:1']
   end
 
@@ -37,17 +35,21 @@ class TestBackend < Test::Unit::TestCase
   end
 
   def test_do_request
-    socket_request = ''
-    socket = Object.new
-    def socket.closed?() false end
-    def socket.send(request, flags) return request.length end
-    def @backend.select(*args) return [true] end
-    def socket.gets() return 'OK 1 you=win' end
+    received = ''
+    tmp = TempServer.new(Proc.new do |serv, port|
+      client, client_addr = serv.accept
+      client.sync = true
+      received = client.recv 4096
+      client.send "OK 1 you=win\r\n", 0
+    end)
 
-    @backend.instance_variable_set '@socket', socket
+    @backend.hosts = "127.0.0.1:#{tmp.port}"
 
     assert_equal({'you' => 'win'},
                  @backend.do_request('go!', { 'fight' => 'team fight!' }))
+    assert_equal "go! fight=team+fight%21\r\n", received
+    ensure
+      TempServer.destroy_all!
   end
 
   def test_do_request_send_error
@@ -58,13 +60,25 @@ class TestBackend < Test::Unit::TestCase
 
     @backend.instance_variable_set '@socket', socket
 
-    assert_raises RuntimeError do
+    assert_raises MogileFS::UnreachableBackendError do
       @backend.do_request 'go!', { 'fight' => 'team fight!' }
     end
 
     assert_equal nil, @backend.instance_variable_get('@socket')
   end
 
+  def test_automatic_exception
+    assert ! MogileFS::Backend.const_defined?('PebkacError')
+    assert @backend.error('pebkac')
+    assert_equal MogileFS::Error, @backend.error('PebkacError').superclass
+    assert MogileFS::Backend.const_defined?('PebkacError')
+
+    assert ! MogileFS::Backend.const_defined?('PebKacError')
+    assert @backend.error('peb_kac')
+    assert_equal MogileFS::Error, @backend.error('PebKacError').superclass
+    assert MogileFS::Backend.const_defined?('PebKacError')
+  end
+
   def test_do_request_truncated
     socket_request = ''
     socket = Object.new
@@ -73,7 +87,7 @@ class TestBackend < Test::Unit::TestCase
 
     @backend.instance_variable_set '@socket', socket
 
-    assert_raises RuntimeError do
+    assert_raises MogileFS::RequestTruncatedError do
       @backend.do_request 'go!', { 'fight' => 'team fight!' }
     end
   end
@@ -86,70 +100,118 @@ class TestBackend < Test::Unit::TestCase
   def test_parse_response
     assert_equal({'foo' => 'bar', 'baz' => 'hoge'},
                  @backend.parse_response('OK 1 foo=bar&baz=hoge'))
-    assert_equal nil, @backend.parse_response('ERR you totally suck')
+
+    err = nil
+    begin
+      @backend.parse_response('ERR you totally suck')
+    rescue MogileFS::Error => err
+      assert_equal 'MogileFS::Backend::YouError', err.class.to_s
+    end
+    assert_equal 'MogileFS::Backend::YouError', err.class.to_s
+
     assert_equal 'you', @backend.lasterr
     assert_equal 'totally suck', @backend.lasterrstr
 
-    assert_raises RuntimeError do
+    assert_raises MogileFS::InvalidResponseError do
       @backend.parse_response 'garbage'
     end
   end
 
   def test_readable_eh_readable
-    socket = Object.new
-    def socket.closed?() false end
-    def @backend.select(*args) return [true] end
-    @backend.instance_variable_set '@socket', socket
-
+    accept_nr = 0
+    tmp = TempServer.new(Proc.new do |serv, port|
+      client, client_addr = serv.accept
+      client.sync = true
+      accept_nr += 1
+      client.send('.', 0)
+      sleep
+    end)
+
+    @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{tmp.port}" ]
     assert_equal true, @backend.readable?
+    assert_equal 1, accept_nr
+    ensure
+      TempServer.destroy_all!
   end
 
   def test_readable_eh_not_readable
-    socket = FakeSocket.new
-    def socket.closed?() false end
-    def @backend.select(*args) return [] end
-    @backend.instance_variable_set '@socket', socket
-
+    tmp = TempServer.new(Proc.new { |a,b| sleep })
+    @backend = MogileFS::Backend.new(:hosts => [ "127.0.0.1:#{tmp.port}" ],
+                                     :timeout => 0.5)
     begin
       @backend.readable?
     rescue MogileFS::UnreadableSocketError => e
-      assert_equal '127.0.0.1:6001 never became readable', e.message
-    rescue Exception
-      flunk "MogileFS::UnreadableSocketError not raised"
+      assert_equal "127.0.0.1:#{tmp.port} never became readable", e.message
+    rescue Exception => err
+      flunk "MogileFS::UnreadableSocketError not raised #{err} #{err.backtrace}"
     else
       flunk "MogileFS::UnreadableSocketError not raised"
+    ensure
+      TempServer.destroy_all!
     end
   end
 
   def test_socket
     assert_equal({}, @backend.dead)
-    assert_raises RuntimeError do @backend.socket end
+    assert_raises MogileFS::UnreachableBackendError do @backend.socket end
     assert_equal(['localhost:1'], @backend.dead.keys)
   end
 
   def test_socket_robust
-    @backend.hosts = ['localhost:6001', 'localhost:6002']
-    def @backend.connect_to(host, port)
-      @first = (defined? @first) ? false : true
-      raise Errno::ECONNREFUSED if @first
+    bad_accept_nr = accept_nr = 0
+    queue = Queue.new
+    bad = Proc.new { |serv,port| sleep; bad_accept_nr += 1 }
+    good = Proc.new do |serv,port|
+      client, client_addr = serv.accept
+      client.sync = true
+      accept_nr += 1
+      client.send '.', 0
+      client.flush
+      queue.push true
+      sleep
     end
-
-    assert_equal({}, @backend.dead)
-    @backend.socket
-    assert_equal false, @backend.dead.keys.empty?
+    nr = 10
+
+    nr.times do
+      begin
+        t1 = TempServer.new(bad)
+        t2 = TempServer.new(good)
+        hosts = ["0:#{t1.port}", "0:#{t2.port}"]
+        @backend = MogileFS::Backend.new(:hosts => hosts)
+        assert_equal({}, @backend.dead)
+        t1.destroy!
+        @backend.socket
+        wait = queue.pop
+      ensure
+        TempServer.destroy_all!
+      end
+    end # nr.times
+    assert_equal 0, bad_accept_nr
+    assert_equal nr, accept_nr
   end
 
   def test_shutdown
-    fake_socket = FakeSocket.new
-    @backend.socket = fake_socket
-    assert_equal fake_socket, @backend.socket
+    accept_nr = 0
+    tmp = TempServer.new(Proc.new do |serv,port|
+      client, client_addr = serv.accept
+      accept_nr += 1
+      sleep
+    end)
+    @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{tmp.port}" ]
+    assert @backend.socket
+    assert ! @backend.socket.closed?
     @backend.shutdown
     assert_equal nil, @backend.instance_variable_get(:@socket)
+    assert_equal 1, accept_nr
+
+    ensure
+      TempServer.destroy_all!
   end
 
   def test_url_decode
     assert_equal({"\272z" => "\360opy", "f\000" => "\272r"},
                  @backend.url_decode("%baz=%f0opy&f%00=%bar"))
+    assert_equal({}, @backend.url_decode(''))
   end
 
   def test_url_encode

data/test/test_bigfile.rb

@@ -0,0 +1,48 @@
+require 'test/setup'
+require 'mogilefs/bigfile'
+
+class TestMogileFS__Bigfile < TestMogileFS
+  include MogileFS::Bigfile
+
+  def setup
+    @klass = MogileFS::MogileFS
+    super
+  end
+
+  def test_parser
+    expect = {:type=>"file",
+     :des=>"no description",
+     :chunks=>2,
+     :parts=>
+      [nil,
+       {:md5=>"d3b4d15c294b24d9f853e26095dfe3d0",
+        :paths=>
+         ["http://foo1:7500/dev2/0/000/144/0000144411.fid",
+          "http://foo2:7500/dev1/0/000/144/0000144411.fid"],
+        :bytes=>12},
+       {:md5=>"d3b4d15c294b24d9f853e26095dfe3d0",
+        :paths=>
+         ["http://foo4:7500/dev2/0/000/144/0000144411.fid",
+          "http://foo3:7500/dev1/0/000/144/0000144411.fid"],
+        :bytes=>6}],
+     :size=>18,
+     :filename=>"foo.tar",
+     :compressed=>false}
+
+    s = <<EOS
+des no description
+type file
+compressed 0
+filename foo.tar
+chunks 2
+size 18
+
+part 1 bytes=12 md5=d3b4d15c294b24d9f853e26095dfe3d0 paths: http://foo1:7500/dev2/0/000/144/0000144411.fid, http://foo2:7500/dev1/0/000/144/0000144411.fid
+part 2 bytes=6 md5=d3b4d15c294b24d9f853e26095dfe3d0 paths: http://foo4:7500/dev2/0/000/144/0000144411.fid, http://foo3:7500/dev1/0/000/144/0000144411.fid
+EOS
+    i = parse_info(s)
+    assert_equal expect, i
+  end
+
+end
+

data/test/test_client.rb

@@ -1,8 +1,13 @@
 require 'test/unit'
+require 'mogilefs'
 
-$TESTING = true
+class MogileFS::Backend
+  attr_accessor :timeout, :lasterr, :lasterrstr, :hosts
+end
 
-require 'mogilefs'
+class MogileFS::Client
+  attr_accessor :hosts
+end
 
 class TestClient < Test::Unit::TestCase
 

data/test/test_db_backend.rb

@@ -0,0 +1,73 @@
+require 'test/setup'
+require 'mogilefs/mysql'
+
+class TestMogileFS__DbBackend < Test::Unit::TestCase
+  def setup
+    @my = FakeMysql.new
+    @mgmy = MogileFS::Mysql.new(:mysql => @my)
+    @mg = MogileFS::MogileFS.new(:db_backend => @mgmy, :domain => 'test')
+  end
+
+  def test_initialize
+    assert_equal 'test', @mg.domain
+    assert @mg.readonly?
+  end
+
+  def test_list_keys_block
+    expect_full = [ [ 'foo', 123, 2 ], [ 'bar', 456, 1 ] ]
+    expect_keys = [ [ 'foo', 'bar' ], 'bar' ]
+    @my.expect << expect_full
+    full = []
+    keys = @mg.list_keys('test') do |dkey,length,devcount|
+      full << [ dkey, length, devcount ]
+    end
+    assert_equal expect_keys, keys
+    assert_equal expect_full, full
+  end
+
+  def test_list_keys
+    expect_full = [ [ 'foo', 123, 2 ], [ 'bar', 456, 1 ] ]
+    expect_keys = [ [ 'foo', 'bar' ], 'bar' ]
+    @my.expect << expect_full
+    keys = @mg.list_keys('test')
+    assert_equal expect_keys, keys
+  end
+
+  def test_size
+    @my.expect << [ [ '123' ] ]
+    assert_equal 123, @mg.size('foo')
+
+    @my.expect << [ [ '456' ] ]
+    assert_equal 456, @mg.size('foo')
+  end
+
+  def test_store_file_readonly
+    assert_raises(MogileFS::ReadOnlyError) do
+      @mg.store_file 'new_key', 'test', '/dev/null'
+    end
+  end
+
+  def test_store_content_readonly
+    assert_raises(MogileFS::ReadOnlyError) do
+      @mg.store_content 'new_key', 'test', 'data'
+    end
+  end
+
+  def test_new_file_readonly
+    assert_raises(MogileFS::ReadOnlyError) { @mg.new_file 'new_key', 'test' }
+  end
+
+  def test_rename_readonly
+    assert_raises(MogileFS::ReadOnlyError) { @mg.rename 'a', 'b' }
+  end
+
+  def test_delete_readonly
+    assert_raises(MogileFS::ReadOnlyError) { @mg.delete 'no_such_key' }
+  end
+
+  def test_sleep
+    assert_nothing_raised { assert_equal({}, @mg.sleep(1)) }
+  end
+
+end
+

data/test/test_mogilefs.rb

@@ -3,24 +3,8 @@ require 'stringio'
 require 'tempfile'
 require 'fileutils'
 
-class URI::HTTP
-
-  class << self
-    attr_accessor :read_data
-    attr_accessor :open_data
-  end
-
-  def read
-    self.class.read_data.shift
-  end
-
-  def open(&block)
-    yield self.class.open_data
-  end
-
-end
-
 class TestMogileFS__MogileFS < TestMogileFS
+  include MogileFS::Util
 
   def setup
     @klass = MogileFS::MogileFS
@@ -29,27 +13,76 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_initialize
     assert_equal 'test', @client.domain
-    assert_equal @root, @client.root
 
     assert_raises ArgumentError do
-      MogileFS::MogileFS.new :hosts => ['kaa:6001'], :root => '/mogilefs/test'
+      MogileFS::MogileFS.new :hosts => ['kaa:6001']
     end
   end
 
   def test_get_file_data_http
-    URI::HTTP.read_data = %w[data!]
+    accept_nr = 0
+    svr = Proc.new do |serv, port|
+      client, client_addr = serv.accept
+      client.sync = true
+      readed = client.recv(4096, 0)
+      assert(readed =~ \
+            %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
+      accept_nr += 1
+      client.close
+    end
+    t1 = TempServer.new(svr)
+    t2 = TempServer.new(svr)
+    path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
+    path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
 
-    path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
-    path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
+    @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
+
+    assert_equal 'data!', @client.get_file_data('key')
+    assert_equal 1, accept_nr
+    ensure
+      TempServer.destroy_all!
+  end
 
+  def test_get_file_data_http_not_found_failover
+    accept_nr = 0
+    svr1 = Proc.new do |serv, port|
+      client, client_addr = serv.accept
+      client.sync = true
+      readed = client.recv(4096, 0)
+      assert(readed =~ \
+            %r{\AGET /dev1/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      client.send("HTTP/1.0 404 Not Found\r\n\r\ndata!", 0)
+      accept_nr += 1
+      client.close
+    end
+
+    svr2 = Proc.new do |serv, port|
+      client, client_addr = serv.accept
+      client.sync = true
+      readed = client.recv(4096, 0)
+      assert(readed =~ \
+            %r{\AGET /dev2/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
+      accept_nr += 1
+      client.close
+    end
+
+    t1 = TempServer.new(svr1)
+    t2 = TempServer.new(svr2)
+    path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
+    path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
     @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
 
     assert_equal 'data!', @client.get_file_data('key')
+    assert_equal 2, accept_nr
+    ensure
+      TempServer.destroy_all!
   end
 
   def test_get_file_data_http_block
     tmpfp = Tempfile.new('test_mogilefs.open_data')
-    nr = 100 # tested with 1000
+    nr = nr_chunks
     chunk_size = 1024 * 1024
     expect_size = nr * chunk_size
     header = "HTTP/1.0 200 OK\r\n" \
@@ -58,18 +91,29 @@ class TestMogileFS__MogileFS < TestMogileFS
     nr.times { assert_equal chunk_size, tmpfp.syswrite(' ' * chunk_size) }
     assert_equal expect_size + header.size, File.size(tmpfp.path)
     tmpfp.sysseek(0)
-    socket = FakeSocket.new(tmpfp)
-    TCPSocket.sockets << socket
 
-    path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
-    path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
+    accept_nr = 0
+    svr = Proc.new do |serv, port|
+      client, client_addr = serv.accept
+      client.sync = true
+      accept_nr += 1
+      readed = client.recv(4096, 0)
+      assert(readed =~ \
+            %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      sysrwloop(tmpfp, client)
+      client.close
+    end
+    t1 = TempServer.new(svr)
+    t2 = TempServer.new(svr)
+    path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
+    path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
 
     @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
 
     data = Tempfile.new('test_mogilefs.dest_data')
+    read_nr = nr = 0
     @client.get_file_data('key') do |fp|
       buf = ''
-      read_nr = nr = 0
       loop do
         begin
           fp.sysread(16384, buf)
@@ -80,17 +124,18 @@ class TestMogileFS__MogileFS < TestMogileFS
           break
         end
       end
-      assert_equal expect_size, nr, "size mismatch"
     end
+    assert_equal expect_size, nr, "size mismatch"
+    assert_equal 1, accept_nr
   end
 
   def test_get_paths
-    path1 = 'rur-1/dev1/0/000/000/0000000062.fid'
-    path2 = 'rur-2/dev2/0/000/000/0000000062.fid'
+    path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
+    path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
 
     @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
 
-    expected = ["#{@root}/#{path1}", "#{@root}/#{path2}"]
+    expected = [ path1, path2 ]
 
     assert_equal expected, @client.get_paths('key').sort
   end
@@ -98,7 +143,9 @@ class TestMogileFS__MogileFS < TestMogileFS
   def test_get_paths_unknown_key
     @backend.get_paths = ['unknown_key', '']
 
-    assert_equal nil, @client.get_paths('key')
+    assert_raises MogileFS::Backend::UnknownKeyError do
+      assert_equal nil, @client.get_paths('key')
+    end
   end
 
   def test_delete_existing
@@ -110,14 +157,14 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_delete_nonexisting
     @backend.delete = 'unknown_key', ''
-    assert_nothing_raised do
-      assert_equal nil, @client.delete('no_such_key')
+    assert_raises MogileFS::Backend::UnknownKeyError do
+      @client.delete('no_such_key')
     end
   end
 
   def test_delete_readonly
     @client.readonly = true
-    assert_raises RuntimeError do
+    assert_raises MogileFS::ReadOnlyError do
       @client.delete 'no_such_key'
     end
   end
@@ -145,99 +192,188 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal 'new_key_2', next_after
   end
 
+  def test_list_keys_block
+    @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_2',
+                           'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
+    http_resp = "HTTP/1.0 200 OK\r\nContent-Length: %u\r\n"
+    srv = Proc.new do |serv, port, size|
+      client, client_addr = serv.accept
+      client.sync = true
+      readed = client.readpartial(4096)
+      assert %r{\AHEAD } =~ readed
+      client.send(http_resp % size, 0)
+      client.close
+    end
+    t1 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
+    t2 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
+    t3 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 10) })
+    @backend.get_paths = { 'paths' => 2,
+                           'path1' => "http://127.0.0.1:#{t1.port}/",
+                           'path2' => "http://127.0.0.1:#{t2.port}/" }
+    @backend.get_paths = { 'paths' => 1,
+                           'path1' => "http://127.0.0.1:#{t3.port}/" }
+
+    res = []
+    keys, next_after = @client.list_keys('new') do |key,length,devcount|
+      res << [ key, length, devcount ]
+    end
+
+    expect_res = [ [ 'new_key_1', 5, 2 ], [ 'new_key_2', 10, 1 ] ]
+    assert_equal expect_res, res
+    assert_equal ['new_key_1', 'new_key_2'], keys.sort
+    assert_equal 'new_key_2', next_after
+    ensure
+      TempServer.destroy_all!
+  end
+
   def test_new_file_http
     @client.readonly = true
-    assert_raises RuntimeError do
+    assert_raises MogileFS::ReadOnlyError do
       @client.new_file 'new_key', 'test'
     end
   end
 
   def test_new_file_readonly
     @client.readonly = true
-    assert_raises RuntimeError do
+    assert_raises MogileFS::ReadOnlyError do
       @client.new_file 'new_key', 'test'
     end
   end
 
   def test_size_http
-    socket = FakeSocket.new <<-EOF
-HTTP/1.0 200 OK\r
-Content-Length: 5\r
-    EOF
-
-    TCPSocket.sockets << socket
-
-    path = 'http://example.com/path'
-
+    accept_nr = 0
+    t = TempServer.new(Proc.new do |serv,port|
+      client, client_addr = serv.accept
+      client.sync = true
+      readed = client.recv(4096, 0) rescue nil
+      assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
+      client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\n", 0)
+      accept_nr += 1
+      client.close
+    end)
+
+    path = "http://127.0.0.1:#{t.port}/path"
     @backend.get_paths = { 'paths' => 1, 'path1' => path }
 
     assert_equal 5, @client.size('key')
-
-    socket.write_s.rewind
-
-    assert_equal "HEAD /path HTTP/1.1\r\n", socket.write_s.gets
-
-    assert_equal ['example.com', 80], TCPSocket.connections.shift
-    assert_empty TCPSocket.connections
+    assert_equal 1, accept_nr
   end
 
-  def test_size_nfs
-    path = File.join @root, 'path'
-
-    File.open path, 'w' do |fp| fp.write 'data!' end
-
-    @backend.get_paths = { 'paths' => 1, 'path1' => 'path' }
+  def test_bad_size_http
+    accept_nr = 0
+    t = TempServer.new(Proc.new do |serv,port|
+      client, client_addr = serv.accept
+      client.sync = true
+      readed = client.recv(4096, 0) rescue nil
+      assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
+      client.send("HTTP/1.0 404 Not Found\r\nContent-Length: 5\r\n\r\n", 0)
+      accept_nr += 1
+      client.close
+    end)
+
+    path = "http://127.0.0.1:#{t.port}/path"
+    @backend.get_paths = { 'paths' => 1, 'path1' => path }
 
-    assert_equal 5, @client.size('key')
+    assert_nil @client.size('key')
+    assert_equal 1, accept_nr
   end
 
   def test_store_content_http
-    socket = FakeSocket.new 'HTTP/1.0 200 OK'
+    received = ''
+    expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
 
-    TCPSocket.sockets << socket
+    t = TempServer.new(Proc.new do |serv, accept|
+      client, client_addr = serv.accept
+      client.sync = true
+      received = client.recv(4096, 0)
+      client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
+      client.close
+    end)
 
     @backend.create_open = {
       'devid' => '1',
-      'path' => 'http://example.com/path',
+      'path' => "http://127.0.0.1:#{t.port}/path",
     }
 
     @client.store_content 'new_key', 'test', 'data'
 
-    expected = <<-EOF.chomp
-PUT /path HTTP/1.0\r
-Content-Length: 4\r
-\r
-data
-    EOF
+    assert_equal expected, received
+    ensure
+      TempServer.destroy_all!
+  end
 
-    assert_equal expected, socket.write_s.string
+  def test_store_content_multi_dest_failover
+    received1 = received2 = nil
+    expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
+
+    t1 = TempServer.new(Proc.new do |serv, accept|
+      client, client_addr = serv.accept
+      client.sync = true
+      received1 = client.recv(4096, 0)
+      client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
+      client.close
+    end)
+
+    t2 = TempServer.new(Proc.new do |serv, accept|
+      client, client_addr = serv.accept
+      client.sync = true
+      received2 = client.recv(4096, 0)
+      client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
+      client.close
+    end)
 
-    assert_equal ['example.com', 80], TCPSocket.connections.shift
-    assert_empty TCPSocket.connections
-  end
+    @backend.create_open = {
+      'dev_count' => '2',
+      'devid_1' => '1',
+      'path_1' => "http://127.0.0.1:#{t1.port}/path",
+      'devid_2' => '2',
+      'path_2' => "http://127.0.0.1:#{t2.port}/path",
+    }
 
-  def test_store_content_http_empty
-    socket = FakeSocket.new 'HTTP/1.0 200 OK'
+    @client.store_content 'new_key', 'test', 'data'
+    assert_equal expected, received1
+    assert_equal expected, received2
+    ensure
+      TempServer.destroy_all!
+  end
 
-    TCPSocket.sockets << socket
+  def test_store_content_http_fail
+    t = TempServer.new(Proc.new do |serv, accept|
+      client, client_addr = serv.accept
+      client.sync = true
+      client.recv(4096, 0)
+      client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
+      client.close
+    end)
 
     @backend.create_open = {
       'devid' => '1',
-      'path' => 'http://example.com/path',
+      'path' => "http://127.0.0.1:#{t.port}/path",
     }
 
-    @client.store_content 'new_key', 'test', ''
+    assert_raises MogileFS::HTTPFile::BadResponseError do
+      @client.store_content 'new_key', 'test', 'data'
+    end
+  end
 
-    expected = <<-EOF
-PUT /path HTTP/1.0\r
-Content-Length: 0\r
-\r
-    EOF
+  def test_store_content_http_empty
+    received = ''
+    expected = "PUT /path HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
+    t = TempServer.new(Proc.new do |serv, accept|
+      client, client_addr = serv.accept
+      client.sync = true
+      received = client.recv(4096, 0)
+      client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
+      client.close
+    end)
 
-    assert_equal expected, socket.write_s.string
+    @backend.create_open = {
+      'devid' => '1',
+      'path' => "http://127.0.0.1:#{t.port}/path",
+    }
 
-    assert_equal ['example.com', 80], TCPSocket.connections.shift
-    assert_empty TCPSocket.connections
+    @client.store_content 'new_key', 'test', ''
+    assert_equal expected, received
   end
 
   def test_store_content_nfs
@@ -246,41 +382,76 @@ Content-Length: 0\r
       'devid_1' => '1',
       'path_1' => '/path',
     }
+    assert_raises MogileFS::UnsupportedPathError do
+      @client.store_content 'new_key', 'test', 'data'
+    end
+  end
 
-    @client.store_content 'new_key', 'test', 'data'
+  def test_new_file_http_large
+    expect = Tempfile.new('test_mogilefs.expect')
+    to_put = Tempfile.new('test_mogilefs.to_put')
+    received = Tempfile.new('test_mogilefs.received')
 
-    dest_file = File.join(@root, 'path')
+    nr = nr_chunks
+    chunk_size = 1024 * 1024
+    expect_size = nr * chunk_size
 
-    assert File.exist?(dest_file)
-    assert_equal 'data', File.read(dest_file)
-  end
+    header = "PUT /path HTTP/1.0\r\n" \
+             "Content-Length: #{expect_size}\r\n\r\n"
+    assert_equal header.size, expect.syswrite(header)
+    nr.times do
+      assert_equal chunk_size, expect.syswrite(' ' * chunk_size)
+      assert_equal chunk_size, to_put.syswrite(' ' * chunk_size)
+    end
+    assert_equal expect_size + header.size, expect.stat.size
+    assert_equal expect_size, to_put.stat.size
+
+    readed = 0
+    t = TempServer.new(Proc.new do |serv, accept|
+      client, client_addr = serv.accept
+      client.sync = true
+      loop do
+        buf = client.readpartial(8192) or break
+        break if buf.length == 0
+        assert_equal buf.length, received.syswrite(buf)
+        readed += buf.length
+        break if readed >= expect.stat.size
+      end
+      client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
+      client.close
+    end)
 
-  def test_store_content_nfs_empty
     @backend.create_open = {
-      'dev_count' => '1',
-      'devid_1' => '1',
-      'path_1' => '/path',
+      'devid' => '1',
+      'path' => "http://127.0.0.1:#{t.port}/path",
     }
 
-    @client.store_content 'new_key', 'test', ''
+    @client.store_file('new_key', 'test', to_put.path)
+    assert_equal expect.stat.size, readed
 
-    dest_file = File.join(@root, 'path')
+    ENV['PATH'].split(/:/).each do |path|
+      cmp_bin = "#{path}/cmp"
+      File.executable?(cmp_bin) or next
+      # puts "running #{cmp_bin} #{expect.path} #{received.path}"
+      assert( system(cmp_bin, expect.path, received.path) )
+      break
+    end
 
-    assert File.exist?(dest_file)
-    assert_equal '', File.read(dest_file)
+    ensure
+      TempServer.destroy_all!
   end
 
   def test_store_content_readonly
     @client.readonly = true
 
-    assert_raises RuntimeError do
+    assert_raises MogileFS::ReadOnlyError do
       @client.store_content 'new_key', 'test', nil
     end
   end
 
   def test_store_file_readonly
     @client.readonly = true
-    assert_raises RuntimeError do
+    assert_raises MogileFS::ReadOnlyError do
       @client.store_file 'new_key', 'test', nil
     end
   end
@@ -294,23 +465,25 @@ Content-Length: 0\r
   def test_rename_nonexisting
     @backend.rename = 'unknown_key', ''
 
-    assert_nil @client.rename('from_key', 'to_key')
+    assert_raises MogileFS::Backend::UnknownKeyError do
+      @client.rename('from_key', 'to_key')
+    end
   end
 
   def test_rename_no_key
-    @backend.rename = 'no_key', ''
+    @backend.rename = 'no_key', 'no_key'
 
-    e = assert_raises RuntimeError do
+    e = assert_raises MogileFS::Backend::NoKeyError do
       @client.rename 'new_key', 'test'
     end
 
-    assert_equal 'unable to rename new_key to test: no_key', e.message
+    assert_equal 'no_key', e.message
   end
 
   def test_rename_readonly
     @client.readonly = true
 
-    e = assert_raises RuntimeError do
+    e = assert_raises MogileFS::ReadOnlyError do
       @client.rename 'new_key', 'test'
     end
 
@@ -324,5 +497,12 @@ Content-Length: 0\r
     end
   end
 
+  private
+
+    # tested with 1000, though it takes a while
+    def nr_chunks
+      ENV['NR_CHUNKS'] ? ENV['NR_CHUNKS'].to_i : 10
+    end
+
 end