mogilefs-client 2.0.0 → 2.0.1

data/.gitignore

@@ -1,2 +1,4 @@
 /pkg
 /doc
+/.config
+/InstalledFiles

data/History.txt

@@ -1,3 +1,9 @@
+= 2.0.1
+* verify_uris method made more robust
+* preliminary Ruby 1.9 compatibility, tests still need some work
+* allow store_content to be used with a streamable object of known length
+* add setup.rb for non-RubyGems users
+
 = 2.0.0
 
 * use a set of standard exceptions based on MogileFS::Error,

data/README.txt

@@ -60,7 +60,7 @@ Then install the gem:
 
   # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
   # 'image' using an open IO.
-  File.open 'image.jpeg' do |fp|
+  File.open 'image.jpeg', 'rb' do |fp|
     mg.store_file 'my_image', 'image', fp
   end
 

data/bin/mog

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 require 'mogilefs'
 require 'optparse'
+[ STDIN, STDOUT, STDERR].each { |io| io.binmode }
 
 trap('INT') { exit 130 }
 trap('PIPE') { exit 0 }

data/lib/mogilefs.rb

@@ -6,7 +6,7 @@
 
 module MogileFS
 
-  VERSION = '2.0.0'.freeze
+  VERSION = '2.0.1'.freeze
 
   ##
   # Raised when a socket remains unreadable for too long.

data/lib/mogilefs/backend.rb

@@ -274,9 +274,14 @@ class MogileFS::Backend
 
   ##
   # Escapes naughty URL characters.
-
-  def url_escape(str)
-    str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1[0] }.tr(' ', '+')
+  if ''.respond_to?(:ord) # Ruby 1.9
+    def url_escape(str)
+      str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1.ord }.tr(' ', '+')
+    end
+  else # Ruby 1.8
+    def url_escape(str)
+      str.gsub(/([^\w\,\-.\/\\\: ])/) { "%%%02x" % $1[0] }.tr(' ', '+')
+    end
   end
 
   ##

data/lib/mogilefs/httpfile.rb

@@ -44,12 +44,15 @@ class MogileFS::HTTPFile < StringIO
 
   attr_accessor :big_io
 
+  attr_accessor :streaming_io
+
   ##
   # Works like File.open.  Use MogileFS::MogileFS#new_file instead of this
   # method.
 
   def self.open(*args)
     fp = new(*args)
+    fp.set_encoding(Encoding::BINARY) if fp.respond_to?(:set_encoding)
 
     return fp unless block_given?
 
@@ -72,6 +75,7 @@ class MogileFS::HTTPFile < StringIO
     @klass = klass
     @key = key
     @big_io = nil
+    @streaming_io = nil
 
     @dests = dests
     @tried = {}
@@ -87,9 +91,16 @@ class MogileFS::HTTPFile < StringIO
     sock = Socket.mogilefs_new(uri.host, uri.port)
     sock.mogilefs_tcp_cork = true
 
-    if @big_io
+    if @streaming_io
+      file_size = @streaming_io.length
+      syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
+                          "Content-Length: #{file_size}\r\n\r\n")
+      @streaming_io.call(Proc.new do |data_to_write|
+        syswrite_full(sock, data_to_write)
+      end)
+    elsif @big_io
       # Don't try to run out of memory
-      File.open(@big_io) do |fp|
+      File.open(@big_io, "rb") do |fp|
         file_size = fp.stat.size
         fp.sync = true
         syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \

data/lib/mogilefs/mogilefs.rb

@@ -148,7 +148,7 @@ class MogileFS::MogileFS < MogileFS::Client
 	  mfp.big_io = file
 	  return
 	else
-          return File.open(file) { |fp| sysrwloop(fp, mfp) }
+          return File.open(file, "rb") { |fp| sysrwloop(fp, mfp) }
         end
       end
     end
@@ -161,7 +161,11 @@ class MogileFS::MogileFS < MogileFS::Client
     raise MogileFS::ReadOnlyError if readonly?
 
     new_file key, klass do |mfp|
-      mfp << content
+      if content.is_a?(MogileFS::Util::StoreContent)
+        mfp.streaming_io = content
+      else
+        mfp << content
+      end
     end
 
     content.length

data/lib/mogilefs/network.rb

@@ -7,8 +7,6 @@ module MogileFS::Network
   # with the expected HTTP code within the timeout period (in seconds).
   def verify_uris(uris = [], expect = '200', timeout = 2.00)
     uri_socks = {}
-    ok_uris = []
-    sockets = []
 
     # first, we asynchronously connect to all of them
     uris.each do |uri|
@@ -18,47 +16,89 @@ module MogileFS::Network
 
     # wait for at least one of them to finish connecting and send
     # HTTP requests to the connected ones
-    begin
-      t0 = Time.now
-      r = IO.select(nil, uri_socks.keys, nil, timeout > 0 ? timeout : 0)
-      timeout -= (Time.now - t0)
-      break unless r && r[1]
-      r[1].each do |sock|
-        begin
-          # we don't about short/interrupted writes here, if the following
-          # request fails or blocks then the server is flat-out hopeless
-          sock.syswrite "HEAD #{uri_socks[sock].request_uri} HTTP/1.0\r\n\r\n"
-          sockets << sock
-        rescue
-          sock.close rescue nil
-        end
-      end
-    end until sockets[0] || timeout < 0
+    sockets, timeout = get_writable_set(uri_socks, timeout)
 
     # Await a response from the sockets we had written to, we only need one
     # valid response, but we'll take more if they return simultaneously
-    if sockets[0]
+    sockets[0] ? get_readable_uris(sockets, uri_socks, expect, timeout) : []
+
+    ensure
+      uri_socks.keys.each { |sock| sock.close rescue nil }
+  end
+
+  private
+    include MogileFS::Util
+
+    # returns an array of writeable Sockets and leftover from timeout
+    def get_writable_set(uri_socks, timeout)
+      sockets = []
       begin
         t0 = Time.now
-        r = IO.select(sockets, nil, nil, timeout > 0 ? timeout : 0)
+        r = begin
+         IO.select(nil, uri_socks.keys, nil, timeout > 0 ? timeout : 0)
+        rescue
+          # get rid of bad descriptors
+          uri_socks.delete_if do |sock, uri|
+            begin
+              sock.recv_nonblock(1)
+              false # should never get here for HTTP, really...
+            rescue Errno::EAGAIN, Errno::EINTR
+              false
+            rescue
+              sock.close rescue nil
+              true
+            end
+          end
+          timeout -= (Time.now - t0)
+          retry if timeout >= 0
+        end
+
+        break unless r && r[1]
+
+        r[1].each do |sock|
+          begin
+            # we don't about short/interrupted writes here, if the following
+            # request fails or blocks then the server is flat-out hopeless
+            sock.syswrite "HEAD #{uri_socks[sock].request_uri} HTTP/1.0\r\n\r\n"
+            sockets << sock
+          rescue
+            sock.close rescue nil
+          end
+        end
+
         timeout -= (Time.now - t0)
-        break unless r && r[0]
-        r[0].each do |sock|
-          buf = sock.recv_nonblock(128, Socket::MSG_PEEK) rescue next
+      end until (sockets[0] || timeout < 0)
+
+      [ sockets, timeout ]
+    end
+
+    # returns an array of URIs from uri_socks that are good
+    def get_readable_uris(sockets, uri_socks, expect, timeout)
+      ok_uris = []
+
+      begin
+        t0 = Time.now
+        r = IO.select(sockets, nil, nil, timeout > 0 ? timeout : 0) rescue nil
+
+        (r && r[0] ? r[0] : sockets).each do |sock|
+          buf = begin
+            sock.recv_nonblock(128, Socket::MSG_PEEK)
+          rescue Errno::EAGAIN, Errno::EINTR
+            next
+          rescue
+            sockets.delete(sock) # socket went bad
+            next
+          end
+
           if buf && /\AHTTP\/[\d\.]+ #{expect} / =~ buf
             ok_uris << uri_socks.delete(sock)
             sock.close rescue nil
           end
         end
-      end
-    end until ok_uris[0] || timeout < 0
-
-    ok_uris
-    ensure
-      uri_socks.keys.each { |sock| sock.close rescue nil }
-  end
+        timeout -= (Time.now - t0)
+      end until ok_uris[0] || timeout < 0
 
-  private
-    include MogileFS::Util
+      ok_uris
+    end
 
 end # module MogileFS::Network

data/lib/mogilefs/pool.rb

@@ -31,8 +31,10 @@ class MogileFS::Pool
   def use
     object = get
     yield object
+    nil
   ensure
     put object
+    nil
   end
 
   def purge

data/lib/mogilefs/util.rb

@@ -67,6 +67,16 @@ module MogileFS::Util
     # should never get here
   end
 
+  class StoreContent < Proc
+    def initialize(total_size, &writer_proc)
+      @total_size = total_size
+      super(&writer_proc)
+    end
+    def length
+      @total_size
+    end
+  end
+
 end
 
 require 'timeout'

data/test/setup.rb

@@ -1,10 +1,12 @@
 STDIN.sync = STDOUT.sync = STDERR.sync = true
 require 'test/unit'
-
+require 'tempfile'
 require 'fileutils'
 require 'tmpdir'
 require 'stringio'
 
+trap('CHLD') { Process.waitpid(-1, Process::WNOHANG) rescue nil }
+
 $TESTING = true
 
 require 'mogilefs'
@@ -52,37 +54,41 @@ end
 
 require 'socket'
 class TempServer
-  attr_reader :port
+  attr_reader :port, :pid
 
   def self.destroy_all!
     ObjectSpace.each_object(TempServer) { |t| t.destroy! }
   end
 
-  def initialize(server_proc)
-    @thr = @port = @sock = nil
+  at_exit { TempServer.destroy_all! }
+
+  def initialize(server_proc, port = nil)
+    @pid = @sock = nil
+    @port = port
     retries = 0
     begin
-      @port = 1024 + rand(32768 - 1024)
+      @port ||= 1024 + rand(32768 - 1024)
       @sock = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-      @sock.bind(Socket.pack_sockaddr_in(@port, '127.0.0.1'))
+      @sock.bind(Socket.pack_sockaddr_in(@port.to_i, '127.0.0.1'))
       @sock.listen(5)
     rescue Errno::EADDRINUSE, Errno::EACCES
       @sock.close rescue nil
       retry if (retries += 1) < 10
     end
-    @thr = Thread.new(@sock, @port) { |s,p| server_proc.call(s, p) }
+    @pid = fork { server_proc.call(@sock, @port) }
+    @sock.close rescue nil
   end
 
   def destroy!
     @sock.close rescue nil
-    Thread.kill(@thr) rescue nil
+    Process.kill('KILL', @pid) rescue nil
   end
 
 end
 
 class TestMogileFS < Test::Unit::TestCase
 
-  undef_method :default_test
+  undef_method :default_test if method_defined?(:default_test)
 
   def setup
     @client = @klass.new :hosts => ['kaa:6001'], :domain => 'test'

data/test/test_backend.rb

@@ -35,19 +35,20 @@ class TestBackend < Test::Unit::TestCase
   end
 
   def test_do_request
-    received = ''
+    received = Tempfile.new('received')
     tmp = TempServer.new(Proc.new do |serv, port|
       client, client_addr = serv.accept
       client.sync = true
-      received = client.recv 4096
+      received.syswrite(client.recv(4096))
       client.send "OK 1 you=win\r\n", 0
     end)
 
-    @backend.hosts = "127.0.0.1:#{tmp.port}"
+    @backend.hosts = [ "127.0.0.1:#{tmp.port}" ]
 
     assert_equal({'you' => 'win'},
                  @backend.do_request('go!', { 'fight' => 'team fight!' }))
-    assert_equal "go! fight=team+fight%21\r\n", received
+    received.sysseek(0)
+    assert_equal "go! fight=team+fight%21\r\n", received.sysread(4096)
     ensure
       TempServer.destroy_all!
   end
@@ -118,24 +119,24 @@ class TestBackend < Test::Unit::TestCase
   end
 
   def test_readable_eh_readable
-    accept_nr = 0
+    accept = Tempfile.new('accept')
     tmp = TempServer.new(Proc.new do |serv, port|
       client, client_addr = serv.accept
       client.sync = true
-      accept_nr += 1
+      accept.syswrite('.')
       client.send('.', 0)
       sleep
     end)
 
     @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{tmp.port}" ]
     assert_equal true, @backend.readable?
-    assert_equal 1, accept_nr
+    assert_equal 1, accept.stat.size
     ensure
       TempServer.destroy_all!
   end
 
   def test_readable_eh_not_readable
-    tmp = TempServer.new(Proc.new { |a,b| sleep })
+    tmp = TempServer.new(Proc.new { |serv,port| serv.accept; sleep })
     @backend = MogileFS::Backend.new(:hosts => [ "127.0.0.1:#{tmp.port}" ],
                                      :timeout => 0.5)
     begin
@@ -158,36 +159,45 @@ class TestBackend < Test::Unit::TestCase
   end
 
   def test_socket_robust
-    bad_accept_nr = accept_nr = 0
-    queue = Queue.new
-    bad = Proc.new { |serv,port| sleep; bad_accept_nr += 1 }
+    bad_accept = Tempfile.new('bad_accept')
+    accept = Tempfile.new('accept')
+    bad = Proc.new do |serv,port|
+      client, client_addr = serv.accept
+      bad_accept.syswrite('!')
+    end
     good = Proc.new do |serv,port|
       client, client_addr = serv.accept
-      client.sync = true
-      accept_nr += 1
-      client.send '.', 0
-      client.flush
-      queue.push true
+      accept.syswrite('.')
+      client.syswrite('.')
+      client.close
       sleep
     end
     nr = 10
 
     nr.times do
       begin
-        t1 = TempServer.new(bad)
+        t1 = TempServer.new(bad, ENV['TEST_DEAD_PORT'])
         t2 = TempServer.new(good)
-        hosts = ["0:#{t1.port}", "0:#{t2.port}"]
-        @backend = MogileFS::Backend.new(:hosts => hosts)
+        hosts = ["127.0.0.1:#{t1.port}", "127.0.0.1:#{t2.port}"]
+        @backend = MogileFS::Backend.new(:hosts => hosts.dup)
         assert_equal({}, @backend.dead)
+        old_chld_handler = trap('CHLD', 'DEFAULT')
         t1.destroy!
-        @backend.socket
-        wait = queue.pop
+        Process.waitpid(t1.pid)
+        trap('CHLD', old_chld_handler)
+        sock = @backend.socket
+        assert_equal Socket, sock.class
+        port = Socket.unpack_sockaddr_in(sock.getpeername).first
+        # p [ 'ports', "port=#{port}", "t1=#{t1.port}", "t2=#{t2.port}" ]
+        assert_equal t2.port, port
+        IO.select([sock])
+        assert_equal '.', sock.sysread(1)
       ensure
         TempServer.destroy_all!
       end
     end # nr.times
-    assert_equal 0, bad_accept_nr
-    assert_equal nr, accept_nr
+    assert_equal 0, bad_accept.stat.size
+    assert_equal nr, accept.stat.size
   end
 
   def test_shutdown
@@ -195,15 +205,18 @@ class TestBackend < Test::Unit::TestCase
     tmp = TempServer.new(Proc.new do |serv,port|
       client, client_addr = serv.accept
       accept_nr += 1
+      r = IO.select([client], [client])
+      client.syswrite(accept_nr.to_s)
       sleep
     end)
     @backend = MogileFS::Backend.new :hosts => [ "127.0.0.1:#{tmp.port}" ]
     assert @backend.socket
     assert ! @backend.socket.closed?
+    IO.select([@backend.socket])
+    resp = @backend.socket.sysread(4096)
     @backend.shutdown
     assert_equal nil, @backend.instance_variable_get(:@socket)
-    assert_equal 1, accept_nr
-
+    assert_equal 1, resp.to_i
     ensure
       TempServer.destroy_all!
   end

data/test/test_mogilefs.rb

@@ -20,15 +20,16 @@ class TestMogileFS__MogileFS < TestMogileFS
   end
 
   def test_get_file_data_http
-    accept_nr = 0
+    tmp = Tempfile.new('accept')
+    accept = File.open(tmp.path, "ab")
     svr = Proc.new do |serv, port|
       client, client_addr = serv.accept
       client.sync = true
       readed = client.recv(4096, 0)
       assert(readed =~ \
             %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      accept.syswrite('.')
       client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
-      accept_nr += 1
       client.close
     end
     t1 = TempServer.new(svr)
@@ -39,13 +40,14 @@ class TestMogileFS__MogileFS < TestMogileFS
     @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
 
     assert_equal 'data!', @client.get_file_data('key')
-    assert_equal 1, accept_nr
+    assert_equal 1, accept.stat.size
     ensure
       TempServer.destroy_all!
   end
 
   def test_get_file_data_http_not_found_failover
-    accept_nr = 0
+    tmp = Tempfile.new('accept')
+    accept = File.open(tmp.path, 'ab')
     svr1 = Proc.new do |serv, port|
       client, client_addr = serv.accept
       client.sync = true
@@ -53,7 +55,7 @@ class TestMogileFS__MogileFS < TestMogileFS
       assert(readed =~ \
             %r{\AGET /dev1/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
       client.send("HTTP/1.0 404 Not Found\r\n\r\ndata!", 0)
-      accept_nr += 1
+      accept.syswrite('.')
       client.close
     end
 
@@ -64,7 +66,7 @@ class TestMogileFS__MogileFS < TestMogileFS
       assert(readed =~ \
             %r{\AGET /dev2/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
       client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
-      accept_nr += 1
+      accept.syswrite('.')
       client.close
     end
 
@@ -75,7 +77,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
 
     assert_equal 'data!', @client.get_file_data('key')
-    assert_equal 2, accept_nr
+    assert_equal 2, accept.stat.size
     ensure
       TempServer.destroy_all!
   end
@@ -92,16 +94,17 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal expect_size + header.size, File.size(tmpfp.path)
     tmpfp.sysseek(0)
 
-    accept_nr = 0
+    accept = Tempfile.new('accept')
     svr = Proc.new do |serv, port|
       client, client_addr = serv.accept
       client.sync = true
-      accept_nr += 1
+      accept.syswrite('.')
       readed = client.recv(4096, 0)
       assert(readed =~ \
             %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
       sysrwloop(tmpfp, client)
       client.close
+      exit 0
     end
     t1 = TempServer.new(svr)
     t2 = TempServer.new(svr)
@@ -120,13 +123,15 @@ class TestMogileFS__MogileFS < TestMogileFS
           read_nr = buf.size
           nr += read_nr
           assert_equal read_nr, data.syswrite(buf), "partial write"
+        rescue Errno::EAGAIN
+          retry
         rescue EOFError
           break
         end
       end
     end
     assert_equal expect_size, nr, "size mismatch"
-    assert_equal 1, accept_nr
+    assert_equal 1, accept.stat.size
   end
 
   def test_get_paths
@@ -241,14 +246,14 @@ class TestMogileFS__MogileFS < TestMogileFS
   end
 
   def test_size_http
-    accept_nr = 0
+    accept = Tempfile.new('accept')
     t = TempServer.new(Proc.new do |serv,port|
       client, client_addr = serv.accept
       client.sync = true
       readed = client.recv(4096, 0) rescue nil
+      accept.syswrite('.')
       assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
       client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\n", 0)
-      accept_nr += 1
       client.close
     end)
 
@@ -256,18 +261,18 @@ class TestMogileFS__MogileFS < TestMogileFS
     @backend.get_paths = { 'paths' => 1, 'path1' => path }
 
     assert_equal 5, @client.size('key')
-    assert_equal 1, accept_nr
+    assert_equal 1, accept.stat.size
   end
 
   def test_bad_size_http
-    accept_nr = 0
+    tmp = Tempfile.new('accept')
     t = TempServer.new(Proc.new do |serv,port|
       client, client_addr = serv.accept
       client.sync = true
       readed = client.recv(4096, 0) rescue nil
       assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
       client.send("HTTP/1.0 404 Not Found\r\nContent-Length: 5\r\n\r\n", 0)
-      accept_nr += 1
+      tmp.syswrite('.')
       client.close
     end)
 
@@ -275,17 +280,17 @@ class TestMogileFS__MogileFS < TestMogileFS
     @backend.get_paths = { 'paths' => 1, 'path1' => path }
 
     assert_nil @client.size('key')
-    assert_equal 1, accept_nr
+    assert_equal 1, tmp.stat.size
   end
 
   def test_store_content_http
-    received = ''
+    received = Tempfile.new('recieved')
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
 
     t = TempServer.new(Proc.new do |serv, accept|
       client, client_addr = serv.accept
       client.sync = true
-      received = client.recv(4096, 0)
+      received.syswrite(client.recv(4096, 0))
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
     end)
@@ -297,19 +302,61 @@ class TestMogileFS__MogileFS < TestMogileFS
 
     @client.store_content 'new_key', 'test', 'data'
 
-    assert_equal expected, received
+    received.sysseek(0)
+    assert_equal expected, received.sysread(4096)
+    ensure
+      TempServer.destroy_all!
+  end
+
+
+  def test_store_content_with_writer_callback
+    received = Tempfile.new('recieved')
+    expected = "PUT /path HTTP/1.0\r\nContent-Length: 40\r\n\r\n"
+    10.times do
+      expected += "data"
+    end
+    t = TempServer.new(Proc.new do |serv, accept|
+      client, client_addr = serv.accept
+      client.sync = true
+      nr = 0
+      loop do
+        buf = client.readpartial(8192) or break
+        break if buf.length == 0
+        assert_equal buf.length, received.syswrite(buf)
+        nr += buf.length
+        break if nr >= expected.size
+      end
+      client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
+      client.close
+    end)
+
+    @backend.create_open = {
+      'devid' => '1',
+      'path' => "http://127.0.0.1:#{t.port}/path",
+    }
+
+    cbk = MogileFS::Util::StoreContent.new(40) do |write_callback|
+      10.times do
+        write_callback.call("data")
+      end
+    end
+    @client.store_content('new_key', 'test', cbk)
+
+    received.sysseek(0)
+    assert_equal expected, received.sysread(4096)
     ensure
       TempServer.destroy_all!
   end
 
   def test_store_content_multi_dest_failover
-    received1 = received2 = nil
+    received1 = Tempfile.new('received')
+    received2 = Tempfile.new('received')
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
 
     t1 = TempServer.new(Proc.new do |serv, accept|
       client, client_addr = serv.accept
       client.sync = true
-      received1 = client.recv(4096, 0)
+      received1.syswrite(client.recv(4096, 0))
       client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
       client.close
     end)
@@ -317,7 +364,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     t2 = TempServer.new(Proc.new do |serv, accept|
       client, client_addr = serv.accept
       client.sync = true
-      received2 = client.recv(4096, 0)
+      received2.syswrite(client.recv(4096, 0))
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
     end)
@@ -331,8 +378,10 @@ class TestMogileFS__MogileFS < TestMogileFS
     }
 
     @client.store_content 'new_key', 'test', 'data'
-    assert_equal expected, received1
-    assert_equal expected, received2
+    received1.sysseek(0)
+    received2.sysseek(0)
+    assert_equal expected, received1.sysread(4096)
+    assert_equal expected, received2.sysread(4096)
     ensure
       TempServer.destroy_all!
   end
@@ -357,12 +406,12 @@ class TestMogileFS__MogileFS < TestMogileFS
   end
 
   def test_store_content_http_empty
-    received = ''
+    received = Tempfile.new('received')
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
     t = TempServer.new(Proc.new do |serv, accept|
       client, client_addr = serv.accept
       client.sync = true
-      received = client.recv(4096, 0)
+      received.syswrite(client.recv(4096, 0))
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
     end)
@@ -373,7 +422,8 @@ class TestMogileFS__MogileFS < TestMogileFS
     }
 
     @client.store_content 'new_key', 'test', ''
-    assert_equal expected, received
+    received.sysseek(0)
+    assert_equal expected, received.sysread(4096)
   end
 
   def test_store_content_nfs
@@ -406,17 +456,19 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal expect_size + header.size, expect.stat.size
     assert_equal expect_size, to_put.stat.size
 
-    readed = 0
+    readed = Tempfile.new('readed')
     t = TempServer.new(Proc.new do |serv, accept|
       client, client_addr = serv.accept
       client.sync = true
+      nr = 0
       loop do
         buf = client.readpartial(8192) or break
         break if buf.length == 0
         assert_equal buf.length, received.syswrite(buf)
-        readed += buf.length
-        break if readed >= expect.stat.size
+        nr += buf.length
+        break if nr >= expect.stat.size
       end
+      readed.syswrite("#{nr}")
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
     end)
@@ -427,7 +479,8 @@ class TestMogileFS__MogileFS < TestMogileFS
     }
 
     @client.store_file('new_key', 'test', to_put.path)
-    assert_equal expect.stat.size, readed
+    readed.sysseek(0)
+    assert_equal expect.stat.size, readed.sysread(4096).to_i
 
     ENV['PATH'].split(/:/).each do |path|
       cmp_bin = "#{path}/cmp"

data/test/test_network.rb

@@ -24,4 +24,32 @@ class TestNetwork < Test::Unit::TestCase
       TempServer.destroy_all!
   end
 
+  def test_verify_non_existent
+    good = TempServer.new(Proc.new do |serv,port|
+      client,client_addr = serv.accept
+      client.readpartial(4096)
+      client.syswrite("HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n")
+    end)
+    bad = TempServer.new(Proc.new { |serv,port| serv.close })
+
+    good_uri = URI.parse("http://127.0.0.1:#{good.port}/")
+    bad_uri = URI.parse("http://127.0.0.1:#{bad.port}/")
+    ok = verify_uris([ good_uri, bad_uri ])
+    assert_equal [ good_uri ], ok
+    ensure
+      TempServer.destroy_all!
+  end
+
+  def test_verify_all_bad
+    good = TempServer.new(Proc.new { |serv,port| serv.close })
+    bad = TempServer.new(Proc.new { |serv,port| serv.close })
+
+    good_uri = URI.parse("http://127.0.0.1:#{good.port}/")
+    bad_uri = URI.parse("http://127.0.0.1:#{bad.port}/")
+    ok = verify_uris([ good_uri, bad_uri ], '200', 1.0)
+    assert ok.empty?, "nothing returned"
+    ensure
+      TempServer.destroy_all!
+  end
+
 end

data/test/test_util.rb

@@ -1,24 +1,23 @@
 require 'test/setup'
-require 'tempfile'
 
 class TestMogileFS__Util < Test::Unit::TestCase
   include MogileFS::Util
 
   def test_mogilefs_write
-    done = Queue.new
+    rd, wr = IO.pipe
 
     svr = Proc.new do |serv, port|
       client, client_addr = serv.accept
       client.sync = true
-      readed = 0
+      nr = 0
       loop do
         begin
-          readed += client.readpartial(16384).length
+          nr += client.readpartial(16384).length
         rescue EOFError
           break
         end
       end
-      done << readed
+      wr.syswrite("#{nr}\n")
       client.close rescue nil
     end
     t = TempServer.new(svr)
@@ -32,14 +31,13 @@ class TestMogileFS__Util < Test::Unit::TestCase
 
     syswrite_full(s, big_string)
     s.close rescue nil
-    readed = done.pop
-
-    assert_equal((sent + big_string.length), readed)
+    IO.select([rd])
+    assert_equal((sent + big_string.length), rd.sysread(4096).to_i)
+    ensure
+      t.destroy!
   end
 
   def test_write_timeout
-    done = Queue.new
-
     svr = Proc.new do |serv, port|
       client, client_addr = serv.accept
       client.sync = true
@@ -54,6 +52,8 @@ class TestMogileFS__Util < Test::Unit::TestCase
 
     assert_raises(MogileFS::Timeout) { syswrite_full(s, big_string, 0.1) }
     s.close rescue nil
+    ensure
+      t.destroy!
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors: 
 - Eric Wong
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-02 00:00:00 -08:00
+date: 2009-02-25 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -31,7 +31,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.8.2
+        version: 1.8.3
     version: 
 description: git://git.bogomips.org/mogilefs-client.git
 email: normalperson@yhbt.net