mogilefs-client 2.0.2 → 2.1.0

data/GNUmakefile

@@ -10,12 +10,26 @@ test: $(T)
 clean:
 	$(RM) $(TO) $(addsuffix +,$(TO))
 
-t = $(basename $(notdir $@))
-t_log = $(subst .rb,.log,$@)
 
+ifndef V
+	quiet_pre = @echo '* $@';
+	quiet_post = >$(t) 2>&1
+else
+	# we can't rely on -o pipefail outside of bash 3+,
+	# so we use a stamp file to indicate success and
+	# have rm fail if the stamp didn't get created
+	stamp = $@$(log_suffix).ok
+	quiet_pre = @echo $(ruby) $@ $(TEST_OPTS); ! test -f $(stamp) && (
+	quiet_post = && > $(stamp) )>&2 | tee $(t); rm $(stamp) 2>/dev/null
+endif
+ruby = ruby
+run_test = $(quiet_pre) setsid $(ruby) -w $@ $(TEST_OPTS) $(quiet_post) || \
+  (sed "s,^,$(extra): ," >&2 < $(t); exit 1)
+
+$(T): t = $(subst .rb,.log,$@)
+$(T): export RUBYLIB := $(CURDIR)/lib:$(RUBYLIB)
 $(T):
-	@echo $(t); ruby -I lib $@ $(TEST_OPTS) > $(t_log)+ 2>&1
-	@mv $(t_log)+ $(t_log)
+	$(run_test)
 
 # using make instead of rake since Rakefile takes too long to load
 manifest: Manifest.txt

data/History.txt

@@ -1,3 +1,12 @@
+= 2.1.0
+* MySQL interface returns integer length and devcount (API change)
+* Ensure store_{content,file} always returns size (API fix)
+* Add get_uris API method
+* Respect timeout when doing get_file_data
+* MySQL interface filters out URLs for down/dead hosts/devices
+* Really remove all NFS support
+* get_file in slurp mode slurps all output correctly
+
 = 2.0.2
 * README.txt: update documentation URL
 * backend: raise exceptions with the error message

data/README.txt

@@ -14,15 +14,20 @@ File bugs:
 
 http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
 
-Source repository (git):
+Please email Eric Wong at normalperson@yhbt.net as well since
+he finds web interfaces difficult to use.
 
-git://git.bogomips.org/mogilefs-client.git
+Source repository (git):
 
-http://git.bogomips.org/mogilefs-client.git
+  git://git.bogomips.org/mogilefs-client.git
+  http://git.bogomips.org/mogilefs-client.git
+  git://repo.or.cz/ruby-mogilefs-client.git (mirror)
+  http://repo.or.cz/r/ruby-mogilefs-client.git (mirror)
 
-Repository browser (cgit):
+Repository browsers:
 
-http://git.bogomips.org/cgit/mogilefs-client.git
+* http://git.bogomips.org/cgit/mogilefs-client.git (cgit)
+* http://repo.or.cz/w/ruby-mogilefs-client.git (gitweb mirror)
 
 == About
 

data/bin/mog

@@ -186,7 +186,6 @@ begin
     cfg[:class] or raise ArgumentError, 'E: --class must be specified'
     buf = ''
     tmp = Tempfile.new('mog-tee') # TODO: explore Transfer-Encoding:chunked :)
-    at_exit { tmp.unlink }
 
     # if stdout is pointing to /dev/null, don't bother installing the filter.
     STDOUT.sync = true
@@ -196,7 +195,7 @@ begin
       sysrwloop(STDIN, tmp, tee_filter)
       store_file_retry(mg, key, cfg[:class], tmp.path)
     ensure
-      tmp.close
+      tmp.close!
     end
   when 'test'
     truth, ok = true, nil

data/lib/mogilefs/bigfile.rb

@@ -56,7 +56,7 @@ module MogileFS::Bigfile
         raise MogileFS::Backend::NoDevices if uris.empty?
       end
 
-      sock = http_get_sock(uris[0])
+      sock = http_read_sock(uris[0])
       md5.reset if md5
       w = sysrwloop(sock, wr_io, filter)
 

data/lib/mogilefs/mogilefs.rb

@@ -19,10 +19,6 @@ class MogileFS::MogileFS < MogileFS::Client
 
   attr_accessor :get_file_data_timeout
 
-  ##
-  # internal Regexp for matching an "HTTP 200 OK" head response
-  HTTP_200_OK = %r{\AHTTP/\d+\.\d+\s+200\s+}.freeze
-
   ##
   # Creates a new MogileFS::MogileFS instance.  +args+ must include a key
   # :domain specifying the domain of this client.
@@ -61,27 +57,20 @@ class MogileFS::MogileFS < MogileFS::Client
   # Retrieves the contents of +key+.
 
   def get_file_data(key, &block)
-    paths = get_paths key
-
-    return nil unless paths
-
+    paths = get_paths(key) or return nil
     paths.each do |path|
-      next unless path
-      case path
-      when /^http:\/\// then
+      begin
+        sock = http_read_sock(URI.parse(path))
         begin
-          sock = http_get_sock(URI.parse(path))
-          return block_given? ? yield(sock) : sock.read
-        rescue MogileFS::Timeout, Errno::ECONNREFUSED,
-               EOFError, SystemCallError, MogileFS::InvalidResponseError
-          next
+          return yield(sock) if block_given?
+          return sysread_full(sock, sock.mogilefs_size, @get_file_data_timeout)
+        ensure
+          sock.close rescue nil
         end
-      else
-        next unless File.exist? path
-        return File.read(path)
+      rescue MogileFS::Timeout, MogileFS::InvalidResponseError,
+             Errno::ECONNREFUSED, EOFError, SystemCallError
       end
     end
-
     nil
   end
 
@@ -93,7 +82,14 @@ class MogileFS::MogileFS < MogileFS::Client
              :noverify => noverify ? 1 : 0, :zone => zone }
     @backend.respond_to?(:_get_paths) and return @backend._get_paths(opts)
     res = @backend.get_paths(opts)
-    (1..res['paths'].to_i).map { |i| res["path#{i}"] }
+    (1..res['paths'].to_i).map { |i| res["path#{i}"] }.compact
+  end
+
+  ##
+  # Get the URIs for +key+.
+
+  def get_uris(key, noverify = true, zone = nil)
+    get_paths(key, noverify, zone).map { |path| URI.parse(path) }
   end
 
   ##
@@ -135,20 +131,22 @@ class MogileFS::MogileFS < MogileFS::Client
 
   ##
   # Copies the contents of +file+ into +key+ in class +klass+.  +file+ can be
-  # either a file name or an object that responds to #read.
+  # either a file name or an object that responds to #sysread.
+  # Returns size of +file+ stored
 
   def store_file(key, klass, file)
     raise MogileFS::ReadOnlyError if readonly?
 
     new_file key, klass do |mfp|
       if file.respond_to? :sysread then
-        return sysrwloop(file, mfp)
+        sysrwloop(file, mfp)
       else
-	if File.size(file) > 0x10000 # Bigass file, handle differently
-	  mfp.big_io = file
-	  return
-	else
-          return File.open(file, "rb") { |fp| sysrwloop(fp, mfp) }
+        size = File.size(file)
+        if size > 0x10000 # Bigass file, handle differently
+          mfp.big_io = file
+          size
+        else
+          File.open(file, "rb") { |fp| sysrwloop(fp, mfp) }
         end
       end
     end
@@ -207,34 +205,13 @@ class MogileFS::MogileFS < MogileFS::Client
 
   def paths_size(paths)
     paths.each do |path|
-      next unless path
-      case path
-      when /^http:\/\// then
-        begin
-          url = URI.parse path
-          s = Socket.mogilefs_new_request(url.host, url.port,
-                                   "HEAD #{url.request_uri} HTTP/1.0\r\n\r\n",
-                                   @get_file_data_timeout)
-          res = s.recv(4096, 0)
-          if res =~ HTTP_200_OK
-            head, body = res.split(/\r\n\r\n/, 2)
-            if head =~ /^Content-Length:\s*(\d+)/i
-              return $1.to_i
-            end
-          end
-          next
-        rescue MogileFS::Timeout, Errno::ECONNREFUSED,
-               EOFError, SystemCallError
-          next
-        ensure
-          s.close rescue nil
-        end
-      else
-        next unless File.exist? path
-        return File.size(path)
+      begin
+        return http_read_sock(URI.parse(path), "HEAD").mogilefs_size
+      rescue MogileFS::InvalidResponseError, MogileFS::Timeout,
+             Errno::ECONNREFUSED, EOFError, SystemCallError => err
+        next
       end
     end
-
     nil
   end
 
@@ -271,19 +248,27 @@ class MogileFS::MogileFS < MogileFS::Client
 
     # given a URI, this returns a readable socket with ready data from the
     # body of the response.
-    def http_get_sock(uri)
+    def http_read_sock(uri, http_method = "GET")
       sock = Socket.mogilefs_new_request(uri.host, uri.port,
-                                    "GET #{uri.request_uri} HTTP/1.0\r\n\r\n",
-                                    @get_file_data_timeout)
-      buf = sock.recv(4096, Socket::MSG_PEEK)
+                    "#{http_method} #{uri.request_uri} HTTP/1.0\r\n\r\n",
+                    @get_file_data_timeout)
+      buf = sock.recv_nonblock(4096, Socket::MSG_PEEK)
       head, body = buf.split(/\r\n\r\n/, 2)
-      if head =~ HTTP_200_OK
-        sock.recv(head.size + 4, 0)
+
+      # we're dealing with a seriously slow/stupid HTTP server if we can't
+      # get the header in a single read(2) syscall.
+      if head =~ %r{\AHTTP/\d+\.\d+\s+200\s*} &&
+         head =~ %r{^Content-Length:\s*(\d+)}i
+        sock.mogilefs_size = $1.to_i
+        case http_method
+        when "HEAD" then sock.close
+        when "GET" then sock.recv(head.size + 4, 0)
+        end
         return sock
       end
+      sock.close rescue nil
       raise MogileFS::InvalidResponseError,
-            "GET on #{uri} returned: #{head.inspect}"
-    end # def http_get_sock
+            "#{http_method} on #{uri} returned: #{head.inspect}"
+    end # def http_read_sock
 
 end
-

data/lib/mogilefs/mysql.rb

@@ -55,7 +55,7 @@ class MogileFS::Mysql
 
     keys = []
     query(sql).each do |dkey,length,devcount|
-      yield(dkey, length, devcount) if block_given?
+      yield(dkey, length.to_i, devcount.to_i) if block_given?
       keys << dkey
     end
 
@@ -101,7 +101,7 @@ class MogileFS::Mysql
         devices = refresh_device(true)
         devinfo = devices[devid.to_i] or next
       end
-
+      devinfo[:readable] or next
       port = devinfo[:http_get_port]
       host = zone && zone == 'alt' ? devinfo[:altip] : devinfo[:hostip]
       nfid = '%010u' % fid
@@ -120,10 +120,10 @@ class MogileFS::Mysql
       GET_DOMAINS = 'SELECT dmid,namespace FROM domain'.freeze
 
       GET_DEVICES = <<-EOS
-        SELECT d.devid, h.hostip, h.altip, h.http_port, h.http_get_port
+        SELECT d.devid, h.hostip, h.altip, h.http_port, h.http_get_port,
+          d.status, h.status
         FROM device d
           LEFT JOIN host h ON d.hostid = h.hostid
-        WHERE d.status IN ('alive','readonly','drain');
       EOS
       GET_DEVICES.freeze
     end
@@ -132,15 +132,24 @@ class MogileFS::Mysql
       @my.send(@query_method, sql)
     end
 
+    DEV_STATUS_READABLE = {
+      "alive" => true,
+      "readonly" => true,
+      "drain" => true,
+    }.freeze
+
     def refresh_device(force = false)
       return @cache_device if ! force && ((Time.now - @last_update_device) < 60)
       tmp = {}
       res = query(GET_DEVICES)
-      res.each do |devid, hostip, altip, http_port, http_get_port|
+      res.each do |devid, hostip, altip, http_port, http_get_port,
+                   dev_status, host_status|
         http_port = http_port ? http_port.to_i : 80
         tmp[devid.to_i] = {
           :hostip => hostip.freeze,
           :altip => (altip || hostip).freeze,
+          :readable => (host_status == "alive" &&
+                        DEV_STATUS_READABLE.include?(dev_status)),
           :http_port => http_port,
           :http_get_port => http_get_port ?  http_get_port.to_i : http_port,
         }.freeze

data/lib/mogilefs/network.rb

@@ -57,9 +57,11 @@ module MogileFS::Network
 
         r[1].each do |sock|
           begin
-            # we don't about short/interrupted writes here, if the following
-            # request fails or blocks then the server is flat-out hopeless
-            sock.syswrite "HEAD #{uri_socks[sock].request_uri} HTTP/1.0\r\n\r\n"
+            # we don't care about short/interrupted writes here, if the
+            # following request fails or blocks then the server is
+            # flat-out hopeless
+            sock.write_nonblock(
+              "HEAD #{uri_socks[sock].request_uri} HTTP/1.0\r\n\r\n")
             sockets << sock
           rescue
             sock.close rescue nil

data/lib/mogilefs/util.rb

@@ -67,6 +67,29 @@ module MogileFS::Util
     # should never get here
   end
 
+  def sysread_full(io_rd, size, timeout = nil, full_timeout = false)
+    tmp = [] # avoid expensive string concatenation with every loop iteration
+    reader = io_rd.method(timeout ? :read_nonblock : :sysread)
+    begin
+      while size > 0
+        tmp << reader.call(size)
+        size -= tmp.last.size
+      end
+    rescue Errno::EAGAIN, Errno::EINTR
+      t0 = Time.now
+      r = IO.select([ io_rd ], nil, nil, timeout)
+      if timeout
+        timeout -= (Time.now - t0) if full_timeout
+        if !(r && r[0]) || timeout < 0
+          raise MogileFS::Timeout, 'sysread_full timeout'
+        end
+      end
+      retry
+    rescue EOFError
+    end
+    tmp.join('')
+  end
+
   class StoreContent < Proc
     def initialize(total_size, &writer_proc)
       @total_size = total_size
@@ -87,7 +110,7 @@ require 'timeout'
 class MogileFS::Timeout < Timeout::Error; end
 
 class Socket
-  attr_accessor :mogilefs_addr, :mogilefs_connected
+  attr_accessor :mogilefs_addr, :mogilefs_connected, :mogilefs_size
 
   TCP_CORK = 3 if ! defined?(TCP_CORK) && RUBY_PLATFORM =~ /linux/
 
@@ -159,9 +182,14 @@ class Socket
       sock = mogilefs_new(host, port, timeout)
       syswrite_full(sock, request, timeout)
       timeout -= (Time.now - t0)
-      raise MogileFS::Timeout, 'socket read timeout' if timeout < 0
+      if timeout < 0
+        sock.close rescue nil
+        raise MogileFS::Timeout, 'socket read timeout'
+      end
       r = IO.select([sock], nil, nil, timeout)
       return sock if r && r[0]
+
+      sock.close rescue nil
       raise MogileFS::Timeout, 'socket read timeout'
     end
 

data/lib/mogilefs.rb

@@ -6,7 +6,7 @@
 
 module MogileFS
 
-  VERSION = '2.0.2'.freeze
+  VERSION = '2.1.0'.freeze
 
   ##
   # Raised when a socket remains unreadable for too long.

data/test/setup.rb

@@ -65,15 +65,18 @@ class TempServer
   def initialize(server_proc, port = nil)
     @pid = @sock = nil
     @port = port
-    retries = 0
+    retries = 10
     begin
       @port ||= 1024 + rand(32768 - 1024)
       @sock = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
       @sock.bind(Socket.pack_sockaddr_in(@port.to_i, '127.0.0.1'))
       @sock.listen(5)
-    rescue Errno::EADDRINUSE, Errno::EACCES
+    rescue Errno::EADDRINUSE, Errno::EACCES => err
       @sock.close rescue nil
-      retry if (retries += 1) < 10
+      @port = nil
+      retry if port.nil? && (retries -= 1) > 0
+      warn "retries failed: #{retries} port=#{port.inspect}"
+      raise err
     end
     @pid = fork { server_proc.call(@sock, @port) }
     @sock.close rescue nil
@@ -106,11 +109,13 @@ end
 class FakeMysql
   attr_reader :expect
   TBL_DEVICES = [
-    # devid, hostip,    altip,         http_port, http_get_port
-    [ 1,    '10.0.0.1', '192.168.0.1', 7500,      7600 ],
-    [ 2,    '10.0.0.2', '192.168.0.2', 7500,      7600 ],
-    [ 3,    '10.0.0.3', nil,           7500,      nil ],
-    [ 4,    '10.0.0.4', nil,           7500,      nil ],
+    # devid, hostip, altip, http_port, http_get_port, dev status, host status
+    [ 1, '10.0.0.1', '192.168.0.1', 7500, 7600, 'readonly', 'alive' ],
+    [ 2, '10.0.0.2', '192.168.0.2', 7500, 7600, 'alive', 'alive' ],
+    [ 3, '10.0.0.3', nil, 7500, nil, 'readonly', 'alive' ],
+    [ 4, '10.0.0.4', nil, 7500, nil, 'alive', 'alive' ],
+    [ 5, '10.0.0.5', nil, 7500, nil, 'dead', 'alive' ],
+    [ 6, '10.0.0.6', nil, 7500, nil, 'alive', 'down' ],
   ]
   TBL_DOMAINS = [
     # dmid, namespace

data/test/test_mogilefs.rb

@@ -54,8 +54,8 @@ class TestMogileFS__MogileFS < TestMogileFS
       readed = client.recv(4096, 0)
       assert(readed =~ \
             %r{\AGET /dev1/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
-      client.send("HTTP/1.0 404 Not Found\r\n\r\ndata!", 0)
       accept.syswrite('.')
+      client.send("HTTP/1.0 404 Not Found\r\n\r\ndata!", 0)
       client.close
     end
 
@@ -65,8 +65,8 @@ class TestMogileFS__MogileFS < TestMogileFS
       readed = client.recv(4096, 0)
       assert(readed =~ \
             %r{\AGET /dev2/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
-      client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
       accept.syswrite('.')
+      client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
       client.close
     end
 
@@ -145,6 +145,18 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal expected, @client.get_paths('key').sort
   end
 
+  def test_get_uris
+    path1 = 'http://rur-1/dev1/0/000/000/0000000062.fid'
+    path2 = 'http://rur-2/dev2/0/000/000/0000000062.fid'
+
+    @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
+
+    expected = [ URI.parse(path1), URI.parse(path2) ]
+
+    assert_equal expected, @client.get_uris('key')
+  end
+
+
   def test_get_paths_unknown_key
     @backend.get_paths = ['unknown_key', '']
 
@@ -189,7 +201,7 @@ class TestMogileFS__MogileFS < TestMogileFS
   end
 
   def test_list_keys
-    @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_2',
+    @backend.list_keys = { 'key_count' => '2', 'next_after' => 'new_key_2',
                            'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
 
     keys, next_after = @client.list_keys 'new'
@@ -198,7 +210,7 @@ class TestMogileFS__MogileFS < TestMogileFS
   end
 
   def test_list_keys_block
-    @backend.list_keys = { 'key_count' => 2, 'next_after' => 'new_key_2',
+    @backend.list_keys = { 'key_count' => '2', 'next_after' => 'new_key_2',
                            'key_1' => 'new_key_1', 'key_2' => 'new_key_2' }
     http_resp = "HTTP/1.0 200 OK\r\nContent-Length: %u\r\n"
     srv = Proc.new do |serv, port, size|
@@ -212,10 +224,10 @@ class TestMogileFS__MogileFS < TestMogileFS
     t1 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
     t2 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 5) })
     t3 = TempServer.new(Proc.new { |serv, port| srv.call(serv, port, 10) })
-    @backend.get_paths = { 'paths' => 2,
+    @backend.get_paths = { 'paths' => '2',
                            'path1' => "http://127.0.0.1:#{t1.port}/",
                            'path2' => "http://127.0.0.1:#{t2.port}/" }
-    @backend.get_paths = { 'paths' => 1,
+    @backend.get_paths = { 'paths' => '1',
                            'path1' => "http://127.0.0.1:#{t3.port}/" }
 
     res = []
@@ -271,8 +283,8 @@ class TestMogileFS__MogileFS < TestMogileFS
       client.sync = true
       readed = client.recv(4096, 0) rescue nil
       assert_equal "HEAD /path HTTP/1.0\r\n\r\n", readed
-      client.send("HTTP/1.0 404 Not Found\r\nContent-Length: 5\r\n\r\n", 0)
       tmp.syswrite('.')
+      client.send("HTTP/1.0 404 Not Found\r\nContent-Length: 5\r\n\r\n", 0)
       client.close
     end)
 
@@ -283,6 +295,32 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal 1, tmp.stat.size
   end
 
+  def test_store_file_small_http
+    received = Tempfile.new('received')
+    to_store = Tempfile.new('small')
+    to_store.syswrite('data')
+
+    expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
+    t = TempServer.new(Proc.new do |serv, accept|
+      client, client_addr = serv.accept
+      client.sync = true
+      received.syswrite(client.recv(4096, 0))
+      client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
+      client.close
+    end)
+
+    @backend.create_open = {
+      'devid' => '1',
+      'path' => "http://127.0.0.1:#{t.port}/path",
+    }
+    nr = @client.store_file 'new_key', 'test', to_store.path
+    assert_equal 4, nr
+    received.sysseek(0)
+    assert_equal expected, received.sysread(4096)
+    ensure
+      TempServer.destroy_all!
+  end
+
   def test_store_content_http
     received = Tempfile.new('recieved')
     expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
@@ -300,7 +338,9 @@ class TestMogileFS__MogileFS < TestMogileFS
       'path' => "http://127.0.0.1:#{t.port}/path",
     }
 
-    @client.store_content 'new_key', 'test', 'data'
+    nr = @client.store_content 'new_key', 'test', 'data'
+    assert nr
+    assert_equal 4, nr
 
     received.sysseek(0)
     assert_equal expected, received.sysread(4096)
@@ -340,7 +380,9 @@ class TestMogileFS__MogileFS < TestMogileFS
         write_callback.call("data")
       end
     end
-    @client.store_content('new_key', 'test', cbk)
+    assert_equal 40, cbk.length
+    nr = @client.store_content('new_key', 'test', cbk)
+    assert_equal 40, nr
 
     received.sysseek(0)
     assert_equal expected, received.sysread(4096)
@@ -377,7 +419,8 @@ class TestMogileFS__MogileFS < TestMogileFS
       'path_2' => "http://127.0.0.1:#{t2.port}/path",
     }
 
-    @client.store_content 'new_key', 'test', 'data'
+    nr = @client.store_content 'new_key', 'test', 'data'
+    assert_equal 4, nr
     received1.sysseek(0)
     received2.sysseek(0)
     assert_equal expected, received1.sysread(4096)
@@ -421,7 +464,8 @@ class TestMogileFS__MogileFS < TestMogileFS
       'path' => "http://127.0.0.1:#{t.port}/path",
     }
 
-    @client.store_content 'new_key', 'test', ''
+    nr = @client.store_content 'new_key', 'test', ''
+    assert_equal 0, nr
     received.sysseek(0)
     assert_equal expected, received.sysread(4096)
   end
@@ -478,7 +522,11 @@ class TestMogileFS__MogileFS < TestMogileFS
       'path' => "http://127.0.0.1:#{t.port}/path",
     }
 
-    @client.store_file('new_key', 'test', to_put.path)
+    orig_size = to_put.size
+    nr = @client.store_file('new_key', 'test', to_put.path)
+    assert nr
+    assert_equal orig_size, nr
+    assert_equal orig_size, to_put.size
     readed.sysseek(0)
     assert_equal expect.stat.size, readed.sysread(4096).to_i
 

data/test/test_mysql.rb

@@ -20,22 +20,38 @@ class TestMogileFS__Mysql < Test::Unit::TestCase
       {:hostip=>"10.0.0.1",
        :http_get_port=>7600,
        :http_port=>7500,
+       :readable=>true,
        :altip=>"192.168.0.1"},
      2=>
       {:hostip=>"10.0.0.2",
        :http_get_port=>7600,
        :http_port=>7500,
+       :readable=>true,
        :altip=>"192.168.0.2"},
      3=>
       {:hostip=>"10.0.0.3",
        :http_get_port=>7500,
        :http_port=>7500,
+       :readable=>true,
        :altip=>"10.0.0.3"},
      4=>
       {:hostip=>"10.0.0.4",
        :http_get_port=>7500,
        :http_port=>7500,
-       :altip=>"10.0.0.4"}
+       :readable=>true,
+       :altip=>"10.0.0.4"},
+     5=>
+      {:hostip=>"10.0.0.5",
+       :http_get_port=>7500,
+       :http_port=>7500,
+       :readable=>false,
+       :altip=>"10.0.0.5"},
+     6=>
+      {:hostip=>"10.0.0.6",
+       :http_get_port=>7500,
+       :http_port=>7500,
+       :readable=>false,
+       :altip=>"10.0.0.6"}
     }
     assert_equal expect, @mg.refresh_device
   end
@@ -53,6 +69,20 @@ class TestMogileFS__Mysql < Test::Unit::TestCase
     assert_equal expect, @mg._get_paths(:domain => 'test', :key => 'fookey')
   end
 
+  def test_get_paths_bad_device
+    @my.expect << [ [ 12 ] ] # fid
+    @my.expect << [ [ 1 ], [ 6 ] ] # devids
+    expect = [ "http://10.0.0.1:7600/dev1/0/000/000/0000000012.fid" ]
+    assert_equal expect, @mg._get_paths(:domain => 'test', :key => 'fookey')
+  end
+
+  def test_get_paths_bad_host
+    @my.expect << [ [ 12 ] ] # fid
+    @my.expect << [ [ 1 ], [ 5 ] ] # devids
+    expect = [ "http://10.0.0.1:7600/dev1/0/000/000/0000000012.fid" ]
+    assert_equal expect, @mg._get_paths(:domain => 'test', :key => 'fookey')
+  end
+
   def test_get_paths_alt
     @my.expect <<  [ [ 12 ] ] # fid
     @my.expect << [ [ 1 ], [ 3 ] ] # devids
@@ -64,8 +94,10 @@ class TestMogileFS__Mysql < Test::Unit::TestCase
 
   def test_list_keys
     expect_full = [ [ 'foo', 123, 2 ], [ 'bar', 456, 1 ] ]
+    result_full = eval(expect_full.inspect)
+    result_full.each { |x| (1..2).each { |i| x[i] = x[i].to_s } }
     expect_keys = [ [ 'foo', 'bar' ], 'bar' ]
-    @my.expect << expect_full
+    @my.expect << result_full
     full = []
     keys = @mg._list_keys('test') do |dkey,length,devcount|
       full << [ dkey, length, devcount ]

data/test/test_util.rb

@@ -56,4 +56,65 @@ class TestMogileFS__Util < Test::Unit::TestCase
       t.destroy!
   end
 
+  def test_sysread_slowly
+    nr = 10
+    str = 'abcde'
+    expect = str * nr
+    rd, wr = IO.pipe
+    pid = fork do
+      rd.close
+      nr.times do
+        syswrite_full(wr, str)
+        sleep(0.1)
+      end
+    end
+    wr.close
+    buf = sysread_full(rd, expect.size)
+    assert_equal expect, buf
+    rd.close
+    ensure
+      Process.kill('TERM', pid) rescue nil
+      Process.waitpid(pid) rescue nil
+  end
+
+  def test_sysread_timeout
+    nr = 10
+    str = 'abcde'
+    expect = str * nr
+    rd, wr = IO.pipe
+    pid = fork do
+      rd.close
+      nr.times do
+        syswrite_full(wr, str)
+        sleep 1
+      end
+    end
+    wr.close
+    assert_raises(MogileFS::Timeout) { sysread_full(rd, expect.size, 0.1) }
+    rd.close
+    ensure
+      Process.kill('TERM', pid) rescue nil
+      Process.waitpid(pid) rescue nil
+  end
+
+  def test_sysread_full_timeout
+    nr = 100
+    str = 'abcde'
+    expect = str * nr
+    rd, wr = IO.pipe
+    pid = fork do
+      rd.close
+      nr.times do
+        syswrite_full(wr, str)
+        sleep 0.01
+      end
+    end
+    wr.close
+    assert_raises(MogileFS::Timeout) { sysread_full(rd,expect.size,0.1,true) }
+    rd.close
+    ensure
+      Process.kill('TERM', pid) rescue nil
+      Process.waitpid(pid) rescue nil
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  version: 2.0.2
+  version: 2.1.0
 platform: ruby
 authors: 
 - Eric Wong
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-03-02 00:00:00 -08:00
+date: 2009-04-11 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -31,9 +31,9 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.8.3
+        version: 1.10.0
     version: 
-description: git://git.bogomips.org/mogilefs-client.git
+description: "Source repository (git):"
 email: normalperson@yhbt.net
 executables: 
 - mog