mogilefs-client 3.4.0 → 3.5.0.rc1

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 CONSTANT = "MogileFS::VERSION"
 RVF = "lib/mogilefs/version.rb"
-DEF_VER = "v3.4.0"
+DEF_VER = "v3.5.0-rc1"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/lib/mogilefs.rb

@@ -64,6 +64,14 @@ module MogileFS
     @io = MogileFS::CopyStream
   end
 
+  begin
+    require 'net/http/persistent'
+    NHP = Net::HTTP::Persistent
+  rescue LoadError
+    require 'mogilefs/nhp_fake'
+    NHP = MogileFS::NhpFake
+  end
+
   # autoload rarely-used things:
   autoload :Mysql, 'mogilefs/mysql'
   autoload :Pool, 'mogilefs/pool'

data/lib/mogilefs/http_file.rb

@@ -95,7 +95,6 @@ class MogileFS::HTTPFile < StringIO
   # returns file size if the socket finished writing
   def upload(devid, uri) # :nodoc:
     sock = MogileFS::Socket.tcp(uri.host, uri.port)
-    set_socket_options(sock)
     file_size = length
 
     if @streaming_io
@@ -136,14 +135,41 @@ class MogileFS::HTTPFile < StringIO
       sock.close if sock
   end
 
+  def nhp_put(devid, uri)
+    clen = @opts[:content_length]
+    if clen && clen != size
+      raise MogileFS::SizeMismatchError,
+        ":content_length expected: #{clen.inspect}, actual: #{size}"
+    end
+
+    put = Net::HTTP::Put.new(uri.path)
+    put["Content-Type"] = "application/octet-stream"
+    if md5 = @opts[:content_md5]
+      if md5.respond_to?(:call)
+        md5 = md5.call.strip
+      elsif md5 == :trailer
+        md5 = [ Digest::MD5.digest(string) ].pack("m").chomp!
+      end
+      put["Content-MD5"] = md5
+    end
+    put.body = string
+    res = @opts[:nhp_put].request(uri, put)
+    return size if Net::HTTPSuccess === res
+    raise BadResponseError, "#{res.code} #{res.message}"
+  rescue => e
+    /\ANet::/ =~ "#{e.class}" and
+        raise RetryableError, "#{e.message} (#{e.class})", e.backtrace
+    raise
+  end
+
   def commit
     errors = nil
     @dests.each do |devid, path|
       begin
         uri = URI.parse(path)
-        bytes_uploaded = upload(devid, uri)
+        bytes_uploaded = size > 0 ? nhp_put(devid, uri) : upload(devid, uri)
         return create_close(devid, uri, bytes_uploaded)
-      rescue SystemCallError, RetryableError => e
+      rescue Timeout::Error, SystemCallError, RetryableError => e
         errors ||= []
         errors << "#{path} - #{e.message} (#{e.class})"
       end

data/lib/mogilefs/http_reader.rb

@@ -26,13 +26,9 @@ class MogileFS::HTTPReader < MogileFS::Socket
           "read=#{rv} bytes, expected=#@content_length from #@uri", []
   end
 
-  def self.first(paths, timeout, count = nil, offset = nil)
+  def self.first(paths, timeout, range = nil)
     errors = nil
-    if offset || count
-      offset ||= 0
-      range_end = count ? offset + count - 1 : ""
-      range = "Range: bytes=#{offset}-#{range_end}\r\n"
-    end
+    range = "Range: bytes=#{range[0]}-#{range[1]}\r\n" if range
 
     paths.each do |path|
       begin
@@ -50,17 +46,18 @@ class MogileFS::HTTPReader < MogileFS::Socket
   # body of the response.
   def self.try(path, timeout, range) # :nodoc:
     uri = URI.parse(path)
+    expire_at = Time.now + timeout
     sock = tcp(uri.host, uri.port, timeout)
     buf = "GET #{uri.request_uri} HTTP/1.0\r\n#{range}\r\n" # no chunking
     sock.timed_write(buf, timeout)
 
-    sock.timed_peek(2048, buf, timeout) or
-      raise MogileFS::InvalidResponseError, "EOF while reading header", []
+    begin
+      raise MogileFS::Timeout if Time.now > expire_at
+      sock.timed_peek(2048, buf, timeout) or
+        raise MogileFS::InvalidResponseError, "EOF while reading header", []
+    end until /\r\n\r\n/ =~ buf
 
     head, _ = buf.split(/\r\n\r\n/, 2)
-
-    # we're dealing with a seriously slow/stupid HTTP server if we can't
-    # get the header in a single recv(2) syscall.
     if ((range && head =~ %r{\AHTTP/\d+\.\d+\s+206\s*}) ||
         (!range && head =~ %r{\AHTTP/\d+\.\d+\s+200\s*})) &&
        head =~ %r{^Content-Length:\s*(\d+)}i

data/lib/mogilefs/mogilefs.rb

@@ -69,6 +69,10 @@ class MogileFS::MogileFS < MogileFS::Client
 
     @get_file_data_timeout = args[:get_file_data_timeout] || 5
     @new_file_max_time = args[:new_file_max_time] || 3600.0
+    @nhp_get = MogileFS::NHP.new('get')
+    @nhp_get.open_timeout = @nhp_get.read_timeout = @get_file_data_timeout
+    @nhp_put = MogileFS::NHP.new('put')
+    @nhp_put.open_timeout = @nhp_put.read_timeout = @new_file_max_time
 
     raise ArgumentError, "you must specify a domain" unless @domain
 
@@ -146,17 +150,40 @@ class MogileFS::MogileFS < MogileFS::Client
   # start position of the copy.
   def get_file_data(key, dst = nil, copy_length = nil, src_offset = nil)
     paths = get_paths(key)
-    sock = MogileFS::HTTPReader.first(paths, @get_file_data_timeout,
-                                      copy_length, src_offset)
+    if src_offset || copy_length
+      src_offset ||= 0
+      range_end = copy_length ? src_offset + copy_length - 1 : nil
+      range = [ src_offset, range_end ]
+    end
+
     if dst
+      sock = MogileFS::HTTPReader.first(paths, @get_file_data_timeout, range)
       sock.stream_to(dst)
     elsif block_given?
+      sock = MogileFS::HTTPReader.first(paths, @get_file_data_timeout, range)
       yield(sock)
     else
-      sock.to_s
+      errors = nil
+      paths.each do |path|
+        uri = URI.parse(path)
+        get = Net::HTTP::Get.new(uri.path)
+        get["range"] = "bytes=#{range[0]}-#{range[1]}" if range
+        begin
+          res = @nhp_get.request(uri, get)
+          case res.code.to_i
+          when 200, 206
+            return res.body
+          end
+          (errors ||= []) << "#{path} - #{res.message} (#{res.class})"
+        rescue => e
+          (errors ||= []) << "#{path} - #{e.message} (#{e.class})"
+        end
+      end
+      raise MogileFS::Error,
+            "all paths failed with GET: #{errors.join(', ')}", []
     end
-    ensure
-      sock.close if sock && ! sock.closed?
+  ensure
+    sock.close if sock && ! sock.closed?
   end
 
   # Get the paths (URLs as strings) for +key+.  If +args+ is specified,
@@ -244,18 +271,26 @@ class MogileFS::MogileFS < MogileFS::Client
   #   Keep in mind most HTTP servers do not support HTTP trailers, so
   #   passing a String is usually the safest way to use this.
   #
+  # [:info => Hash]
+  #
+  #   This is an empty hash that will be filled the same information
+  #   MogileFS::MogileFS#file_info.
+  #
+  #   Additionally, it contains one additional key: :uris,
+  #   an array of URI::HTTP objects to the stored destinations
   def new_file(key, args = nil, bytes = nil) # :yields: file
     raise MogileFS::ReadOnlyError if readonly?
-    opts = { :key => key, :multi_dest => 1 }
+    opts = { :key => key, :multi_dest => 1, :nhp_put => @nhp_put }
     case args
     when Hash
       opts[:domain] = args[:domain]
+      open_args = args[:create_open_args]
       klass = args[:class] and "default" != klass and opts[:class] = klass
     when String
       opts[:class] = args if "default" != args
     end
     opts[:domain] ||= @domain
-    res = @backend.create_open(opts)
+    res = @backend.create_open(open_args ? open_args.merge(opts) : opts)
 
     dests = if dev_count = res['dev_count'] # multi_dest succeeded
       (1..dev_count.to_i).map { |i| [res["devid_#{i}"], res["path_#{i}"]] }
@@ -275,6 +310,7 @@ class MogileFS::MogileFS < MogileFS::Client
     opts[:content_length] ||= bytes if bytes
     opts[:new_file_max_time] ||= @new_file_max_time
     opts[:start_time] = Time.now
+    info = opts[:info] and info["class"] = klass || "default"
 
     case (dests[0][1] rescue nil)
     when %r{\Ahttp://}

data/lib/mogilefs/new_file/common.rb

@@ -21,6 +21,7 @@ module MogileFS::NewFile::Common
   def read_response(sock)
     tout = @opts[:new_file_max_time] || 3600.0
     start_time = @opts[:start_time] and tout -= Time.now - start_time
+    set_socket_options(sock)
     case line = sock.timed_read(23, "", tout > 0.0 ? tout : 0)
     when %r{^HTTP/\d\.\d\s+(2\d\d)\s} # success!
     when nil
@@ -34,21 +35,23 @@ module MogileFS::NewFile::Common
   end
 
   def create_close(devid, uri, bytes_uploaded)
-    args = {
-      :fid => @opts[:fid],
-      :devid => devid,
-      :key => @opts[:key],
-      :domain => @opts[:domain],
-      :size => bytes_uploaded,
-      :path => uri.to_s,
-    }
+    close_args = @opts[:create_close_args]
+
+    dest_info = @opts[:info] ||= {}
+    dest_info["fid"] = @opts[:fid].to_i
+    dest_info["key"] = @opts[:key]
+    dest_info["domain"] = @opts[:domain]
+    dest_info[:devid] = devid
+    dest_info[:path] = uri.to_s
+    dest_info[:size] = bytes_uploaded
     if @md5
-      args[:checksum] = "MD5:#{@md5.hexdigest}"
+      dest_info["checksum"] = "MD5:#{@md5.hexdigest}"
     elsif String === @opts[:content_md5]
       hex = @opts[:content_md5].unpack('m')[0].unpack('H*')[0]
-      args[:checksum] = "MD5:#{hex}"
+      dest_info["checksum"] = "MD5:#{hex}"
     end
-    args[:checksumverify] = 1 if @opts[:checksumverify]
+
+    dest_info[:checksumverify] = 1 if @opts[:checksumverify]
     backend = @opts[:backend]
 
     # upload could've taken a long time, ping and try to ensure socket
@@ -61,7 +64,16 @@ module MogileFS::NewFile::Common
     # twice will hurt us...
     backend.noop
 
-    backend.create_close(args)
+    backend.create_close(close_args ? close_args.merge(dest_info) : dest_info)
+
+    # make this look like file_info + get_uris
+    dest_info.delete(:checksumverify)
+    dest_info.delete(:path)
+    dest_info[:uris] = [ uri ]
+    dest_info["devcount"] = 1
+    dest_info["devids"] = [ dest_info.delete(:devid).to_i ]
+    dest_info["length"] = dest_info.delete(:size)
+
     bytes_uploaded
   end
 

data/lib/mogilefs/new_file/content_range.rb

@@ -1,6 +1,5 @@
 # -*- encoding: binary -*-
 # here are internal implementation details, do not rely on them in your code
-require 'net/http'
 require 'mogilefs/new_file/writer'
 
 # an IO-like object
@@ -9,18 +8,8 @@ class MogileFS::NewFile::ContentRange
   include MogileFS::NewFile::Common
   attr_reader :md5
 
-  # :stopdoc:
-  begin
-    require 'net/http/persistent'
-    NHP = Net::HTTP::Persistent.new('mogilefs')
-
-    def hit(uri, req)
-      NHP.request(uri, req).value
-    end
-  rescue LoadError
-    def hit(uri, req)
-      Net::HTTP.start(uri.host, uri.port) { |h| h.request(req).value }
-    end
+  def hit(uri, req)
+    @opts[:nhp_put].request(uri, req).value
   end
   # :startdoc:
 

data/lib/mogilefs/new_file/stream.rb

@@ -18,7 +18,6 @@ class MogileFS::NewFile::Stream
       begin
         uri = URI.parse(path)
         sock = MogileFS::Socket.tcp(uri.host, uri.port)
-        set_socket_options(sock)
         start_sock(sock, uri) # raise on errors
         @to_io = sock
         @uri = uri

data/lib/mogilefs/nhp_fake.rb

@@ -0,0 +1,18 @@
+# :enddoc:
+require 'net/http'
+
+# This is just for folks that don't have net-http-persistent
+class MogileFS::NhpFake # :nodoc:
+  attr_accessor :read_timeout, :open_timeout
+
+  def initialize(name)
+    @read_timeout = @open_timeout = nil
+  end
+
+  def request(uri, req) # :nodoc:
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.read_timeout = @read_timeout
+    http.open_timeout = @open_timeout
+    http.request(req)
+  end
+end

data/test/fresh.rb

@@ -90,17 +90,14 @@ EOF
     @admin.create_domain(domain)
     yield_for_monitor_update { @admin.get_domains.include?(domain) and break }
 
-    assert_nothing_raised do
-      @admin.create_class(domain, "klassy", 1)
-    end
+    @admin.create_class(domain, "klassy", 1)
+
     assert_raises(MogileFS::Backend::ClassExistsError) do
       @admin.create_class(domain, "klassy", 1)
     end
 
-    assert_nothing_raised do
-      @admin.update_class(domain, "klassy",
-                          :mindevcount => 1, :replpolicy => "MultipleHosts(1)")
-    end
+    @admin.update_class(domain, "klassy",
+                        :mindevcount => 1, :replpolicy => "MultipleHosts(1)")
 
     tmp = nil
     yield_for_monitor_update do
@@ -110,7 +107,7 @@ EOF
     assert tmp, "domain did not show up"
     assert_equal 1, tmp["mindevcount"]
     assert_equal "MultipleHosts(1)", tmp["replpolicy"]
-    assert_nothing_raised { @admin.update_class(domain, "klassy", 2) }
+    @admin.update_class(domain, "klassy", 2)
     ensure
       @admin.delete_class(domain, "klassy") rescue nil
   end

data/test/socket_test.rb

@@ -11,11 +11,9 @@ module SocketTest
   def test_start
     sock = MogileFS::Socket.start(@host, @port)
     assert_instance_of MogileFS::Socket, sock, sock.inspect
-    assert_nothing_raised do
-      begin
-        sock.write_nonblock("a")
-      rescue Errno::EAGAIN
-      end
+    begin
+      sock.write_nonblock("a")
+    rescue Errno::EAGAIN
     end
     thr = Thread.new { @srv.accept }
     accepted = thr.value
@@ -26,9 +24,7 @@ module SocketTest
   def test_new
     sock = MogileFS::Socket.tcp(@host, @port)
     assert_instance_of MogileFS::Socket, sock, sock.inspect
-    assert_nothing_raised do
-      sock.write_nonblock("a")
-    end
+    sock.write_nonblock("a")
     thr = Thread.new { @srv.accept }
     accepted = thr.value
     assert_instance_of TCPSocket, accepted, accepted.inspect
@@ -99,7 +95,7 @@ module SocketTest
   def test_read_in_full
     sock = MogileFS::Socket.tcp(@host, @port)
     accepted = @srv.accept
-    thr = Thread.new do
+    Thread.new do
       accepted.write "HELLO"
       sleep 0.1
       accepted.write " "

data/test/test_backend.rb

@@ -59,9 +59,6 @@ class TestBackend < Test::Unit::TestCase
     assert_equal MogileFS::Error, @backend.error('PebKacError').superclass
     assert MogileFS::Backend.const_defined?('PebKacError')
 
-    assert_nothing_raised do
-      MogileFS::Backend::OMFGWTFBBQError
-    end
     assert_equal MogileFS::Error, MogileFS::Backend::OMFGWTFBBQError.superclass
     assert_raises(NameError) do
       MogileFS::Backend::FailFailFail
@@ -146,10 +143,7 @@ class TestBackend < Test::Unit::TestCase
         client.write("OK 1 foo=bar\r\n")
         client
       end
-      rv = nil
-      assert_nothing_raised do
-        rv = @backend.do_request('test', { "all" => "ALL" })
-      end
+      rv = @backend.do_request('test', { "all" => "ALL" })
       accepted = thr.value
       assert_equal "test all=ALL\r\n", accepted.readpartial(666)
       assert_equal({"foo"=>"bar"}, rv)

data/test/test_db_backend.rb

@@ -66,7 +66,7 @@ class TestMogileFS__DbBackend < Test::Unit::TestCase
   end
 
   def test_sleep
-    assert_nothing_raised { assert_equal({}, @mg.sleep(1)) }
+    assert_equal({}, @mg.sleep(1))
   end
 
 end

data/test/test_fresh.rb

@@ -32,4 +32,119 @@ class TestMogFresh < Test::Unit::TestCase
     client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
     assert_equal false, client.exist?("non-existent")
   end
+
+  def test_new_file_info(checksum = nil)
+    add_host_device_domain unless checksum
+    @client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    info = {}
+    opts = { :info => info }
+    key = "new_file_info"
+    content = "ZZZZ"
+    if checksum
+      opts[:content_md5] = [ Digest::MD5.digest(content) ].pack('m').rstrip
+      opts[:class] = "check"
+    end
+    @client.new_file(key, opts) do |http_file|
+      http_file << content
+    end
+
+    uris = info.delete(:uris)
+    assert_kind_of Array, uris
+    assert_equal(uris, (@client.get_uris(key) & uris))
+    expect_info = @client.file_info(key, :devices => true)
+    match_keys = %w(class fid key domain length)
+    match_keys << "checksum" if checksum
+    match_keys.each do |field|
+      assert_equal expect_info.delete(field), info.delete(field)
+    end
+    assert_operator expect_info.delete("devcount"), :>=, info.delete("devcount")
+    devids = info.delete("devids")
+    assert_equal(devids, (expect_info.delete("devids") & devids))
+
+    assert info.empty?, info.inspect
+    assert expect_info.empty?, expect_info.inspect
+  ensure
+    @client.delete(key)
+  end
+
+  def test_new_file_info_checksum
+    add_host_device_domain
+    opts = @admin.get_domains[@domain]["default"]
+    opts["hashtype"] = "MD5"
+    @admin.create_class(@domain, "check", opts)
+    yield_for_monitor_update do
+      tmp = @admin.get_domains[@domain]["check"]
+      if tmp
+        case tmp["hashtype"]
+        when "MD5"
+          break
+        when nil
+          warn "skipping checksum test, MogileFS server too old"
+          return
+        else
+          raise "Unhandled hashtype: #{tmp['hashtype']}"
+        end
+      end
+    end
+    test_new_file_info(:md5)
+  ensure
+    @admin.delete_class @domain, "check"
+  end
+
+  def test_create_open_close_opts
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    socket = client.backend.socket
+    args = {
+      :create_open_args => { :hello => "world" },
+      :create_close_args => { :farewell => "goodnight" },
+    }
+    io = client.new_file("foo", args)
+    socket.write "!recent\n"
+    buf = ""
+    buf << socket.readpartial(666) until buf =~ /\.\r?\n\z/
+    line = buf.split(/\r?\n/).grep(/\screate_open\s/)[0]
+    assert_equal 0, buf.split(/\r?\n/).grep(/\screate_close\s/).size
+    assert_equal 0, buf.split(/\r?\n/).grep(/\sfarewell\s/).size
+    assert_match(/\bhello=world\b/, line)
+    assert_equal 1, io.write('.')
+    assert_nil io.close
+
+    socket.write "!recent\n"
+    buf = ""
+    buf << socket.readpartial(666) until buf =~ /\.\r?\n\z/
+    line = buf.split(/\r?\n/).grep(/\screate_close\s/)[0]
+    assert_match(/\bfarewell=goodnight\b/, line)
+  end
+
+  def test_get_file_data_range
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    data = "data"
+    client.store_content("key", "default", data)
+
+    assert_equal data, client.get_file_data("key")
+
+    # ensure offset/length matches IO.copy_stream
+    src = Tempfile.new("tmp")
+    src.write(data)
+    src.flush
+    [ [1,nil], [1,2], [3,1] ].each do |range|
+      dst2 = StringIO.new
+      client.get_file_data("key", dst2, *range)
+
+      src.rewind
+
+      if IO.respond_to?(:copy_stream)
+        # ensure we match IO.copy_stream semantics
+        dst = StringIO.new
+        IO.copy_stream(src, dst, *range)
+        assert_equal dst.string, dst2.string
+        assert_equal dst.string, client.get_file_data("key", nil, *range)
+      end
+
+      assert_equal dst2.string, client.get_file_data("key", nil, *range)
+    end
+    src.close!
+  end
 end

data/test/test_mogilefs.rb

@@ -10,6 +10,14 @@ class TestMogileFS__MogileFS < TestMogileFS
     super
   end
 
+  def read_headers(client)
+    headers = ""
+    while line = client.gets
+      headers << line
+      return headers if line == "\r\n"
+    end
+  end
+
   def test_initialize
     assert_equal 'test', @client.domain
 
@@ -24,9 +32,10 @@ class TestMogileFS__MogileFS < TestMogileFS
     svr = Proc.new do |serv, port|
       client, _ = serv.accept
       client.sync = true
-      readed = client.recv(4096, 0)
-      assert(readed =~ \
-            %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      readed = read_headers(client)
+      assert_match(
+        %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n},
+        readed)
       accept.syswrite('.')
       client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
       client.close
@@ -50,9 +59,10 @@ class TestMogileFS__MogileFS < TestMogileFS
     svr1 = Proc.new do |serv, port|
       client, _ = serv.accept
       client.sync = true
-      readed = client.recv(4096, 0)
-      assert(readed =~ \
-            %r{\AGET /dev1/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      readed = read_headers(client)
+      assert_match(
+        %r{\AGET /dev1/0/000/000/0000000062\.fid HTTP/1.[01]\r\n},
+        readed)
       accept.syswrite('.')
       client.send("HTTP/1.0 404 Not Found\r\n\r\ndata!", 0)
       client.close
@@ -61,9 +71,10 @@ class TestMogileFS__MogileFS < TestMogileFS
     svr2 = Proc.new do |serv, port|
       client, _ = serv.accept
       client.sync = true
-      readed = client.recv(4096, 0)
-      assert(readed =~ \
-            %r{\AGET /dev2/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      readed = read_headers(client)
+      assert_match(
+        %r{\AGET /dev2/0/000/000/0000000062\.fid HTTP/1.[01].*\r\n},
+        readed)
       accept.syswrite('.')
       client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
       client.close
@@ -98,9 +109,10 @@ class TestMogileFS__MogileFS < TestMogileFS
       client, _ = serv.accept
       client.sync = true
       accept.syswrite('.')
-      readed = client.recv(4096, 0)
-      assert(readed =~ \
-            %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n\r\n\Z})
+      readed = read_headers(client)
+      assert_match(
+        %r{\AGET /dev[12]/0/000/000/0000000062\.fid HTTP/1.[01]\r\n},
+        readed)
       MogileFS.io.copy_stream(tmpfp, client)
       client.close
       exit 0
@@ -166,9 +178,7 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_delete_existing
     @backend.delete = { }
-    assert_nothing_raised do
-      @client.delete 'no_such_key'
-    end
+    @client.delete 'no_such_key'
   end
 
   def test_delete_nonexisting
@@ -227,11 +237,12 @@ class TestMogileFS__MogileFS < TestMogileFS
     to_store = Tempfile.new('small')
     to_store.syswrite('data')
 
-    expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
     t = TempServer.new(Proc.new do |serv, accept|
       client, _ = serv.accept
-      client.sync = true
-      received.syswrite(client.read(expected.bytesize))
+      while buf = client.readpartial(666)
+        received.syswrite(buf)
+        break if buf =~ /data/
+      end
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
     end)
@@ -243,14 +254,18 @@ class TestMogileFS__MogileFS < TestMogileFS
     nr = @client.store_file 'new_key', 'test', to_store.path
     assert_equal 4, nr
     received.sysseek(0)
-    assert_equal expected, received.sysread(4096)
+
+    a = received.sysread(999999).split(/\r\n/)
+    assert_match(%r{\APUT /path HTTP/1\.[01]\z}, a[0])
+    assert_equal("data", a[-1])
+    assert_equal("", a[-2])
+    assert a.grep(%r{\AContent-Length: 4\z})[0]
     ensure
       TempServer.destroy_all!
   end
 
   def test_store_content_http
     received = Tempfile.new('received')
-    expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
 
     t = TempServer.new(Proc.new do |serv, accept|
       client, _ = serv.accept
@@ -275,7 +290,11 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal 4, nr
 
     received.sysseek(0)
-    assert_equal expected, received.sysread(4096)
+    a = received.sysread(999999).split(/\r\n/)
+    assert_match(%r{\APUT /path HTTP/1\.[01]\z}, a[0])
+    assert_equal("data", a[-1])
+    assert_equal("", a[-2])
+    assert a.grep(%r{\AContent-Length: 4\z})[0]
     ensure
       TempServer.destroy_all!
   end
@@ -291,12 +310,14 @@ class TestMogileFS__MogileFS < TestMogileFS
       client, _ = serv.accept
       client.sync = true
       nr = 0
+      seen = ''
       loop do
         buf = client.readpartial(8192) or break
         break if buf.length == 0
         assert_equal buf.length, received.syswrite(buf)
         nr += buf.length
-        break if nr >= expected.size
+        seen << buf
+        break if seen =~ /\r\n\r\n(?:data){10}/
       end
       client.send("HTTP/1.0 200 OK\r\n\r\n", 0)
       client.close
@@ -317,7 +338,11 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal 40, nr
 
     received.sysseek(0)
-    assert_equal expected, received.sysread(4096)
+    a = received.sysread(999999).split(/\r\n/)
+    assert_match(%r{\APUT /path HTTP/1\.[01]\z}, a[0])
+    assert_equal("data" * 10, a[-1])
+    assert_equal("", a[-2])
+    assert a.grep(%r{\AContent-Length: 40\z})[0]
     ensure
       TempServer.destroy_all!
   end
@@ -329,7 +354,6 @@ class TestMogileFS__MogileFS < TestMogileFS
   def test_store_content_multi_dest_failover(big_io = false)
     received1 = Tempfile.new('received')
     received2 = Tempfile.new('received')
-    expected = "PUT /path HTTP/1.0\r\nContent-Length: 4\r\n\r\ndata"
 
     t1 = TempServer.new(Proc.new do |serv, accept|
       client, _ = serv.accept
@@ -375,8 +399,17 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal 4, nr
     received1.sysseek(0)
     received2.sysseek(0)
-    assert_equal expected, received1.sysread(4096)
-    assert_equal expected, received2.sysread(4096)
+    a = received1.sysread(4096).split(/\r\n/)
+    b = received2.sysread(4096).split(/\r\n/)
+    assert_equal a[0], b[0]
+    assert_match(%r{\APUT /path HTTP/1\.[01]\z}, a[0])
+    assert_match(%r{\APUT /path HTTP/1\.[01]\z}, b[0])
+    assert_equal("data", a[-1])
+    assert_equal("data", b[-1])
+    assert_equal("", a[-2])
+    assert_equal("", b[-2])
+    assert a.grep(%r{\AContent-Length: 4\z})[0]
+    assert b.grep(%r{\AContent-Length: 4\z})[0]
     ensure
       TempServer.destroy_all!
   end
@@ -385,7 +418,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     t = TempServer.new(Proc.new do |serv, accept|
       client, _ = serv.accept
       client.sync = true
-      client.recv(4096, 0)
+      read_headers(client)
       client.send("HTTP/1.0 500 Internal Server Error\r\n\r\n", 0)
       client.close
     end)
@@ -402,7 +435,6 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_store_content_http_empty
     received = Tempfile.new('received')
-    expected = "PUT /path HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
     t = TempServer.new(Proc.new do |serv, accept|
       client, _ = serv.accept
       client.sync = true
@@ -419,7 +451,9 @@ class TestMogileFS__MogileFS < TestMogileFS
     nr = @client.store_content 'new_key', 'test', ''
     assert_equal 0, nr
     received.sysseek(0)
-    assert_equal expected, received.sysread(4096)
+    a = received.sysread(4096).split(/\r\n/)
+    assert_match(%r{\APUT /path HTTP/1\.[01]\z}, a[0])
+    assert a.grep(%r{\AContent-Length: 0\z})[0]
   end
 
   def test_store_content_nfs
@@ -613,7 +647,6 @@ class TestMogileFS__MogileFS < TestMogileFS
     timeout = 1
     args = { :hosts => hosts, :domain => "foo", :timeout => timeout }
     c = MogileFS::MogileFS.new(args)
-    received = []
     secs = timeout + 1
     th = Thread.new do
       close_later = []
@@ -832,9 +865,7 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_sleep
     @backend.sleep = {}
-    assert_nothing_raised do
-      assert_equal({}, @client.sleep(2))
-    end
+    assert_equal({}, @client.sleep(2))
   end
 
   private

data/test/test_mogilefs_integration.rb

@@ -47,10 +47,8 @@ class TestMogileFSIntegration < TestMogIntegration
       w.close
     end
     assert_equal data.size, @client.store_file("pipe", nil, r)
-    assert_nothing_raised do
-      r.close
-      th.join
-    end
+    r.close
+    th.join
     assert_equal(data, @client.get_file_data("pipe"))
 
     cbk = MogileFS::Util::StoreContent.new(nil) do |write_callback|
@@ -177,19 +175,19 @@ class TestMogileFSIntegration < TestMogIntegration
 
   def test_new_file_no_block
     rv = @client.new_file("no_block")
-    assert_nothing_raised { rv.write "HELLO" }
+    rv.write "HELLO"
     assert_nil rv.close
     assert_equal "HELLO", @client.get_file_data("no_block")
   end
 
   def test_new_file_known_content_length
     rv = @client.new_file("a", :content_length => 5)
-    assert_nothing_raised { rv.write "HELLO" }
+    rv.write "HELLO"
     assert_nil rv.close
     assert_equal "HELLO", @client.get_file_data("a")
 
     rv = @client.new_file("a", :content_length => 6)
-    assert_nothing_raised { rv.write "GOOD" }
+    rv.write "GOOD"
     assert_raises(MogileFS::SizeMismatchError) { rv.close }
     assert_equal "HELLO", @client.get_file_data("a")
 
@@ -204,19 +202,18 @@ class TestMogileFSIntegration < TestMogIntegration
     r, w = IO.pipe
     b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
     rv = @client.new_file("a", :content_md5 => b64digest, :content_length => 5)
-    assert_nothing_raised { rv.write "HELLO" }
+    rv.write "HELLO"
     assert_nil rv.close
     assert_equal "HELLO", @client.get_file_data("a")
 
-    assert_nothing_raised { w.write "HIHI"; w.close }
+    w.write "HIHI"
+    w.close
     assert_raises(ArgumentError) do
       @client.new_file("a", :content_md5 => b64digest) { |f| f.big_io = r }
     end
     assert_equal "HELLO", @client.get_file_data("a")
 
-    assert_nothing_raised do
-      @client.new_file("a", :content_md5 => :trailer) { |f| f.big_io = r }
-    end
+    @client.new_file("a", :content_md5 => :trailer) { |f| f.big_io = r }
     assert_equal "HIHI", @client.get_file_data("a")
 
     # legacy, in case anybody used it
@@ -241,9 +238,7 @@ class TestMogileFSIntegration < TestMogIntegration
 
   def test_store_content_opts
     b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
-    assert_nothing_raised do
-      @client.store_content("c", nil, "HELLO", :content_md5 => b64digest)
-    end
+    @client.store_content("c", nil, "HELLO", :content_md5 => b64digest)
     assert_raises(MogileFS::SizeMismatchError) do
       @client.store_content("c", nil, "GOODBYE", :content_length => 2)
     end
@@ -253,9 +248,7 @@ class TestMogileFSIntegration < TestMogIntegration
   def test_store_file_opts
     b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
     io = StringIO.new("HELLO")
-    assert_nothing_raised do
-      @client.store_file("c", nil, io, :content_md5 => b64digest)
-    end
+    @client.store_file("c", nil, io, :content_md5 => b64digest)
 
     io = StringIO.new("GOODBYE")
     assert_raises(MogileFS::SizeMismatchError) do
@@ -271,9 +264,7 @@ class TestMogileFSIntegration < TestMogIntegration
       [ Digest::MD5.digest("HELLO") ].pack('m').strip
     end
     io = StringIO.new("HELLO")
-    assert_nothing_raised do
-      @client.store_file("c", nil, io, :content_md5 => expect_md5)
-    end
+    @client.store_file("c", nil, io, :content_md5 => expect_md5)
 
     assert_equal true, checked, "expect_md5 lambda called"
   end
@@ -284,9 +275,7 @@ class TestMogileFSIntegration < TestMogIntegration
     tmp.sync = true
     tmp.write "HIHI"
     tmp.rewind
-    assert_nothing_raised do
-      @client.store_file("unlinked", nil, tmp)
-    end
+    @client.store_file("unlinked", nil, tmp)
 
     assert_equal "HIHI", @client.get_file_data("unlinked")
   end

data/test/test_mogtool_bigfile.rb

@@ -51,7 +51,7 @@ class TestMogtoolBigfile < TestMogIntegration
 
     # corrupt the existing data in part1
     @client.store_content(part1, nil, "HELLO")
-    assert_nothing_raised { @client.get_uris(part1) }
+    @client.get_uris(part1)
 
     # corruption is detected on verify
     junk = tmpfile("junk")
@@ -61,9 +61,7 @@ class TestMogtoolBigfile < TestMogIntegration
 
     # corruption is NOT detected on verify
     junk = tmpfile("junk")
-    assert_nothing_raised do
-      @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => false)
-    end
+    @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => false)
 
     # restoring no-corrupted data succeeds!
     @client.store_file(part1, nil, tmp.path)

data/test/test_mysql.rb

@@ -120,7 +120,7 @@ class TestMogileFS__Mysql < Test::Unit::TestCase
   end
 
   def test_sleep
-    assert_nothing_raised { assert_equal({}, @mg.sleep(:duration => 1)) }
+    assert_equal({}, @mg.sleep(:duration => 1))
   end
 
 end

metadata

@@ -1,66 +1,59 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mogilefs-client
-version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: 
-  segments: 
-  - 3
-  - 4
-  - 0
-  version: 3.4.0
+version: !ruby/object:Gem::Version
+  version: 3.5.0.rc1
+  prerelease: 6
 platform: ruby
-authors: 
+authors:
 - Eric Wong
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-10-09 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-10-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rdoc
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 19
-        segments: 
-        - 3
-        - 10
-        version: "3.10"
+      - !ruby/object:Gem::Version
+        version: '3.10'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: hoe
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 3
-        - 0
-        version: "3.0"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :development
-  version_requirements: *id002
-description: |-
-  A MogileFS client library for Ruby.  MogileFS is an open source
-  distributed filesystem, see: http://mogilefs.org for more details.  This
-  library allows any Ruby application to read, write and delete files in a
-  MogileFS instance.
-email: 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: ! '["A MogileFS client library for Ruby.  MogileFS is an open source\ndistributed
+  filesystem, see: http://mogilefs.org for more details.  This\nlibrary allows any
+  Ruby application to read, write and delete files in a\nMogileFS instance."]'
+email:
 - normalperson@yhbt.net
-executables: 
+executables:
 - mog
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - Manifest.txt
-files: 
+files:
 - .document
 - .gitignore
 - .wrongdoc.yml
@@ -92,6 +85,7 @@ files:
 - lib/mogilefs/new_file/stream.rb
 - lib/mogilefs/new_file/tempfile.rb
 - lib/mogilefs/new_file/writer.rb
+- lib/mogilefs/nhp_fake.rb
 - lib/mogilefs/paths_size.rb
 - lib/mogilefs/pool.rb
 - lib/mogilefs/socket.rb
@@ -131,53 +125,45 @@ files:
 - .gemtest
 homepage: http://bogomips.org/mogilefs-client/
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --main
 - README
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
 requirements: []
-
 rubyforge_project: seattlerb
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: MogileFS client library for Ruby
-test_files: 
-- test/test_http_reader.rb
-- test/test_mogilefs_integration_list_keys.rb
+test_files:
+- test/test_bigfile.rb
+- test/test_admin.rb
+- test/test_pool.rb
 - test/test_db_backend.rb
-- test/test_mogilefs_socket_pure.rb
-- test/test_mysql.rb
-- test/test_backend.rb
-- test/test_mogilefs.rb
+- test/test_mogilefs_integration_list_keys.rb
+- test/test_http_reader.rb
 - test/test_mogtool_bigfile.rb
+- test/test_backend.rb
 - test/test_client.rb
-- test/test_admin.rb
-- test/test_mogilefs_socket_kgio.rb
 - test/test_mogilefs_integration.rb
+- test/test_mogilefs_socket_pure.rb
+- test/test_mogilefs_socket_kgio.rb
 - test/test_fresh.rb
-- test/test_pool.rb
-- test/test_bigfile.rb
-- test/test_mogilefs_integration_large_pipe.rb
 - test/test_mogstored_rack.rb
+- test/test_mogilefs.rb
+- test/test_mogilefs_integration_large_pipe.rb
+- test/test_mysql.rb