mogilefs-client 3.0.0 → 3.1.0

data/lib/mogilefs/socket/pure_ruby.rb

@@ -24,7 +24,7 @@ class MogileFS::Socket < Socket
 
   def timed_read(len, dst = "", timeout = 5)
     begin
-      IO.select([self], nil, nil, timeout) or unreadable_socket!
+      IO.select([self], nil, nil, timeout) or unreadable_socket!(timeout)
       return read_nonblock(len, dst)
     rescue Errno::EAGAIN
     rescue EOFError
@@ -34,7 +34,7 @@ class MogileFS::Socket < Socket
 
   def timed_peek(len, dst, timeout = 5)
     begin
-      IO.select([self], nil, nil, timeout) or unreadable_socket!
+      IO.select([self], nil, nil, timeout) or unreadable_socket!(timeout)
       rc = recv_nonblock(len, Socket::MSG_PEEK)
       return rc.empty? ? nil : dst.replace(rc)
     rescue Errno::EAGAIN
@@ -62,7 +62,7 @@ class MogileFS::Socket < Socket
       end
     rescue Errno::EAGAIN
       IO.select(nil, [self], nil, timeout) or
-        request_truncated!(written, expect)
+        request_truncated!(written, expect, timeout)
     end while true
   end
 end

data/lib/mogilefs/socket_common.rb

@@ -12,14 +12,15 @@ module MogileFS::SocketCommon
     self
   end
 
-  def unreadable_socket!
+  def unreadable_socket!(timeout)
     raise MogileFS::UnreadableSocketError,
-          "#@mogilefs_addr never became readable"
+          "#@mogilefs_addr never became readable (timeout=#{timeout.inspect})"
   end
 
-  def request_truncated!(written, expect)
+  def request_truncated!(written, expect, timeout)
+    timeout = timeout.inspect
     raise MogileFS::RequestTruncatedError,
-          "request truncated (sent #{written} expected #{expect})"
+     "request truncated (sent #{written} expected #{expect} timeout=#{timeout})"
   end
 
   SEP_RE = /\A(.*?#{Regexp.escape("\n")})/
@@ -43,9 +44,9 @@ module MogileFS::SocketCommon
   def read(size, buf = "", timeout = 5)
     timed_read(size, buf, timeout) or return # nil/EOF
 
-    while size > buf.bytesize
+    while (size -= buf.bytesize) > 0
       tmp ||= ""
-      timed_read(size - buf.bytesize, tmp, timeout) or return buf # truncated
+      timed_read(size, tmp, timeout) or return buf # truncated
       buf << tmp
     end
 

data/test/fresh.rb

@@ -120,8 +120,8 @@ EOF
     args = { :ip => @test_host, :port => @mogstored_http_port }
     args[:status] = "alive"
     @admin.create_host("me", args)
-    Dir.mkdir("#@docroot/dev1")
-    Dir.mkdir("#@docroot/dev2")
+    assert File.directory?("#@docroot/dev1")
+    assert File.directory?("#@docroot/dev2")
     yield_for_monitor_update { @admin.get_hosts.empty? or break }
 
     # TODO: allow adding devices via our MogileFS::Admin class
@@ -136,6 +136,7 @@ EOF
       out.close! if out
       err.close! if err
       status, out, err = mogadm("check")
+      assert status.success?, status.inspect
       if (tries += 1) > 100
         warn err.read
         puts out.read
@@ -198,6 +199,8 @@ EOF
 
   def setup_mogstored
     @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
+    Dir.mkdir("#@docroot/dev1")
+    Dir.mkdir("#@docroot/dev2")
     @mogstored_mgmt = TCPServer.new(@test_host, 0)
     @mogstored_http = TCPServer.new(@test_host, 0)
     @mogstored_mgmt_port = @mogstored_mgmt.addr[1]

data/test/test_backend.rb

@@ -58,14 +58,14 @@ class TestBackend < Test::Unit::TestCase
     assert @backend.error('peb_kac')
     assert_equal MogileFS::Error, @backend.error('PebKacError').superclass
     assert MogileFS::Backend.const_defined?('PebKacError')
-  end
 
-  def test_size_verify_error_defined
-    # "ErrorError" just looks dumb, but we used to get it
-    # since mogilefs would send us "size_verify_error" and we'd
-    # blindly append "Error" to the exception
-    assert ! MogileFS::Backend.const_defined?('SizeVerifyErrorError')
-    assert MogileFS::Backend.const_defined?('SizeVerifyError')
+    assert_nothing_raised do
+      MogileFS::Backend::OMFGWTFBBQError
+    end
+    assert_equal MogileFS::Error, MogileFS::Backend::OMFGWTFBBQError.superclass
+    assert_raises(NameError) do
+      MogileFS::Backend::FailFailFail
+    end
   end
 
   def test_make_request
@@ -117,7 +117,7 @@ class TestBackend < Test::Unit::TestCase
     begin
       @backend.do_request 'foo', {}
     rescue MogileFS::UnreadableSocketError => e
-      assert_equal "127.0.0.1:#{port} never became readable", e.message
+      assert_match(/127\.0\.0\.1:#{port} never became readable/, e.message)
     rescue Exception => err
       flunk "MogileFS::UnreadableSocketError not raised #{err} #{err.backtrace}"
     else
@@ -221,5 +221,13 @@ class TestBackend < Test::Unit::TestCase
     assert_equal expected, actual
   end
 
+  def test_fail_timeout
+    o = { :domain => "none", :hosts => %w(0:666 0:6 0:66) }
+    c = MogileFS::MogileFS.new(o)
+    assert_equal 5, c.backend.instance_variable_get(:@fail_timeout)
+    o[:fail_timeout] = 0.666
+    c = MogileFS::MogileFS.new(o)
+    assert_equal 0.666, c.backend.instance_variable_get(:@fail_timeout)
+  end
 end
 

data/test/test_mogilefs_integration.rb

@@ -8,7 +8,9 @@ class TestMogileFSIntegration < TestMogIntegration
   end
 
   def test_CRUD
+    assert ! @client.exist?("CRUD")
     assert_equal 4, @client.store_content("CRUD", "default", "DATA")
+    assert @client.exist?("CRUD")
     assert_equal 4, @client.size("CRUD")
     assert_equal "DATA", @client.get_file_data("CRUD")
     assert_equal "DAT", @client.get_file_data("CRUD", nil, 3)
@@ -65,6 +67,9 @@ class TestMogileFSIntegration < TestMogIntegration
     def tmp.size
       666
     end
+    def tmp.read(len, buf = "")
+      raise Errno::EIO
+    end
 
     assert_raises(MogileFS::HTTPFile::NonRetryableError) do
       @client.store_file("non_rewindable", nil, tmp)
@@ -169,4 +174,120 @@ class TestMogileFSIntegration < TestMogIntegration
     end
     assert_equal count, seen.size
   end if ENV["TEST_EXPENSIVE"]
+
+  def test_new_file_no_block
+    rv = @client.new_file("no_block")
+    assert_nothing_raised { rv.write "HELLO" }
+    assert_nil rv.close
+    assert_equal "HELLO", @client.get_file_data("no_block")
+  end
+
+  def test_new_file_known_content_length
+    rv = @client.new_file("a", :content_length => 5)
+    assert_nothing_raised { rv.write "HELLO" }
+    assert_nil rv.close
+    assert_equal "HELLO", @client.get_file_data("a")
+
+    rv = @client.new_file("a", :content_length => 6)
+    assert_nothing_raised { rv.write "GOOD" }
+    assert_raises(MogileFS::SizeMismatchError) { rv.close }
+    assert_equal "HELLO", @client.get_file_data("a")
+
+    rv = @client.new_file("large", :content_length => 6, :largefile => true)
+    assert_instance_of MogileFS::NewFile::Stream, rv
+    assert_equal 6, rv.write("HIHIHI")
+    assert_nil rv.close
+    assert_equal "HIHIHI", @client.get_file_data("large")
+  end
+
+  def test_new_file_content_md5
+    r, w = IO.pipe
+    b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
+    rv = @client.new_file("a", :content_md5 => b64digest, :content_length => 5)
+    assert_nothing_raised { rv.write "HELLO" }
+    assert_nil rv.close
+    assert_equal "HELLO", @client.get_file_data("a")
+
+    assert_nothing_raised { w.write "HIHI"; w.close }
+    assert_raises(ArgumentError) do
+      @client.new_file("a", :content_md5 => b64digest) { |f| f.big_io = r }
+    end
+    assert_equal "HELLO", @client.get_file_data("a")
+
+    assert_nothing_raised do
+      @client.new_file("a", :content_md5 => :trailer) { |f| f.big_io = r }
+    end
+    assert_equal "HIHI", @client.get_file_data("a")
+
+    # legacy, in case anybody used it
+    rv = @client.new_file("a",{:class => "default"}, 6)
+    assert_equal 2, rv.write("HI")
+    assert_raises(MogileFS::SizeMismatchError) { rv.close }
+    assert_equal "HIHI", @client.get_file_data("a")
+
+    rv = @client.new_file("a",{:class => "default"}, 2)
+    assert_equal 2, rv.write("HI")
+    assert_nil rv.close
+    assert_equal "HI", @client.get_file_data("a")
+    assert_raises(MogileFS::Backend::UnregClassError) {
+      @client.new_file("a", "non-existent", 2)
+    }
+    assert_raises(MogileFS::Backend::UnregClassError) {
+      @client.new_file("a", :class => "non-existent")
+    }
+    ensure
+      r.close if r
+  end
+
+  def test_store_content_opts
+    b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
+    assert_nothing_raised do
+      @client.store_content("c", nil, "HELLO", :content_md5 => b64digest)
+    end
+    assert_raises(MogileFS::SizeMismatchError) do
+      @client.store_content("c", nil, "GOODBYE", :content_length => 2)
+    end
+    assert_equal "HELLO", @client.get_file_data("c")
+  end
+
+  def test_store_file_opts
+    b64digest = [ Digest::MD5.digest("HELLO") ].pack('m').strip
+    io = StringIO.new("HELLO")
+    assert_nothing_raised do
+      @client.store_file("c", nil, io, :content_md5 => b64digest)
+    end
+
+    io = StringIO.new("GOODBYE")
+    assert_raises(MogileFS::SizeMismatchError) do
+      @client.store_content("c", nil, io, :content_length => 2)
+    end
+    assert_equal "HELLO", @client.get_file_data("c")
+  end
+
+  def test_store_file_content_md5_lambda
+    checked = false
+    expect_md5 = lambda do
+      checked = true
+      [ Digest::MD5.digest("HELLO") ].pack('m').strip
+    end
+    io = StringIO.new("HELLO")
+    assert_nothing_raised do
+      @client.store_file("c", nil, io, :content_md5 => expect_md5)
+    end
+
+    assert_equal true, checked, "expect_md5 lambda called"
+  end
+
+  def test_store_file_unlinked_tempfile
+    tmp = Tempfile.new("store_file_unlinked")
+    tmp.unlink
+    tmp.sync = true
+    tmp.write "HIHI"
+    tmp.rewind
+    assert_nothing_raised do
+      @client.store_file("unlinked", nil, tmp)
+    end
+
+    assert_equal "HIHI", @client.get_file_data("unlinked")
+  end
 end

data/test/test_mogstored_rack.rb

@@ -1,5 +1,12 @@
 # -*- encoding: binary -*-
 require "./test/fresh"
+begin
+  require 'mogstored_rack'
+  require 'unicorn'
+  ok = true
+rescue LoadError
+  ok = false
+end
 
 class TestMogstoredRack < Test::Unit::TestCase
   include TestFreshSetup
@@ -7,6 +14,106 @@ class TestMogstoredRack < Test::Unit::TestCase
     setup_mogilefs
   end
 
+  def test_range_put_new_file
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+
+    io = client.new_file "range0", :largefile => :content_range
+    assert_nil io.close
+    assert_equal "", client.get_file_data("range0")
+
+    io = client.new_file "writes", :largefile => :content_range
+    %w(a b c d e).each { |x| io.write(x) }
+    assert_nil io.close
+    assert_equal "abcde", client.get_file_data("writes")
+
+    io = client.new_file "puts", :largefile => :content_range
+    %w(a b c d e).each { |x| io.puts(x) }
+    assert ! client.exist?("puts")
+    assert_nil io.close
+    assert_equal "a\nb\nc\nd\ne\n", client.get_file_data("puts")
+  end
+
+  def test_stream_new_file
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    client.new_file("chunky", :largefile => :stream) do |io|
+      assert_instance_of MogileFS::NewFile::Stream, io
+      assert_equal(5, io.write("HELLO"))
+      assert_nil io.md5
+    end
+    assert_equal "HELLO", client.get_file_data("chunky")
+
+    io = client.new_file("puts", :largefile => :stream)
+    assert_instance_of MogileFS::NewFile::Stream, io
+    assert_equal io, IO.select(nil, [io])[1][0], "IO.select-able"
+
+    assert_nil(io.puts("PUTS!"))
+    assert_nil(io.puts("PUTZ"))
+    assert_nil io.close
+    assert_equal "PUTS!\nPUTZ\n", client.get_file_data("puts")
+
+    io = client.new_file("putc", :largefile => :stream)
+    assert_equal(0x20, io.putc(0x20))
+    assert_nil io.close
+    assert_equal " ", client.get_file_data("putc")
+
+    io = client.new_file("print splat", :largefile => :stream)
+    io.print(1, 2, 3)
+    assert_nil io.close
+    assert_equal "123", client.get_file_data("print splat")
+
+    io = client.new_file("printf", :largefile => :stream)
+    assert_nil io.printf("%x", 1638)
+    assert_nil io.close
+    assert_equal "666", client.get_file_data("printf")
+
+    io = client.new_file("syswrite", :largefile => :stream)
+    assert_equal 4, io.syswrite("good")
+    assert_equal 7, io.syswrite("morning")
+    assert_nil io.close
+    assert_equal "goodmorning", client.get_file_data("syswrite")
+
+    io = client.new_file("md5", :largefile=>:stream, :content_md5=>:trailer)
+    assert_instance_of Digest::MD5, io.md5
+    assert_nil io.puts("HIHI")
+    assert_nil io.close
+    assert_equal "HIHI\n", client.get_file_data("md5")
+    assert_equal Digest::MD5.hexdigest("HIHI\n"), io.md5.hexdigest
+
+    io = client.new_file("<<", :largefile=>:stream)
+    assert_equal(io, io << ">>")
+    assert_nil io.close
+    assert_equal ">>", client.get_file_data("<<")
+  end
+
+  def test_stream_new_file_with_content_length
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    io = client.new_file("clen", :largefile=>:stream,:content_length=>6)
+    io << "HIHIHI"
+    assert_nil io.close
+    assert_equal "HIHIHI", client.get_file_data("clen")
+
+    io = client.new_file("clen", :largefile=>:stream,:content_length=>1)
+    io << "FAIL"
+    assert_raises(MogileFS::SizeMismatchError) { io.close }
+    assert_equal "HIHIHI", client.get_file_data("clen")
+
+    io = client.new_file("md5", :largefile=>:stream,
+                                :content_length=>6, :content_md5=>:trailer)
+    assert_equal(io, io << "MD5MD5")
+    assert_nil io.close
+    assert_equal "MD5MD5", client.get_file_data("md5")
+    assert_equal Digest::MD5.hexdigest("MD5MD5"), io.md5.hexdigest
+
+    io = client.new_file("md5", :largefile=>:stream,
+                                :content_length=>6, :content_md5=>:trailer)
+    assert_equal(io, io << "MD5MD")
+    assert_raises(MogileFS::SizeMismatchError) { io.close }
+    assert_equal Digest::MD5.hexdigest("MD5MD"), io.md5.hexdigest
+  end
+
   def test_md5_check
     add_host_device_domain
     client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
@@ -23,6 +130,8 @@ class TestMogstoredRack < Test::Unit::TestCase
 
   def setup_mogstored
     @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
+    Dir.mkdir("#@docroot/dev1")
+    Dir.mkdir("#@docroot/dev2")
     @mogstored_mgmt = TCPServer.new(@test_host, 0)
     @mogstored_http = TCPServer.new(@test_host, 0)
     @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
@@ -47,12 +156,8 @@ EOF
 
   # I would use Rainbows! + *Threads + Ruby 1.9.3 in production
   def unicorn_setup
-    examples_dir = Dir.pwd + "/examples"
-    assert File.directory?(examples_dir)
     @ru = Tempfile.new(%w(mogstored_rack .ru))
     @ru.write <<EOF
-require "mogstored_rack"
-use Rack::Head
 run MogstoredRack.new("#@docroot")
 EOF
     @ru.flush
@@ -62,6 +167,7 @@ EOF
     @unicorn_stderr = Tempfile.new(%w(unicorn .stderr))
     @unicorn_stdout = Tempfile.new(%w(unicorn .stdout))
     @unicorn_conf.write <<EOF
+require "mogstored_rack"
 listen "#@test_host:#{@mogstored_http_port}"
 pid "#{@unicorn_pid.path}"
 stderr_path "#{@unicorn_stderr.path}"
@@ -71,7 +177,7 @@ EOF
     @unicorn_conf.flush
 
     @mogstored_http.close
-    x!("unicorn", "-I", examples_dir, "-E", "deployment",
+    x!("unicorn", "-E", "deployment",
        "--daemon", "--config", @unicorn_conf.path, @ru.path)
     wait_for_port @mogstored_http_port
     40.times do
@@ -86,4 +192,4 @@ EOF
     teardown_mogilefs
     puts(@unicorn_stderr.read) if $DEBUG
   end
-end if `which unicorn`.chomp.size > 0
+end if ok && `which unicorn`.chomp.size > 0

data/test/test_pool.rb

@@ -19,6 +19,22 @@ class ResourceWithArgs
   end
 
 end
+class PoolClient < MogileFS::Client
+  attr_reader :alive
+
+  def initialize(*args)
+    @args = args
+    @alive = true
+  end
+
+  def backend
+    self
+  end
+
+  def shutdown
+    @alive = false
+  end
+end
 
 class TestPool < Test::Unit::TestCase
 
@@ -95,5 +111,15 @@ class TestPool < Test::Unit::TestCase
     assert_equal o1, o2, "Objects must be reused"
   end
 
+  def test_auto_shutdown
+    pool = MogileFS::Pool.new(PoolClient, 666)
+    tmp = []
+    6.times { tmp << pool.get }
+    tmp.each { |obj| pool.put(obj) }
+    alive = Hash.new { |h,k| h[k] = 0 }
+    tmp.each { |obj| alive[obj.alive] += 1 }
+    assert_equal 3, alive[true]
+    assert_equal 3, alive[false]
+  end
 end