mogilefs-client 2.2.0 → 3.0.0.rc1

data/test/test_mogilefs_integration.rb

@@ -0,0 +1,174 @@
+# -*- encoding: binary -*-
+require './test/integration'
+
+class TestMogileFSIntegration < TestMogIntegration
+  def setup
+    super
+    @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
+  end
+
+  def test_CRUD
+    assert_equal 4, @client.store_content("CRUD", "default", "DATA")
+    assert_equal 4, @client.size("CRUD")
+    assert_equal "DATA", @client.get_file_data("CRUD")
+    assert_equal "DAT", @client.get_file_data("CRUD", nil, 3)
+    assert_equal "AT", @client.get_file_data("CRUD", nil, 2, 1)
+
+    tmp = Tempfile.new("z")
+    tmp_path = tmp.path
+    tmp.close!
+    assert_equal 4, @client.get_file_data("CRUD", tmp_path)
+    assert_equal "DATA", File.read(tmp_path)
+    st = File.stat(tmp_path)
+    assert_equal 0100600, st.mode
+    File.unlink(tmp_path)
+
+    sio = StringIO.new("")
+    rv = @client.get_file_data("CRUD", sio)
+    assert_equal 4, rv
+    assert_equal "DATA", sio.string
+    assert_equal 8, @client.store_content("CRUD", "default", "MOARDATA")
+    assert_equal "MOARDATA", @client.get_file_data("CRUD")
+    assert_equal true, @client.delete("CRUD")
+    assert_raises(MogileFS::Backend::UnknownKeyError) { @client.delete("CRUD") }
+
+    data = "hello world\n".freeze
+    tmp = tmpfile("blob")
+    tmp.sync = true
+    tmp.write(data)
+    tmp.rewind
+    assert_equal tmp.size, @client.store_file("blob", nil, tmp)
+    assert_equal(data, @client.get_file_data("blob"))
+
+    data = "pipe!\n".freeze
+    r, w = IO.pipe
+    th = Thread.new do
+      w.write(data)
+      w.close
+    end
+    assert_equal data.size, @client.store_file("pipe", nil, r)
+    assert_nothing_raised do
+      r.close
+      th.join
+    end
+    assert_equal(data, @client.get_file_data("pipe"))
+
+    cbk = MogileFS::Util::StoreContent.new(nil) do |write_callback|
+      10.times { write_callback.call("data") }
+    end
+    assert_nil cbk.length
+    nr = @client.store_content('store_content', nil, cbk)
+    assert_equal 40, nr
+    assert_equal("data" * 10, @client.get_file_data('store_content'))
+  end
+
+  def test_store_non_rewindable
+    tmp = Object.new
+    def tmp.size
+      666
+    end
+
+    assert_raises(MogileFS::HTTPFile::NonRetryableError) do
+      @client.store_file("non_rewindable", nil, tmp)
+    end
+  end
+
+  def test_file_info
+    assert_equal 3, @client.store_content("file_info", "default", "FOO")
+    res = @client.file_info("file_info")
+    assert_kind_of Integer, res["fid"]
+    assert_equal 3, res["length"]
+    assert ! res.include?("devids")
+    assert_kind_of Integer, res["devcount"]
+
+    res = @client.file_info("file_info", :devices => true)
+    assert_kind_of Integer, res["fid"]
+    assert_equal 3, res["length"]
+    assert_kind_of Integer, res["devcount"]
+    devids = res.delete("devids")
+    assert_instance_of Array, devids
+    devids.each { |devid| assert_kind_of Integer, devid }
+    assert_equal res["devcount"], devids.size
+  end
+
+  def test_file_debug
+    assert_equal 3, @client.store_content("file_debug", "default", "BUG")
+    a = @client.file_debug("file_debug")
+    b = @client.file_debug(:key => "file_debug")
+    fid = @client.file_info("file_debug")["fid"]
+    c = @client.file_debug(fid)
+    d = @client.file_debug(:fid => fid)
+
+    [ a, b, c, d ].each do |res|
+      assert_equal fid, res["fid_fid"]
+      assert_equal 0, res["fid_classid"]
+      assert_equal "file_debug", res["fid_dkey"]
+      assert_equal 3, res["fid_length"]
+      assert_kind_of Array, res["devids"]
+      assert_kind_of Integer, res["devids"][0]
+      res["devids"].each do |devid|
+        uri = URI.parse(res["devpath_#{devid}"])
+        assert_equal "http", uri.scheme
+      end
+      assert_equal "default", res["fid_class"]
+    end
+    @client.delete("file_debug")
+    rv = @client.file_debug(fid)
+    assert rv.keys.grep(/\Afid_/).empty?, rv.inspect
+  end
+
+  def test_file_debug_in_progress
+    rv = @client.new_file("file_debug_in_progress") do |http_file|
+      http_file << "ZZZZ"
+      dests = http_file.instance_variable_get(:@dests)
+      dests[0][1] =~ %r{/(\d+)\.fid\z}
+      fid = $1.to_i
+      rv = @client.file_debug(fid)
+      devids = dests.map { |x| x[0].to_i }.sort
+      assert_equal devids, rv["tempfile_devids"].sort
+      assert_equal "file_debug_in_progress", rv["tempfile_dkey"]
+    end
+    assert_equal 4, rv
+  end
+
+  def test_admin_get_devices
+    admin = MogileFS::Admin.new(:hosts => @trackers)
+    devices = admin.get_devices
+    if any_device = devices[0]
+      %w(mb_asof mb_free mb_used mb_total devid weight hostid).each do |field|
+        case value = any_device[field]
+        when nil
+        when Integer
+          assert value >= 0, "#{field}=#{value.inspect} is negative"
+        else
+          assert false, "#{field}=#{value.inspect} is #{value.class}"
+        end
+      end
+
+      field = "utilization"
+      case value = any_device[field]
+      when nil
+      when Float
+        assert value >= 0.0, "#{field}=#{value.inspect} is negative"
+      else
+        assert false, "#{field}=#{value.inspect} is #{value.class}"
+      end
+    end
+  end
+
+  # TODO: move this to a fresh instance
+  def test_admin_each_fid
+    admin = MogileFS::Admin.new(:hosts => @trackers)
+    seen = {}
+    count = admin.each_fid do |info|
+      seen[info["fid"]] = true
+      assert_kind_of Integer, info["fid"]
+      assert_kind_of Integer, info["length"]
+      assert_kind_of Integer, info["devcount"]
+      assert_kind_of String, info["key"]
+      assert_kind_of String, info["class"]
+      assert_kind_of String, info["domain"]
+    end
+    assert_equal count, seen.size
+  end if ENV["TEST_EXPENSIVE"]
+end

data/test/test_mogilefs_integration_large_pipe.rb

@@ -0,0 +1,62 @@
+# -*- encoding: binary -*-
+require './test/integration'
+require "digest/sha1"
+
+class TestMogileFSLargePipe< TestMogIntegration
+  def setup
+    super
+    @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
+  end
+
+  def test_large_pipe_test
+    junk = File.open("/dev/urandom") { |fp| fp.read(1024) }
+    junk *= 32
+    nr = rand(666) + 1024
+    r, w = IO.pipe
+    sha1 = Digest::SHA1.new
+    th = Thread.new do
+      nr.times do
+        sha1.update(junk)
+        w.write(junk)
+      end
+      w.close
+    end
+    assert_equal(nr * junk.size, @client.store_file("a", nil, r))
+    r.close
+    th.join
+    @client.get_file_data("a") do |rd|
+      assert_equal(nr * junk.size, @client.store_file("b", nil, rd))
+    end
+    a = Thread.new { @client.get_file_data("a") { |rd| sha1read(rd) } }
+    b = Thread.new { @client.get_file_data("b") { |rd| sha1read(rd) } }
+    a = a.value
+    b = b.value
+    assert_equal a, b
+    assert_equal sha1, a
+
+    # We should be able to open FIFOs
+    tmp = tmpfile("fifo")
+    tmp_path = tmp.path
+    File.unlink(tmp_path)
+    x!("mkfifo", tmp_path)
+    pid = fork do
+      File.open(tmp_path, "wb") do |wr|
+        nr.times { wr.write(junk) }
+      end
+    end
+    assert_equal(nr * junk.size, @client.store_file("fifo", nil, tmp_path))
+    _, status = Process.waitpid2(pid)
+    assert status.success?, status.inspect
+    fifo_sha1 = @client.get_file_data("fifo") { |rd| sha1read(rd) }
+    assert_equal sha1, fifo_sha1
+  end
+
+  def sha1read(rd)
+    buf = ""
+    d = Digest::SHA1.new
+    while rd.read(16384, buf)
+      d << buf
+    end
+    d
+  end
+end

data/test/test_mogilefs_integration_list_keys.rb

@@ -0,0 +1,40 @@
+
+# -*- encoding: binary -*-
+require './test/integration'
+
+class TestMogileFSIntegrationListKeys < TestMogIntegration
+  def setup
+    super
+    @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
+  end
+
+  def test_list_keys
+    k = %w(a b c d e f g)
+    k.each { |x| @client.store_content("lk_#{x}", nil, x) }
+    expect = k.map { |x| "lk_#{x}" }
+    rv = @client.list_keys
+    assert_equal([ expect, expect.last ] , rv)
+    nr = 0
+    @client.list_keys do |key, length, devcount|
+      assert_equal 1, length
+      assert_kind_of Integer, devcount
+      assert_equal expect[nr], key
+      nr += 1
+    end
+  end
+
+  def test_list_keys_strange
+    @client.store_content("hello+world", nil, "HI")
+    rv = @client.list_keys
+    assert_equal "hello+world", rv[0][0]
+  end
+
+  def test_each_key
+    9.times { |i| @client.store_content("ek_#{i}", nil, i.to_s) }
+    n = 0
+    @client.each_key do |key|
+      assert_equal "ek_#{n.to_s}", key
+      n += 1
+    end
+  end
+end

data/test/test_mogilefs_socket_kgio.rb

@@ -0,0 +1,11 @@
+require "./test/socket_test"
+require "mogilefs"
+begin
+  require "kgio"
+  require "mogilefs/socket/kgio"
+rescue LoadError
+end unless ENV["MOGILEFS_CLIENT_PURE"]
+
+class TestSocketKgio < Test::Unit::TestCase
+  include SocketTest
+end if defined?(Kgio)

data/test/test_mogilefs_socket_pure.rb

@@ -0,0 +1,7 @@
+ENV["MOGILEFS_CLIENT_PURE"] = "true"
+require "./test/socket_test"
+require "mogilefs"
+
+class TestSocketPure < Test::Unit::TestCase
+  include SocketTest
+end

data/test/test_mogstored_rack.rb

@@ -0,0 +1,89 @@
+# -*- encoding: binary -*-
+require "./test/fresh"
+
+class TestMogstoredRack < Test::Unit::TestCase
+  include TestFreshSetup
+  def setup
+    setup_mogilefs
+  end
+
+  def test_md5_check
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    node = "#@test_host:#@mogstored_http_port"
+    pid = fork do
+      # not modifying this hash in the same process
+      MogileFS::HTTPFile::MD5_TRAILER_NODES[node] = true
+      client.store_content("md5_me", nil, "HELLO WORLD")
+    end
+    _, status = Process.waitpid2(pid)
+    assert status.success?, status.inspect
+    assert_equal "HELLO WORLD", client.get_file_data("md5_me")
+  end
+
+  def setup_mogstored
+    @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
+    @mogstored_mgmt = TCPServer.new(@test_host, 0)
+    @mogstored_http = TCPServer.new(@test_host, 0)
+    @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
+    @mogstored_http_port = @mogstored_http.addr[1]
+    @mogstored_conf = Tempfile.new(["mogstored", "conf"])
+    @mogstored_pid = Tempfile.new(["mogstored", "pid"])
+    @mogstored_conf.write <<EOF
+pidfile = #{@mogstored_pid.path}
+maxconns = 1000
+mgmtlisten = #@test_host:#{@mogstored_mgmt_port}
+server = none
+docroot = #@docroot
+EOF
+    @mogstored_conf.flush
+    @mogstored_mgmt.close
+
+    unicorn_setup
+
+    x!("mogstored", "--daemon", "--config=#{@mogstored_conf.path}")
+    wait_for_port @mogstored_mgmt_port
+  end
+
+  # I would use Rainbows! + *Threads + Ruby 1.9.3 in production
+  def unicorn_setup
+    examples_dir = Dir.pwd + "/examples"
+    assert File.directory?(examples_dir)
+    @ru = Tempfile.new(%w(mogstored_rack .ru))
+    @ru.write <<EOF
+require "mogstored_rack"
+use Rack::Head
+run MogstoredRack.new("#@docroot")
+EOF
+    @ru.flush
+
+    @unicorn_pid = Tempfile.new(%w(unicorn .pid))
+    @unicorn_conf = Tempfile.new(%w(unicorn.conf .rb))
+    @unicorn_stderr = Tempfile.new(%w(unicorn .stderr))
+    @unicorn_stdout = Tempfile.new(%w(unicorn .stdout))
+    @unicorn_conf.write <<EOF
+listen "#@test_host:#{@mogstored_http_port}"
+pid "#{@unicorn_pid.path}"
+stderr_path "#{@unicorn_stderr.path}"
+stdout_path "#{@unicorn_stdout.path}"
+rewindable_input false
+EOF
+    @unicorn_conf.flush
+
+    @mogstored_http.close
+    x!("unicorn", "-I", examples_dir, "-E", "deployment",
+       "--daemon", "--config", @unicorn_conf.path, @ru.path)
+    wait_for_port @mogstored_http_port
+    40.times do
+      break if File.size(@unicorn_pid.path) > 0
+      sleep 0.1
+    end
+  end
+
+  def teardown
+    pid = File.read(@unicorn_pid.path).to_i
+    Process.kill(:QUIT, pid) if pid > 0
+    teardown_mogilefs
+    puts(@unicorn_stderr.read) if $DEBUG
+  end
+end if `which unicorn`.chomp.size > 0

data/test/test_mogtool_bigfile.rb

@@ -0,0 +1,116 @@
+# -*- encoding: binary -*-
+require "./test/integration"
+require "net/http"
+ok = true
+unless File.executable?(`which mogtool 2>/dev/null`.strip)
+  warn "mogtool not found, skipping #{__FILE__}"
+  ok = false
+end
+
+class TestMogtoolBigfile < TestMogIntegration
+  buf = File.open("/dev/urandom") { |fp| fp.read(1024) }
+  buf *= 1024
+  RAND = Tempfile.new("rand")
+  RAND.sync = true
+  sha1 = Digest::SHA1.new
+  100.times { sha1 << buf; RAND.write(buf) }
+  buf = nil
+  RAND_SHA1 = sha1.hexdigest
+
+  def setup
+    super
+    RAND.rewind
+    @big_uuid = "big-#{uuid}"
+    @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
+  end
+
+  def mogtool!(*args)
+    x!("mogtool", "--trackers=#{@trackers.join(',')}",
+       "--domain=#@domain", *args)
+  end
+
+  # the mogtool definition of gzip is wrong and just raw zlib deflate
+  def test_bigfile_gzip_mogtool
+    mogtool!("inject", "--gzip", "--bigfile", RAND.path, @big_uuid)
+    sha1_check
+  end
+
+  def test_bigfile_mogtool
+    mogtool!("inject", "--bigfile", RAND.path, @big_uuid)
+    sha1_check
+
+    # ensure fallback works for rebalanced/replaced files
+    part1 = "#@big_uuid,1"
+    tmp = tmpfile("part1")
+    before_uris = @client.get_uris(part1)
+    @client.get_file_data(part1, tmp)
+    @client.delete(part1)
+    @client.store_file(part1, nil, tmp.path)
+    wait_for_DELETE(before_uris)
+    sha1_check
+
+    # corrupt the existing data in part1
+    @client.store_content(part1, nil, "HELLO")
+    assert_nothing_raised { @client.get_uris(part1) }
+
+    # corruption is detected on verify
+    junk = tmpfile("junk")
+    assert_raises(MogileFS::ChecksumMismatchError) do
+      @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => true)
+    end
+
+    # corruption is NOT detected on verify
+    junk = tmpfile("junk")
+    assert_nothing_raised do
+      @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => false)
+    end
+
+    # restoring no-corrupted data succeeds!
+    @client.store_file(part1, nil, tmp.path)
+    sha1_check
+
+    # missing parts fail
+    before_uris = @client.get_uris(part1)
+    @client.delete(part1)
+    junk = tmpfile("junk")
+    assert_raises(MogileFS::Backend::UnknownKeyError) do
+      @client.bigfile_write("_big_info:#@big_uuid", junk, :verify => true)
+    end
+  end
+
+  def wait_for_DELETE(uris)
+    uris.each do |uri|
+      tries = 0
+      begin
+        Net::HTTP.start(uri.host, uri.port) do |http|
+          sleep(0.1) while Net::HTTPOK === http.head(uri.path)
+        end
+      rescue
+        if (tries += 1) < 666
+          sleep(0.1)
+          retry
+        end
+        raise
+      end
+    end
+  end
+
+  def sha1_check
+    r, w = IO.pipe
+    @to_close << r
+    @to_close << w
+    th = Thread.new do
+      sha1 = Digest::SHA1.new
+      buf = ""
+      while r.read(16384, buf)
+        sha1 << buf
+      end
+      sha1.hexdigest
+    end
+    res = @client.bigfile_write("_big_info:#@big_uuid", w, :verify => true)
+    w.close
+    read_sha1 = th.value
+    assert_equal RAND_SHA1, read_sha1
+    assert_equal RAND.size, res[0]
+  end
+end if ok