mogilefs-client 2.2.0 → 3.0.0.rc1

data/test/exec.rb

@@ -0,0 +1,72 @@
+# -*- encoding: binary -*-
+$stdout.sync = $stderr.sync = true
+Thread.abort_on_exception = true
+require 'test/unit'
+require 'securerandom'
+require 'tempfile'
+require 'digest'
+require 'stringio'
+require 'pp'
+require 'mogilefs'
+
+module TestExec
+  def uuid
+    SecureRandom.uuid
+  rescue NoMethodError
+    ary = SecureRandom.random_bytes(16).unpack("NnnnnN")
+    ary[2] = (ary[2] & 0x0fff) | 0x4000
+    ary[3] = (ary[3] & 0x3fff) | 0x8000
+    "%08x-%04x-%04x-%04x-%04x%08x" % ary
+  end
+
+  def yield_for_monitor_update # mogilefsd should update every 4 seconds
+    50.times do
+      yield
+      sleep 0.1
+    end
+  end
+
+  def mogadm(*args)
+    x("mogadm", "--trackers=#{@trackers.join(',')}", *args)
+  end
+
+  def x(*cmd)
+    out, err = tmpfile("out"), tmpfile("err")
+    puts cmd.join(' ') if $VERBOSE
+    pid = fork do
+      $stderr.reopen(err.path, "a")
+      $stdout.reopen(out.path, "a")
+      out.close
+      err.close
+      ObjectSpace.each_object(Tempfile) do |tmp|
+        next if tmp.closed?
+        ObjectSpace.undefine_finalizer(tmp)
+        tmp.close_on_exec = true if tmp.respond_to?(:close_on_exec=)
+      end
+      exec(*cmd)
+    end
+    _, status = Process.waitpid2(pid)
+    out.rewind
+    err.rewind
+    [ status, out, err ]
+  end
+
+  def mogadm!(*args)
+    status, out, err = mogadm(*args)
+    assert status.success?, "#{status.inspect} / #{out.read} / #{err.read}"
+    [ status, out, err ]
+  end
+
+  def x!(*cmd)
+    status, out, err = x(*cmd)
+    assert status.success?, "#{status.inspect} / #{out.read} / #{err.read}"
+    [ status, out, err ]
+  end
+
+  def tmpfile(name)
+    tmp = Tempfile.new(name)
+    defined?(@to_close) or @to_close = []
+    @to_close << tmp
+    tmp
+  end
+end

data/test/fresh.rb

@@ -0,0 +1,222 @@
+# -*- encoding: binary -*-
+require "./test/exec"
+require "tmpdir"
+require "fileutils"
+require "net/http"
+
+module TestFreshSetup
+  include TestExec
+
+  def setup
+    setup_mogilefs
+  end
+
+  def setup_mogilefs(plugins = nil)
+    @test_host = "127.0.0.1"
+    setup_mogstored
+    @tracker = TCPServer.new(@test_host, 0)
+    @tracker_port = @tracker.addr[1]
+
+    @dbname = Tempfile.new(["mogfresh", ".sqlite3"])
+    @mogilefsd_conf = Tempfile.new(["mogilefsd", "conf"])
+    @mogilefsd_pid = Tempfile.new(["mogilefsd", "pid"])
+
+    cmd = %w(mogdbsetup --yes --type=SQLite --dbname) << @dbname.path
+    x!(*cmd)
+
+    @mogilefsd_conf.puts "db_dsn DBI:SQLite:#{@dbname.path}"
+    @mogilefsd_conf.write <<EOF
+conf_port #@tracker_port
+listen #@test_host
+pidfile #{@mogilefsd_pid.path}
+replicate_jobs 1
+fsck_jobs 1
+query_jobs 1
+mogstored_stream_port #{@mogstored_mgmt_port}
+node_timeout 10
+EOF
+    @mogilefsd_conf.flush
+
+    @trackers = @hosts = [ "#@test_host:#@tracker_port" ]
+    @tracker.close
+    x!("mogilefsd", "--daemon", "--config=#{@mogilefsd_conf.path}")
+    wait_for_port @tracker_port
+    @admin = MogileFS::Admin.new(:hosts => @hosts)
+    50.times do
+      break if File.size(@mogstored_pid.path) > 0
+      sleep 0.1
+    end
+  end
+
+  def wait_for_port(port)
+    tries = 50
+    begin
+      TCPSocket.new(@test_host, port).close
+      return
+    rescue
+      sleep 0.1
+    end while (tries -= 1) > 0
+    raise "#@test_host:#{port} never became ready"
+  end
+
+  def test_admin_setup_new_host_and_devices
+    assert_equal [], @admin.get_hosts
+    args = { :ip => @test_host, :port => @mogstored_http_port }
+    @admin.create_host("me", args)
+    yield_for_monitor_update { @admin.get_hosts.empty? or break }
+    hosts = @admin.get_hosts
+    assert_equal 1, hosts.size
+    host = @admin.get_hosts[0]
+    assert_equal "me", host["hostname"]
+    assert_equal @mogstored_http_port, host["http_port"]
+    assert_nil host["http_get_port"]
+    assert_equal @test_host, host["hostip"]
+    assert_kind_of Integer, host["hostid"]
+    assert_equal hosts, @admin.get_hosts(host["hostid"])
+
+    assert_equal [], @admin.get_devices
+  end
+
+  def test_replicate_now
+    assert_equal({"count" => 0}, @admin.replicate_now)
+  end
+
+  def test_clear_cache
+    assert_nil @admin.clear_cache
+  end
+
+  def test_create_update_delete_class
+    domain = "rbmogtest#{Time.now.strftime('%Y%m%d%H%M%S')}.#{uuid}"
+    @admin.create_domain(domain)
+    yield_for_monitor_update { @admin.get_domains.include?(domain) and break }
+
+    assert_nothing_raised do
+      @admin.create_class(domain, "klassy", 1)
+    end
+    assert_raises(MogileFS::Backend::ClassExistsError) do
+      @admin.create_class(domain, "klassy", 1)
+    end
+
+    assert_nothing_raised do
+      @admin.update_class(domain, "klassy",
+                          :mindevcount => 1, :replpolicy => "MultipleHosts(1)")
+    end
+
+    tmp = nil
+    yield_for_monitor_update do
+      tmp = @admin.get_domains[domain]["klassy"]
+      break if tmp && tmp["replpolicy"] == "MultipleHosts(1)"
+    end
+    assert tmp, "domain did not show up"
+    assert_equal 1, tmp["mindevcount"]
+    assert_equal "MultipleHosts(1)", tmp["replpolicy"]
+    assert_nothing_raised { @admin.update_class(domain, "klassy", 2) }
+    ensure
+      @admin.delete_class(domain, "klassy") rescue nil
+  end
+
+  def add_host_device_domain
+    assert_equal [], @admin.get_hosts
+    args = { :ip => @test_host, :port => @mogstored_http_port }
+    args[:status] = "alive"
+    @admin.create_host("me", args)
+    Dir.mkdir("#@docroot/dev1")
+    Dir.mkdir("#@docroot/dev2")
+    yield_for_monitor_update { @admin.get_hosts.empty? or break }
+
+    # TODO: allow adding devices via our MogileFS::Admin class
+    mogadm!("device", "add", "me", "dev1")
+    yield_for_monitor_update { @admin.get_devices.empty? or break }
+    wait_for_usage_file "dev1"
+    mogadm!("device", "add", "me", "dev2")
+    wait_for_usage_file "dev2"
+    out = err = nil
+    tries = 0
+    begin
+      out.close! if out
+      err.close! if err
+      status, out, err = mogadm("check")
+      if (tries += 1) > 100
+        warn err.read
+        puts out.read
+        raise "mogadm failed"
+      end
+      sleep 0.1
+    end until out.read =~ /write?able/
+
+    domain = "rbmogtest.#$$"
+    @admin.create_domain(domain)
+    yield_for_monitor_update { @admin.get_domains.include?(domain) and break }
+    @domain = domain
+  end
+
+  def test_device_file_add
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    r, w = IO.pipe
+    thr = Thread.new do
+      (0..9).each do |i|
+        sleep 0.05
+        w.write("#{i}\n")
+      end
+      w.close
+      :ok
+    end
+    assert_equal 20, client.store_file("pipe", nil, r)
+    assert_equal :ok, thr.value
+    r.close
+    assert_equal "0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n", client.get_file_data("pipe")
+  end
+
+  def teardown_mogilefs
+    if @mogstored_pid
+      pid = File.read(@mogstored_pid.path).to_i
+      Process.kill(:TERM, pid) if pid > 0
+    end
+    if @mogilefsd_pid
+      s = TCPSocket.new(@test_host, @tracker_port)
+      s.write "!shutdown\r\n"
+      s.close
+    end
+    FileUtils.rmtree(@docroot)
+  end
+
+  def wait_for_usage_file(device)
+    uri = URI("http://#@test_host:#@mogstored_http_port/#{device}/usage")
+    res = nil
+    100.times do
+      res = Net::HTTP.get_response(uri)
+      if Net::HTTPOK === res
+        puts res.body if $DEBUG
+        return
+      end
+      puts res.inspect if $DEBUG
+      sleep 0.1
+    end
+    raise "#{uri} failed to appear: #{res.inspect}"
+  end
+
+  def setup_mogstored
+    @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
+    @mogstored_mgmt = TCPServer.new(@test_host, 0)
+    @mogstored_http = TCPServer.new(@test_host, 0)
+    @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
+    @mogstored_http_port = @mogstored_http.addr[1]
+    @mogstored_conf = Tempfile.new(["mogstored", "conf"])
+    @mogstored_pid = Tempfile.new(["mogstored", "pid"])
+    @mogstored_conf.write <<EOF
+pidfile = #{@mogstored_pid.path}
+maxconns = 1000
+httplisten = #@test_host:#{@mogstored_http_port}
+mgmtlisten = #@test_host:#{@mogstored_mgmt_port}
+docroot = #@docroot
+EOF
+    @mogstored_conf.flush
+    @mogstored_mgmt.close
+    @mogstored_http.close
+
+    x!("mogstored", "--daemon", "--config=#{@mogstored_conf.path}")
+    wait_for_port @mogstored_mgmt_port
+    wait_for_port @mogstored_http_port
+  end
+end

data/test/integration.rb

@@ -0,0 +1,43 @@
+# -*- encoding: binary -*-
+require './test/exec'
+
+class TestMogIntegration < Test::Unit::TestCase
+  include TestExec
+
+  def test_dummy
+    assert true, "Ruby 1.8 Test::Unit is broken"
+  end unless defined?(MiniTest)
+
+  def setup
+    @to_close = []
+    @trackers = ENV["MOG_TEST_TRACKERS"].split(/,/)
+    domain = "rbmogtest#{Time.now.strftime('%Y%m%d%H%M%S')}.#{uuid}"
+    @admin = MogileFS::Admin.new(:hosts => @trackers)
+    @admin.create_domain(domain)
+    yield_for_monitor_update do
+      @admin.get_domains.include?(domain) and break
+    end
+    @domain = domain
+  end
+
+  def teardown
+    if defined?(@domain)
+      client = MogileFS::MogileFS.new :hosts => @trackers, :domain => @domain
+      client.each_key("") { |key|
+        p [ :delete, key ] if $VERBOSE
+        client.delete(key)
+      }
+      assert_equal true, @admin.delete_domain(@domain)
+      assert_raises(MogileFS::Backend::DomainNotFoundError) do
+        @admin.delete_domain(@domain)
+      end
+    end
+    @to_close.each do |io|
+      io.closed? or io.close
+    end
+  end
+end if ENV["MOG_TEST_TRACKERS"]
+
+class TestMogIntegration
+  warn "MOG_TEST_TRACKERS not defined"
+end unless ENV["MOG_TEST_TRACKERS"]

data/test/setup.rb

@@ -1,5 +1,6 @@
 # -*- encoding: binary -*-
 STDIN.sync = STDOUT.sync = STDERR.sync = true
+Thread.abort_on_exception = true
 require 'test/unit'
 require 'tempfile'
 require 'fileutils'

data/test/socket_test.rb

@@ -0,0 +1,98 @@
+require "socket"
+require "test/unit"
+module SocketTest
+
+  def setup
+    @host = ENV["TEST_HOST"] || '127.0.0.1'
+    @srv = TCPServer.new(@host, 0)
+    @port = @srv.addr[1]
+  end
+
+  def test_start
+    sock = MogileFS::Socket.start(@host, @port)
+    assert_instance_of MogileFS::Socket, sock, sock.inspect
+    assert_nothing_raised do
+      begin
+        sock.write_nonblock("a")
+      rescue Errno::EAGAIN
+      end
+    end
+    thr = Thread.new { @srv.accept }
+    accepted = thr.value
+    assert_instance_of TCPSocket, accepted, accepted.inspect
+    assert_nil sock.close
+  end
+
+  def test_new
+    sock = MogileFS::Socket.tcp(@host, @port)
+    assert_instance_of MogileFS::Socket, sock, sock.inspect
+    assert_nothing_raised do
+      sock.write_nonblock("a")
+    end
+    thr = Thread.new { @srv.accept }
+    accepted = thr.value
+    assert_instance_of TCPSocket, accepted, accepted.inspect
+    assert_equal "a", accepted.read(1)
+    assert_nil sock.close
+  end
+
+  def test_timed_peek
+    sock = MogileFS::Socket.tcp(@host, @port)
+    accepted = @srv.accept
+    buf = ""
+    assert_raises(MogileFS::UnreadableSocketError) do
+      sock.timed_peek(2, buf, 0.01)
+    end
+    accepted.write "HI"
+    assert_equal "HI", sock.timed_peek(2, buf, 0.1)
+    assert_equal "HI", buf
+    assert_equal "HI", sock.timed_peek(2, buf)
+    assert_equal "HI", sock.timed_read(2, buf)
+    accepted.close
+    assert_nil sock.timed_peek(2, buf)
+  end
+
+  def test_timed_read
+    sock = MogileFS::Socket.tcp(@host, @port)
+    accepted = @srv.accept
+    buf = ""
+    assert_raises(MogileFS::UnreadableSocketError) do
+      sock.timed_read(2, buf, 0.01)
+    end
+    accepted.write "HI"
+    assert_equal "HI", sock.timed_read(2, buf, 0.1)
+    assert_equal "HI", buf
+    assert_raises(MogileFS::UnreadableSocketError) do
+      sock.timed_read(2, buf, 0.01)
+    end
+    accepted.close
+    assert_nil sock.timed_read(2, buf)
+  end
+
+  def test_timed_write
+    sock = MogileFS::Socket.tcp(@host, @port)
+    accepted = @srv.accept
+    buf = "A" * 100000
+    written = 0
+    assert_raises(MogileFS::RequestTruncatedError) do
+      loop do
+        assert_equal buf.bytesize, sock.timed_write(buf, 0.01)
+        written += buf.size
+      end
+    end
+    tmp = accepted.read(written)
+    assert_equal written, tmp.size
+  end
+
+  def timed_gets
+    sock = MogileFS::Socket.tcp(@host, @port)
+    accepted = @srv.accept
+    assert_raises(MogileFS::UnreadableSocketError) { sock.timed_gets(0.01) }
+    accepted.write "HI"
+    assert_raises(MogileFS::UnreadableSocketError) { sock.timed_gets(0.01) }
+    accepted.write "\n"
+    assert_equal "HI\n", sock.timed_gets
+    accepted.close
+    assert_nil sock.timed_gets
+  end
+end

data/test/test_admin.rb

@@ -1,5 +1,5 @@
 # -*- encoding: binary -*-
-require 'test/setup'
+require './test/setup'
 
 class TestMogileFS__Admin < TestMogileFS
 
@@ -35,11 +35,6 @@ class TestMogileFS__Admin < TestMogileFS
   end
 
   def test_each_fid
-    @backend.stats = {
-      'fidmax' => '182',
-      'fidcount' => '2',
-    }
-
     @backend.list_fids = {
       'fid_count' => '1',
       'fid_1_fid' => '99',
@@ -59,22 +54,23 @@ class TestMogileFS__Admin < TestMogileFS
       'fid_1_key' => 'new_new_key',
       'fid_1_length' => '9',
     }
+    @backend.list_fids = { 'fid_count' => 0 }
 
     fids = []
     @client.each_fid { |fid| fids << fid }
 
     expected = [
-      { "fid"      => "99",
+      { "fid"      => 99,
         "class"    => "normal",
         "domain"   => "test",
-        "devcount" => "2",
-        "length"   => "4",
+        "devcount" => 2,
+        "length"   => 4,
         "key"      => "file_key" },
-      { "fid"      => "182",
+      { "fid"      => 182,
         "class"    => "normal",
-        "devcount" => "2",
+        "devcount" => 2,
         "domain"   => "test",
-        "length"   => "9",
+        "length"   => 9,
         "key"      => "new_new_key" },
     ]
 
@@ -104,25 +100,6 @@ class TestMogileFS__Admin < TestMogileFS
     assert_equal expected, @client.get_domains
   end
 
-  def disabled_test_get_stats
-    @backend.stats = {}
-
-    expected = {
-      'fids' => { 'max' => '99', 'count' => '2' },
-      'device' => [
-        { 'status' => 'alive', 'files' => '2', 'id' => '1', 'host' => 'rur-1' },
-        { 'status' => 'alive', 'files' => '2', 'id' => '2', 'host' => 'rur-2' }
-      ],
-      'replication' => [
-        { 'files' => '2', 'class' => 'normal', 'devcount' => '2',
-          'domain' => 'test' }
-      ],
-      'file' => [{ 'files' => '2', 'class' => 'normal', 'domain' => 'test' }]
-    }
-
-    assert_equal
-  end
-
   def test_get_stats_fids
     @backend.stats = {
       'fidmax' => 99,
@@ -154,17 +131,17 @@ class TestMogileFS__Admin < TestMogileFS
     }
 
     expected = [
-      { "fid"      => "99",
+      { "fid"      => 99,
         "class"    => "normal",
         "domain"   => "test",
-        "devcount" => "2",
-        "length"   => "4",
+        "devcount" => 2,
+        "length"   => 4,
         "key"      => "file_key" },
-      { "fid"      => "82",
+      { "fid"      => 82,
         "class"    => "normal",
-        "devcount" => "2",
+        "devcount" => 2,
         "domain"   => "test",
-        "length"   => "9",
+        "length"   => 9,
         "key"      => "new_new_key" },
     ]