roma 0.8.13 → 0.8.14

data/ruby/server/lib/roma/tools/safecopy_test.rb

@@ -0,0 +1,184 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'date'
+require 'roma/client/rclient'
+
+@cnt = 0
+@tmax = 0
+@tmin = 100
+
+Thread.new do
+  sleep_time=10
+  while(true)
+    sleep sleep_time
+    printf("qps=%d max=%f min=%f ave=%f\n",@cnt/sleep_time,@tmax,@tmin,sleep_time/@cnt.to_f)
+    @cnt=0
+    @tmax=0
+    @tmin=100
+  end
+end
+
+def random_rquest_sender(ini_nodes, n)
+  puts __method__
+  rc=Roma::Client::RomaClient.new(ini_nodes)
+
+  loop do
+    i=rand(n)
+    ts = DateTime.now
+    case rand(3)
+    when 0
+      res=rc.set(i.to_s,'hoge'+i.to_s)
+      puts "set k=#{i} #{res}" if res==nil || res.chomp != 'STORED'
+    when 1
+      res=rc.get(i.to_s)
+      puts "get k=#{i} #{res}" if res == :error
+    when 2
+      res=rc.delete(i.to_s)
+      puts "del k=#{i} #{res}" if res != 'DELETED' && res != 'NOT_FOUND'
+    end
+    t=(DateTime.now - ts).to_f * 86400.0
+    @tmax=t if t > @tmax
+    @tmin=t if t < @tmin
+    @cnt+=1
+  end
+end
+
+def set_counts(ini_nodes, range, c)
+  puts "#{__method__} #{range} #{c}"
+  rc=Roma::Client::RomaClient.new(ini_nodes)
+
+  range.each do |i|
+    ts = DateTime.now
+    res=rc.set("key_#{i}","#{c}")
+    puts "set k=#{i} #{res}" if res==nil || res.chomp != 'STORED'
+    t=(DateTime.now - ts).to_f * 86400.0
+    @tmax=t if t > @tmax
+    @tmin=t if t < @tmin
+    @cnt+=1
+  end
+end
+
+def check_count(ini_nodes, range, c)
+  puts "#{__method__} #{range} #{c}"
+  rc=Roma::Client::RomaClient.new(ini_nodes)
+
+  range.each do |i|
+    ts = DateTime.now
+    res = rc.get("key_#{i}")
+    if res != c.to_s
+      puts "error k=key_#{i} #{res}" 
+    end
+    t=(DateTime.now - ts).to_f * 86400.0
+    @tmax=t if t > @tmax
+    @tmin=t if t < @tmin
+    @cnt+=1
+  end
+end
+
+def send_cmd(nid, cmd)
+  conn = Roma::Client::ConPool.instance.get_connection(nid)
+  conn.write "#{cmd}\r\n"
+  ret = conn.gets
+  Roma::Client::ConPool.instance.return_connection(nid, conn)
+  ret
+rescue =>e
+  STDERR.puts "#{nid} #{cmd} #{e.inspect}"
+  nil
+end
+
+def stats(nid, regexp=nil)
+  conn = Roma::Client::ConPool.instance.get_connection(nid)
+  if regexp
+    conn.write "stats #{regexp}\r\n"
+  else
+    conn.write "stats\r\n"
+  end
+  ret = ""
+  while(conn.gets != "END\r\n")
+    ret << $_
+  end
+  Roma::Client::ConPool.instance.return_connection(nid, conn)
+  ret
+rescue =>e
+  STDERR.puts "#{nid} #{e.inspect}"
+  nil  
+end
+
+def safecopy_stats(nid)
+  ret = stats(nid, 'storage.safecopy_stats')
+  return eval $1 if ret =~ /^.+\s(\[.+\])/
+  nil
+end
+
+def set_storage_status(nid, fno, stat)
+  send_cmd(ARGV[0], "set_storage_status #{fno} #{stat}")
+end
+
+def wait_status(nid, fno, stat)
+  while safecopy_stats(nid)[fno] != stat
+    sleep 5
+  end
+  stat
+end
+
+
+def test_change_status
+
+  puts "write (0...10000) = 0"
+  set_counts(ARGV, 0...10000, 0)
+  Thread.new { random_rquest_sender(ARGV, 10000) }
+
+  nid = ARGV[0]
+
+  sleep(5)
+
+  10.times do |n|
+    t = Thread.new { set_counts(ARGV, (n * 1000)...(n * 1000 + 2000), n * 10 + 1) }
+    p set_storage_status(nid, n, 'safecopy')
+    p wait_status(nid, n, :safecopy_flushed)
+    t.join
+    p safecopy_stats(nid)
+  
+    #sleep(30)
+    t = Thread.new { set_counts(ARGV, (n * 1000 + 500)...(n * 1000 + 2000), n * 10 + 2) }
+    p set_storage_status(nid, n, 'normal')
+    p wait_status(nid, n, :normal)
+    t.join
+    p safecopy_stats(nid)
+    check_count(ARGV, (n * 1000)...(n * 1000 + 500), n * 10 + 1)
+    check_count(ARGV, (n * 1000 + 500)...(n * 1000 + 2000), n * 10 + 2)
+    if (n * 1000 + 2000) < 10000
+      check_count(ARGV, (n * 1000 + 2000)...10000, 0)
+    end
+  end
+end
+
+def test_round
+  n = 0
+  1000.times do |i|
+    set_counts(ARGV, 0...10000, i)
+    check_count(ARGV, 0...10000, i)
+  end
+end
+
+param = { :num=>10000, :th=>1 }
+
+opts = OptionParser.new
+
+opts.on("-r", "--round", "round request"){|v| param[:round] = v }
+
+opts.banner = "usage:#{File.basename($0)} [options] addr:port"
+opts.parse!(ARGV)
+
+if ARGV.length == 0
+  STDERR.puts opts.help
+  exit
+end
+
+if param.key?(:round)
+  test_round
+else
+  test_change_status
+end
+
+puts "#{File.basename($0)} has done."

data/ruby/server/lib/roma/tools/simple_bench.rb

@@ -3,18 +3,18 @@ require 'optparse'
 require 'date'
 require 'roma/client/rclient'
 
-@@cnt = 0
-@@tmax = 0
-@@tmin = 100
+@cnt = 0
+@tmax = 0
+@tmin = 100
 
 Thread.new do
   sleep_time=10
   while(true)
     sleep sleep_time
-    printf("qps=%d max=%f min=%f ave=%f\n",@@cnt/sleep_time,@@tmax,@@tmin,sleep_time/@@cnt.to_f)
-    @@cnt=0
-    @@tmax=0
-    @@tmin=100
+    printf("qps=%d max=%f min=%f ave=%f\n",@cnt/sleep_time,@tmax,@tmin,sleep_time/@cnt.to_f)
+    @cnt=0
+    @tmax=0
+    @tmin=100
   end
 end
 
@@ -37,9 +37,9 @@ def random_request_sender(ini_nodes, n)
       puts "del k=#{i} #{res}" if res != 'DELETED' && res != 'NOT_FOUND'
     end
     t=(DateTime.now - ts).to_f * 86400.0
-    @@tmax=t if t > @@tmax
-    @@tmin=t if t < @@tmin
-    @@cnt+=1
+    @tmax=t if t > @tmax
+    @tmin=t if t < @tmin
+    @cnt+=1
   end
 end
  
@@ -52,9 +52,9 @@ def set_sequence(ini_nodes, n)
     res=rc.set("key_#{i}","value_#{i}")
     puts "set k=#{i} #{res}" if res==nil || res.chomp != 'STORED'
     t=(DateTime.now - ts).to_f * 86400.0
-    @@tmax=t if t > @@tmax
-    @@tmin=t if t < @@tmin
-    @@cnt+=1
+    @tmax=t if t > @tmax
+    @tmin=t if t < @tmin
+    @cnt+=1
   end
 end
 
@@ -67,9 +67,9 @@ def get_sequence(ini_nodes, n)
     res=rc.get("key_#{i}")
     puts "get #{i} #{res}" if res != "value_#{i}"
     t=(DateTime.now - ts).to_f * 86400.0
-    @@tmax=t if t > @@tmax
-    @@tmin=t if t < @@tmin
-    @@cnt+=1
+    @tmax=t if t > @tmax
+    @tmin=t if t < @tmin
+    @cnt+=1
   end
 end
 

data/ruby/server/lib/roma/version.rb

@@ -1,4 +1,4 @@
 
 module Roma
-  VERSION = "0.8.13"
+  VERSION = "0.8.14"
 end

data/ruby/server/test/t_storage.rb

@@ -36,7 +36,7 @@ class TCStorageTest < Test::Unit::TestCase
 
   def test_set_get
     assert_equal( 'abc_data',@st.set(0,'abc',0,0xffffffff,'abc_data')[4])
-    assert_equal( 'abc_data', @st.get(0,'abc',0)  )
+    assert_equal( 'abc_data', @st.get(0,'abc',0) )
   end
 
   def test_set_delete
@@ -435,61 +435,63 @@ class TCStorageTest < Test::Unit::TestCase
     }
   end
 
-  def test_dump_file
+  def test_each_vn_dump_in_normal
     n=100
     n.times{|i|
       @st.set(0,"key#{i}",0,0x7fffffff,"val#{i}")
     }
-    @st.dump_file('storage_test_dump')
-
-    count = 0
-    open("storage_test_dump/#{@st.hdiv[0]}.dump",'rb'){|f|
-      until(f.eof?)
-        b1 = f.read(5 * 4)
-        vn, last, clk, expt, klen = b1.unpack('NNNNN')
-        key = f.read(klen)
-        b2 = f.read(4)
-        vlen = b2.unpack('N')[0]
-        val = f.read(vlen)
-        
-        count += 1
-        assert_equal('key',key[0..2])
-        assert_equal('val',val[0..2])
-        assert_equal(val[3..-1],key[3..-1]  )
-      end
+    (90..99).each{|i|
+      @st.delete(0, "key#{i}", 0)
     }
-    assert_equal(n,count)
-    rmtestdir('storage_test_dump')
-
-    @st.dump_file('storage_test_dump',{0=>0})
     count = 0
-    open("storage_test_dump/#{@st.hdiv[0]}.dump",'rb'){|f|
-      until(f.eof?)
-        b1 = f.read(5 * 4)
-        vn, last, clk, expt, klen = b1.unpack('NNNNN')
-        key = f.read(klen)
-        b2 = f.read(4)
-        vlen = b2.unpack('N')[0]
-        val = f.read(vlen)
-        
-        count += 1
-        assert_equal('key',key[0..2] )
-        assert_equal('val',val[0..2] )
-        assert_equal(val[3..-1],key[3..-1] )
-      end
+    @st.each_vn_dump(0){|data|
+      vn, last, clk, expt, klen = data.slice!(0..19).unpack('NNNNN')
+      k = data.slice!(0..(klen-1))
+      vlen, = data.slice!(0..3).unpack('N')
+      v = data
+      count += 1
+#      puts "#{vn} #{last} #{clk} #{expt} #{klen} #{k} #{vlen} #{v}"
+      assert_equal('key',k[0..2])
+      assert_equal('val',v[0..2]) if k[3..-1].to_i < 90
+
+      assert_nil( @st.load_stream_dump(vn, last, clk, expt, k, v) )
+      @st.load_stream_dump(2, last, clk, expt, k, v)
     }
+    assert_equal(100,count)
+    
+    count = 0
+    @st.each_vn_dump(1){|data| count += 1 }
     assert_equal(0,count )
-    rmtestdir('storage_test_dump')    
+
+    count = 0
+    @st.each_vn_dump(2){|data| count += 1 }
+    assert_equal(100,count )
   end
 
-  def test_each_vn_dump
+  def test_each_vn_dump_not_normal
+    dn = @st.instance_eval{ @hdiv[0] }
+
     n=100
     n.times{|i|
       @st.set(0,"key#{i}",0,0x7fffffff,"val#{i}")
     }
+
+    # :normal -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(dn, :safecopy_flushing))
+    # :safecopy_flushing -> :safecopy_flushed
+    assert_equal(:safecopy_flushed, @st.set_db_stat(dn, :safecopy_flushed))
+
+    (80..99).each{|i|
+      @st.set(0,"key#{i}",0,0x7fffffff,"val#{i + 1}")
+    }
+
+    # :safecopy_flushed -> :cachecleaning
+    assert_equal(:cachecleaning, @st.set_db_stat(dn, :cachecleaning))
+
     (90..99).each{|i|
       @st.delete(0, "key#{i}", 0)
     }
+
     count = 0
     @st.each_vn_dump(0){|data|
       vn, last, clk, expt, klen = data.slice!(0..19).unpack('NNNNN')
@@ -498,8 +500,14 @@ class TCStorageTest < Test::Unit::TestCase
       v = data
       count += 1
 #      puts "#{vn} #{last} #{clk} #{expt} #{klen} #{k} #{vlen} #{v}"
-      assert_equal('key',k[0..2])
-      assert_equal('val',v[0..2]) if k[3..-1].to_i < 90
+      assert_match(/key\d/, k)
+      if k[3..-1].to_i < 80
+        assert_match("val#{k[3..-1]}", v)
+      elsif k[3..-1].to_i < 90
+        assert_match("val#{k[3..-1].to_i + 1}", v)
+      else
+        assert(vlen == 0)
+      end
 
       assert_nil( @st.load_stream_dump(vn, last, clk, expt, k, v) )
       @st.load_stream_dump(2, last, clk, expt, k, v)
@@ -515,6 +523,180 @@ class TCStorageTest < Test::Unit::TestCase
     assert_equal(100,count )
   end
 
+  def test_db_stat
+    assert_nil(@st.instance_eval{ @hdbc[0] })
+    # :normal -> error case
+    assert_equal(false, @st.set_db_stat(0, :safecopy_flushed))
+    assert_equal(false, @st.set_db_stat(0, :cachecleaning))
+    assert_equal(false, @st.set_db_stat(0, :normal))
+    # :normal -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(0, :safecopy_flushing))
+    assert_equal(:safecopy_flushing, @st.instance_eval{ @dbs[0] })
+    assert(@st.instance_eval{ @hdbc[0] })
+
+    # :safecopy_flushing -> error case
+    assert_equal(false, @st.set_db_stat(0, :safecopy_flushing))
+    assert_equal(false, @st.set_db_stat(0, :cachecleaning))
+    assert_equal(false, @st.set_db_stat(0, :normal))
+    # :safecopy_flushing -> :safecopy_flushed
+    assert_equal(:safecopy_flushed, @st.set_db_stat(0, :safecopy_flushed))
+    assert_equal(:safecopy_flushed, @st.instance_eval{ @dbs[0] })
+    assert(@st.instance_eval{ @hdbc[0] })
+    
+    # :safecopy_flushed -> error case
+    assert_equal(false, @st.set_db_stat(0, :safecopy_flushing))
+    assert_equal(false, @st.set_db_stat(0, :safecopy_flushed))
+    assert_equal(false, @st.set_db_stat(0, :normal))
+    # :safecopy_flushed -> :cachecleaning
+    assert_equal(:cachecleaning, @st.set_db_stat(0, :cachecleaning))
+    assert_equal(:cachecleaning, @st.instance_eval{ @dbs[0] })
+    assert(@st.instance_eval{ @hdbc[0] })
+
+    # :cachecleaning -> error case
+    assert_equal(false, @st.set_db_stat(0, :safecopy_flushed))
+    assert_equal(false, @st.set_db_stat(0, :cachecleaning))
+    # :cachecleaning -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(0, :safecopy_flushing))
+    assert_equal(:safecopy_flushing, @st.instance_eval{ @dbs[0] })
+    assert_equal(:safecopy_flushed, @st.set_db_stat(0, :safecopy_flushed))
+    assert_equal(:cachecleaning, @st.set_db_stat(0, :cachecleaning))
+    assert(@st.instance_eval{ @hdbc[0] })
+    # :cachecleaning -> :normal
+    assert_equal(:normal, @st.set_db_stat(0, :normal))
+    assert_equal(:normal, @st.instance_eval{ @dbs[0] })
+    assert_nil(@st.instance_eval{ @hdbc[0] })
+  end
+
+  def test_clock_along_status
+    vn = 0
+    dn = @st.instance_eval{ @hdiv[vn] }
+    assert_equal('abc_data', @st.set(vn,'abc',0,0xffffffff,'abc_data')[4])
+    assert( @st.get_context(vn, 'abc', 0)[2] == 0 )
+    # :normal -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(dn, :safecopy_flushing))
+    assert_equal('abc_data', @st.set(vn,'abc',0,0xffffffff,'abc_data')[4])
+    assert( @st.get_context(vn, 'abc', 0)[2] == 1 )
+    # :safecopy_flushing -> :safecopy_flushed
+    assert_equal(:safecopy_flushed, @st.set_db_stat(dn, :safecopy_flushed))
+    assert_equal('abc_data', @st.set(vn,'abc',0,0xffffffff,'abc_data')[4])
+    assert( @st.get_context(vn, 'abc', 0)[2] == 2 )
+    # :safecopy_flushed -> :cachecleaning
+    assert_equal(:cachecleaning, @st.set_db_stat(dn, :cachecleaning))
+    assert_equal('abc_data', @st.set(vn,'abc',0,0xffffffff,'abc_data')[4])
+    assert( @st.get_context(vn, 'abc', 0)[2] == 3 )
+    # :cachecleaning -> :normal
+    assert_equal(:normal, @st.set_db_stat(dn, :normal))
+    assert_equal('abc_data', @st.set(vn,'abc',0,0xffffffff,'abc_data')[4])
+    assert( @st.get_context(vn, 'abc', 0)[2] == 4 )
+  end
+
+  def test_set_get_in_safecopy
+    vn = 0
+    dn = @st.instance_eval{ @hdiv[vn] }
+    # :normal -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(dn, :safecopy_flushing))
+    assert_equal(:safecopy_flushing, @st.instance_eval{ @dbs[dn] })
+    assert_equal('abc_data', @st.set(vn,'abc',0,0xffffffff,'abc_data')[4])
+    assert_equal('abc_data', @st.get(vn,'abc',0) )
+    # not stored
+    assert_nil(@st.instance_eval{ @hdb[dn].get('abc') })
+    # stored
+    assert(@st.instance_eval{ @hdbc[dn].get('abc') })
+
+    # :safecopy_flushing -> :safecopy_flushed
+    assert_equal(:safecopy_flushed, @st.set_db_stat(dn, :safecopy_flushed))
+    assert_equal(:safecopy_flushed, @st.instance_eval{ @dbs[dn] })
+    assert_equal('def_data', @st.set(vn,'def',0,0xffffffff,'def_data')[4])
+    assert_equal('def_data', @st.get(vn,'def',0) )
+    # not stored
+    assert_nil(@st.instance_eval{ @hdb[dn].get('def') })
+    # stored
+    assert(@st.instance_eval{ @hdbc[dn].get('def') })
+  end
+
+  def test_set_get_in_cachecleaning
+    vn = 0
+    dn = @st.instance_eval{ @hdiv[vn] }
+    assert_equal( 'abc_data',@st.set(vn,'abc',0,0xffffffff,'abc_data')[4])
+    assert_equal( 'abc_data', @st.get(vn,'abc',0) ) # database
+    # :normal -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(dn, :safecopy_flushing))
+    # :safecopy_flushing -> :safecopy_flushed
+    assert_equal(:safecopy_flushed, @st.set_db_stat(dn, :safecopy_flushed))
+    assert_equal( 'abc_data1',@st.set(vn,'abc',0,0xffffffff,'abc_data1')[4])
+    assert_equal( 'abc_data1', @st.get(vn,'abc',0) ) # cache
+    # :safecopy_flushed -> :cachecleaning
+    assert_equal(:cachecleaning, @st.set_db_stat(dn, :cachecleaning))
+    assert_equal( 'abc_data1', @st.get(vn,'abc',0) ) # cache
+    assert_equal( 'abc_data2',@st.set(vn,'abc',0,0xffffffff,'abc_data2')[4])
+    assert_equal( 'abc_data2', @st.get(vn,'abc',0) ) # database
+  end
+
+  def test_out_cache
+    vn = 0
+    dn = @st.instance_eval{ @hdiv[vn] }
+    # :normal -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(dn, :safecopy_flushing))
+    assert_equal(:safecopy_flushing, @st.instance_eval{ @dbs[dn] })
+    assert_equal('abc_data', @st.set(vn,'abc',0,0xffffffff,'abc_data')[4])
+    assert_equal('abc_data', @st.get(vn,'abc',0) )
+    # not stored
+    assert_nil(@st.instance_eval{ @hdb[dn].get('abc') })
+    # stored
+    assert(@st.instance_eval{ @hdbc[dn].get('abc') })
+    # out
+    @st.out_cache(dn, 'abc')
+    # not stored
+    assert_nil(@st.instance_eval{ @hdb[dn].get('abc') })
+    # not stored
+    assert_nil(@st.instance_eval{ @hdbc[dn].get('abc') })
+  end
+
+  def test_each_cache_dump_pack
+    vn = 0
+    dn = @st.instance_eval{ @hdiv[vn] }
+    # :normal -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(dn, :safecopy_flushing))
+    keys = []
+    10.times do |i|
+      k = "key#{i}"
+      v = "val#{i}"
+      assert_equal(v, @st.set(vn,k,0,0xffffffff,v)[4])
+      keys << k
+    end
+    @st.each_cache_dump_pack(dn, keys) do |data|
+      vn, last, clk, expt, klen = data.slice!(0..19).unpack('NNNNN')
+      k = data.slice!(0..(klen-1))
+      vlen, = data.slice!(0..3).unpack('N')
+      v = data
+#      puts "#{vn} #{last} #{clk} #{expt} #{klen} #{k} #{vlen} #{v}"
+      assert_match(/key\d/, k)
+      assert_match("val#{k[3..-1]}", v)
+    end
+  end
+
+  def test_get_keys_in_cache
+    vn = 0
+    dn = @st.instance_eval{ @hdiv[vn] }
+    # :normal -> :safecopy_flushing
+    assert_equal(:safecopy_flushing, @st.set_db_stat(dn, :safecopy_flushing))
+    keys = []
+    100.times do |i|
+      k = "key#{i}"
+      v = "val#{i}"
+      assert_equal(v, @st.set(vn,k,0,0xffffffff,v)[4])
+    end
+
+    assert(@st.instance_eval{ @hdbc[dn].rnum } == 100)
+    10.times do
+      keys = @st.get_keys_in_cache(dn, 10)
+      assert(keys.length == 10)
+      keys.each do |k|
+        @st.out_cache(dn, k)
+      end
+    end
+    assert(@st.instance_eval{ @hdbc[dn].rnum } == 0)
+  end
 end
 
 class DbmStorageTest < TCStorageTest