flare-tools 0.1.4 → 0.4.5.1

data/test/test/integration/dump_test.rb

@@ -0,0 +1,128 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*- 
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../../../lib")
+
+require 'rubygems'
+gem 'test-unit'
+require 'test/unit'
+require 'flare/tools'
+require 'flare/tools/cli'
+require 'flare/test/cluster'
+require 'subcommands'
+
+class DumpTest < Test::Unit::TestCase
+  include Flare::Tools::Common
+  include Subcommands
+
+  def setup
+    File.delete("src.tch") if File.exist?("src.tch")
+  end
+
+  def teardown
+    File.delete("src.tch") if File.exist?("src.tch")
+  end
+
+  def prepare cluster, partitions, ranges
+    sleep 1 # XXX
+    nodes = (1..6).map {|n| cluster.create_node("node#{n}")}
+    sleep 1 # XXX
+    cluster.wait_for_ready
+    partitions.each do |p|
+      cluster.prepare_master_and_slaves(nodes.values_at(*ranges[p]), p)
+    end
+    nodes
+  end
+
+  def dump_and_restore(range, prefixes, extra_dump_options, extra_restore_options, &check)
+    expected_data = [1, 2, 3, 4, 5, 6, 7, 8, 9].pack("c*")
+    expected_flag = 1234
+    expected_expire = 2000000000
+    # prepare
+    ranges = [[0, 1], [2, 3], [4, 5]]
+    src_cluster = Flare::Test::Cluster.new('src')
+    puts "preparing src"
+    src_nodes = prepare(src_cluster, [0], ranges)
+    puts "storing"
+    range.each do |i|
+      src_nodes[0].open do |n|
+        prefixes.each do |prefix|
+          n.set("#{prefix}::#{i}", expected_data, expected_flag, expected_expire)
+        end
+      end
+    end
+    src_nodes = prepare(src_cluster, [1, 2], ranges)
+    puts "preparing dest"
+    dest_cluster = Flare::Test::Cluster.new('dest')
+    dest_nodes = prepare(dest_cluster, [0, 1, 2], ranges)
+
+    puts "dumping"
+    @config = {
+      :index_server_hostname => src_cluster.indexname,
+      :index_server_port => src_cluster.indexport,
+      :timeout => 10
+    }
+    args = %w(--format=tch --output=src.tch)
+    args.concat extra_dump_options
+    assert_equal(S_OK, dump(*args))
+
+    @config = {
+      :index_server_hostname => dest_cluster.indexname,
+      :index_server_port => dest_cluster.indexport,
+      :timeout => 10
+    }
+    n = dest_nodes[0]
+    args = %w(--format=tch --input=src.tch)
+    args.concat extra_restore_options
+    args << "#{n.hostname}:#{n.port}"
+    assert_equal(S_OK, restore(*args))
+
+    # check - total number
+    restored = [0, 2, 4].inject(0) do |r,i|
+      dest_nodes[i].open do |n|
+        r+n.stats["curr_items"].to_i
+      end
+    end
+
+    # check - flags
+    count = 0
+    (0..2).each do |partition|
+      dest_nodes[ranges[partition][0]].open do |n|
+        n.dump(0, partition, ranges.size, 0) do |data, key, flag, len, version, expire|
+          assert_equal(expected_data, data)
+          assert_equal(expected_flag, flag)
+          assert_equal(expected_expire, expire)
+          count += 1
+        end
+      end
+    end
+    assert_not_equal(0, count)
+
+    # check - call
+    check.call(restored)
+
+    # destroy
+    src_cluster.shutdown
+    dest_cluster.shutdown
+  end
+
+  def test_dump_and_restore1
+    dump_and_restore (0...100), ["prefix", "xiferp", "pre"], %w(--all), %w(--include=^prefix) do |restored|
+      assert_equal(100, restored)
+    end
+  end
+
+  def test_dump_and_restore2
+    dump_and_restore (0...100), ["prefix", "xiferp", "pre"], %w(--all), %w(--prefix-include=prefix) do |restored|
+      assert_equal(100, restored)
+    end
+  end
+
+  def test_dump_and_restore3
+    dump_and_restore (0...100), ["prefix", "xiferp", "pre"], %w(--all --raw), %w(--prefix-include=prefix) do |restored|
+      assert_equal(100, restored)
+    end
+  end
+
+end

data/test/test/integration/index_server_test.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../../../lib")
+
+require 'test/unit'
+require 'flare/tools.rb'
+require 'flare/test/daemon'
+require 'flare/test/cluster'
+
+class IndexServerTest < Test::Unit::TestCase
+  def setup
+    @flare_cluster = Flare::Test::Cluster.new('test')
+    sleep 1
+    @node_servers = ['node1', 'node2', 'node3'].map {|name| @flare_cluster.create_node(name)}
+    sleep 1
+    @flare_cluster.prepare_master_and_slaves(@node_servers)
+  end
+  
+  def teardown
+    @flare_cluster.shutdown
+  end
+
+  def test_index_cluster_info
+    hostname = @flare_cluster.indexname
+    port = @flare_cluster.indexport
+    stats = Flare::Tools::IndexServer.new(hostname, port, 10)
+    nodes = stats.stats_nodes
+    cluster = Flare::Tools::Cluster.new(hostname, port, nodes)
+    assert_equal(cluster.size, @node_servers.size)
+  end
+  
+end
+

data/test/test/integration/node_test.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../../../lib")
+
+require 'test/unit'
+require 'flare/tools'
+require 'flare/test/daemon'
+require 'flare/test/cluster'
+
+class NodeTest < Test::Unit::TestCase
+  def setup
+    @flare_cluster = Flare::Test::Cluster.new('test')
+    sleep 1 # XXX
+    @node_servers = ['node1', 'node2', 'node3'].map {|name| @flare_cluster.create_node(name)}
+    sleep 1 # XXX
+    @flare_cluster.wait_for_ready
+  end
+  
+  def teardown
+    @flare_cluster.shutdown
+  end
+
+  def test_dummy
+    @flare_cluster.prepare_master_and_slaves(@node_servers)
+  end
+
+  def test_one_million_entry
+    @flare_cluster.prepare_master_and_slaves(@node_servers)
+    @flare_cluster.prepare_data(@node_servers[0], "key", 1000000)
+  end if ENV['FLARE_TOOLS_STRESS_TEST']
+
+  def test_update1
+    @flare_cluster.prepare_master_and_slaves(@node_servers)
+    Flare::Tools::Node.open(@flare_cluster.indexname, @flare_cluster.indexport, 10) do |s|
+      node = @node_servers[1]
+      Flare::Tools::Node.open(node.hostname, node.port, 10) do |n|
+        n.set("incr", "0")
+        n.set("decr", "9")
+        sleep 1
+        assert_equal(0.to_s, n.get("incr"))
+        assert_equal(9.to_s, n.get("decr"))
+        n.set_noreply("incr_noreply", "0")
+        n.set_noreply("decr_noreply", "9")
+        sleep 1
+        assert_equal(0.to_s, n.get("incr_noreply"))
+        assert_equal(9.to_s, n.get("decr_noreply"))
+        (1...10).each do |i|
+          assert_equal(i.to_s, n.incr("incr", 1))
+          assert_equal((9-i.to_i).to_s, n.decr("decr", 1))
+          n.incr("incr_noreply", 1)
+          n.decr("decr_noreply", 1)
+        end
+        assert_equal(9.to_s, n.get("incr_noreply"))
+        assert_equal(0.to_s, n.get("decr_noreply"))
+      end
+    end
+  end
+
+  def test_delete1
+    @flare_cluster.prepare_master_and_slaves(@node_servers)
+    Flare::Tools::Node.open(@flare_cluster.indexname, @flare_cluster.indexport, 10) do |s|
+      node = @node_servers[1]
+      Flare::Tools::Node.open(node.hostname, node.port, 10) do |n|
+        n.set("key", "0")
+        n.delete("key")
+        assert_equal(false, n.get("key"))
+        n.set("key", "0")
+        n.delete_noreply("key")
+        sleep 0.1
+        assert_equal(false, n.get("key"))
+      end
+    end
+  end
+
+end
+

data/test/test/integration/partition_test.rb

@@ -0,0 +1,235 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*- 
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../../../lib")
+
+require 'test/unit'
+require 'flare/tools'
+require 'flare/tools/cli'
+require 'flare/test/cluster'
+require 'subcommands'
+
+begin
+  require 'progressbar'
+rescue LoadError => e
+end
+
+class Result
+  def initialize(resultq)
+    @resultq = resultq
+  end
+
+  def get
+    @resultq.pop
+  end
+
+  def sync
+    @resultq.pop
+  end
+end
+
+class FlareClient
+  def initialize(node)
+    @queue = Queue.new
+    @resultq = Queue.new
+    @node = node
+    @th = Thread.new do
+      while item = @queue.pop
+        case item[0]
+        when :get
+          value = @node.get(item[1])
+          @resultq.push(value)
+        when :set
+          @node.set(item[1], item[2])
+          @resultq.push(nil)
+        when :quit 
+          @node.quit
+          break
+        when :detach
+          break
+        else
+          p item
+        end
+      end
+    end
+  end
+
+  def get(key)
+    @queue.push([:get, key.to_s])
+    Result.new(@resultq)
+  end
+
+  def set(key, value)
+    @queue.push([:set, key.to_s, value.to_s])
+    Result.new(@resultq)
+  end
+
+  def quit
+    @queue.push([:quit])
+    @th.join
+  end
+
+  def detach
+    @queue.push([:detach])
+    @th.join
+  end
+end
+
+
+class PartitionTest < Test::Unit::TestCase
+  include Flare::Tools::Common
+  include Subcommands
+
+  S_OK = 0
+  S_NG = 1
+  
+  def setup
+    @flare_cluster = Flare::Test::Cluster.new('test')
+    sleep 1 # XXX
+    @node_servers = ['node1', 'node2', 'node3', 'node4', 'node5', 'node6', 'node7'].map {|name|
+      @flare_cluster.create_node(name)
+    }
+    sleep 1 # XXX
+    @flare_cluster.wait_for_ready
+    @config = {
+      :command => 'dummy',
+      :index_server_hostname => @flare_cluster.indexname,
+      :index_server_port => @flare_cluster.indexport,
+      :dry_run => false,
+      :timeout => 10
+    }
+    @nodes = @node_servers.map {|node| node.open}
+    @wait = 0.005
+  end
+
+  def teardown
+    @nodes.map {|n| n.close}
+    @flare_cluster.shutdown
+  end
+
+  def test_dynamic_partition_creation1
+    @flare_cluster.prepare_master_and_slaves(@node_servers[0..1])
+    @flare_cluster.prepare_data(@node_servers[0], "key", 10)
+    master = FlareClient.new(@nodes[0])
+    args = @node_servers[2...3].map{|n| "#{n.hostname}:#{n.port}:1:1"}
+    assert_equal(S_OK, master(*args))
+    list
+    for i in 0..10
+      r = master.set("k#{i}", "piyo")
+      r.sync
+      sleep @wait
+      master_items = @nodes[0].stats["curr_items"].to_i
+      slave_items = @nodes[1].stats["curr_items"].to_i
+      puts "master_items=#{master_items}, slave_items=#{slave_items}"
+      assert_equal(master_items, slave_items)
+    end
+    args = @node_servers[2..3].map{|n| "#{n.hostname}:#{n.port}"}
+    assert_equal(S_OK, activate(*args))
+    master.detach
+  end
+
+  def test_dynamic_partition_creation2
+    @flare_cluster.prepare_master_and_slaves(@node_servers[0..1])
+    @flare_cluster.prepare_data(@node_servers[0], "key", 10)
+    master = FlareClient.new(@nodes[0])
+    args = @node_servers[2...3].map{|n| "#{n.hostname}:#{n.port}:1:1"}
+    assert_equal(S_OK, master(*args))
+    args = @node_servers[3...4].map{|n| "#{n.hostname}:#{n.port}:0:1"}
+    assert_equal(S_OK, slave(*args))
+    list
+    basesize = @nodes[0].stats["curr_items"].to_i
+    size = 10
+    for i in 0...size
+      r = master.set("k#{i}", "piyo")
+      r.sync
+      sleep @wait
+      master_items = @nodes[0].stats["curr_items"].to_i
+      slave_items = @nodes[1].stats["curr_items"].to_i
+      preparing_master_items = @nodes[2].stats["curr_items"].to_i
+      preparing_slave_items = @nodes[3].stats["curr_items"].to_i
+      puts "master=#{master_items}, slaves=#{slave_items}, preparing_master=#{preparing_master_items}, preparing_slave=#{preparing_slave_items}"
+      assert_equal(master_items, slave_items)
+    end
+    assert_equal(size+basesize, master_items)
+    args = @node_servers[2..3].map{|n| "#{n.hostname}:#{n.port}"}
+    assert_equal(S_OK, activate(*args))
+    master.detach
+  end
+
+  def dynamic_partition_creation3(ntarget)
+    p, m0, s0, m1, s1, m2, s2 = 0, 1, 2, 3, 4, 5, 6
+    @flare_cluster.prepare_master_and_slaves(@node_servers[m0..s0])
+    target = FlareClient.new(ntarget)
+    assert_equal(S_OK, master(*@node_servers[m1..m1].map{|n| "#{n.hostname}:#{n.port}:1:1"}))
+    assert_equal(S_OK, slave(*@node_servers[s1..s1].map{|n| "#{n.hostname}:#{n.port}:1:1"}))
+    assert_equal(S_OK, activate(*@node_servers[m1..m1].map{|n| "#{n.hostname}:#{n.port}"}))
+    assert_equal(S_OK, master(*@node_servers[m2..m2].map{|n| "#{n.hostname}:#{n.port}:1:2"}))
+    assert_equal(S_OK, slave(*@node_servers[s2..s2].map{|n| "#{n.hostname}:#{n.port}:1:2"}))
+    list
+    size = 10
+    for i in 0...size
+      r = target.set("k#{i}", "piyo")
+      r.sync
+      sleep @wait
+      items = @nodes.map {|n| n.stats["curr_items"].to_i}
+      assert_equal(items[m0], items[s0])
+      assert_equal(items[m1], items[s1])
+      assert_equal(items[m2], items[s2])
+    end
+    assert_equal(size, items[m0]+items[m1])
+    assert_equal(S_OK, activate(*@node_servers[m2..m2].map{|n| "#{n.hostname}:#{n.port}"}))
+    for i in 0...size
+      r = target.set("k#{i}", "piyo")
+      r.sync
+      sleep @wait
+      items = @nodes.map {|n| n.stats["curr_items"].to_i}
+      assert_equal(items[m0], items[s0])
+      assert_equal(items[m1], items[s1])
+      assert_equal(items[m2], items[s2])
+    end
+    target.detach
+  end
+
+  def self.deftest_all(name)
+    syms = ["p", "m0", "s0", "m1", "s1", "m2", "s2"]
+    for i in (0...syms.size)
+      self.class_eval %{
+        def test_#{name.to_s}_#{syms[i]}
+          #{name.to_s}(@nodes[#{i}])
+        end
+      }
+    end
+  end
+
+  deftest_all :dynamic_partition_creation3
+
+  def test_dynamic_partition_creation4
+    p, m0, s0, m1, s1, m2, s2 = 0, 1, 2, 3, 4, 5, 6
+    @flare_cluster.prepare_master_and_slaves(@node_servers[m0..s0])
+    assert_equal(S_OK, master(*@node_servers[m1..m1].map{|n| "#{n.hostname}:#{n.port}:1:1"}))
+    assert_equal(S_OK, slave(*@node_servers[s1..s1].map{|n| "#{n.hostname}:#{n.port}:1:1"}))
+    assert_equal(S_OK, activate(*@node_servers[m1..m1].map{|n| "#{n.hostname}:#{n.port}"}))
+    assert_equal(S_OK, master(*@node_servers[m2..m2].map{|n| "#{n.hostname}:#{n.port}:1:2"}))
+    assert_equal(S_OK, slave(*@node_servers[s2..s2].map{|n| "#{n.hostname}:#{n.port}:1:2"}))
+    targets = @nodes.map {|n| FlareClient.new(n)}
+    list
+    size = 1000
+    pbar = ProgressBar.new('test_dynamic_partition_creation4', size, $stderr) if defined? ProgressBar
+    for i in 0...size
+      r = targets[rand(targets.size)].set("k#{i}", "piyo")
+      r.sync
+      sleep @wait
+      items = @nodes.map {|n| n.stats["curr_items"].to_i}
+      assert_equal(items[m0], items[s0])
+      assert_equal(items[m1], items[s1])
+      assert_equal(items[m2], items[s2])
+      if defined? ProgressBar
+        pbar.inc
+      end
+      assert_equal(S_OK, activate(*@node_servers[m2..m2].map{|n| "#{n.hostname}:#{n.port}"})) if i == size/2
+    end
+    targets.each {|c| c.detach}
+  end
+
+end