flare-tools 0.1.4 → 0.4.5.1

data/lib/flare/util/result.rb

@@ -0,0 +1,53 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+# 
+module Flare
+  module Util
+
+    # == Description
+    # Result is a class for handling result code.
+    module Result
+      None = nil
+      Ok = :OK
+      End = :END
+      Stored = :STORED
+      NotStored = :NOT_STORED
+      Exists = :EXISTS
+      NotFound= :NOT_FOUND
+      Deleted = :DELETED
+      Found = :FOUND
+      Error = :ERROR
+      ClientError = :CLIENT_ERROR
+      ServerError = :SERVER_ERROR
+
+      # Converts a result code to its string representation.
+      def self.string_of_result(result)
+        case result
+        when None
+          ""
+        when Ok,End,Stored,NotStored,Exists,NotFound,Deleted,Found,Error,ClientError,ServerError
+          result.to_s
+        else
+          raise "Invalid argument '"+result.to_s+"'"
+        end
+      end
+      
+      # Converts a string representation of a request result to its result code.
+      def self.result_of_string(string)
+        case string
+        when ""
+          None
+        else
+          [Ok,End,Stored,NotStored,Exists,NotFound,Deleted,Found,Error,ClientError,ServerError].each do |x|
+            return x if x.to_s == string
+          end
+          raise "Invalid arugument '"+string.to_s+"'"
+        end
+      end
+
+    end
+  end
+end

data/test/test/experimental/cache_test.rb

@@ -0,0 +1,113 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*- 
+
+$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
+
+require 'test/unit'
+require 'flare/tools'
+require 'flare/tools/cli'
+require 'flare/test/cluster'
+require 'subcommands'
+
+class CacheTest < Test::Unit::TestCase
+  include Flare::Tools::Common
+  include Subcommands
+
+  S_OK = 0
+  S_NG = 1
+
+  def setup
+    @flare_cluster = Flare::Test::Cluster.new('test')
+    sleep 1 # XXX
+    @node_servers = ['localmaster1', 'localproxy1', 'remoteproxy1'].map {|name|
+      @flare_cluster.create_node(name, { 'proxy-cache-size' => 1000, 'proxy-cache-expire' => 60 }, "/usr/local/proxy_cache/bin/flared")
+    }
+    sleep 1 # XXX
+    @flare_cluster.wait_for_ready
+    @config = {
+      :command => 'dummy',
+      :index_server_hostname => @flare_cluster.indexname,
+      :index_server_port => @flare_cluster.indexport,
+      :dry_run => false,
+      :timeout => 10
+    }
+    @nodes = @node_servers.map do |node|
+      Flare::Tools::Node.open(node.hostname, node.port, 10)
+    end
+    @localmaster1, @localproxy1, @remoteproxy1 = @nodes
+  end
+  
+  def teardown
+    @nodes.map {|n| n.close}
+  end
+
+  def test_proxy_gets_from_various_routes
+    @flare_cluster.prepare_master_and_slaves(@node_servers[0..0])
+    key = "hoge"
+    @localmaster1.set(key, "v1")
+    value, version = @localmaster1.gets(key)
+    assert_equal(1, version);
+    assert_equal("v1", value);
+    @localproxy1.set(key, "v2")
+    value, version = @localmaster1.gets(key)
+    assert_equal(2, version);
+    assert_equal("v2", value);
+    @remoteproxy1.set(key, "v3")
+    value, version = @localmaster1.gets(key)
+    assert_equal(3, version);
+    assert_equal("v3", value);
+  end
+
+  def test_proxy_cas_to_master
+    @flare_cluster.prepare_master_and_slaves(@node_servers[0..0])
+    key = "hoge"
+    @localmaster1.set(key, "v1")
+    value, version = @localmaster1.gets(key)
+    assert_equal(1, version);
+    @localmaster1.cas(key, "v2", version)
+    value, version = @localmaster1.gets(key)
+    assert_equal(2, version);
+    @localproxy1.cas(key, "v3", version)
+    value, version = @localproxy1.gets(key)
+    assert_equal(3, version);
+  end
+
+  class Client
+    def initialize(node)
+      @node = node
+    end
+    
+    def gets(*keys)
+      @node.gets(*keys)
+    end
+    
+    def cas(*keys)
+      @node.cas(*keys)
+    end
+  end
+
+  def test_proxy_gets_and_cas
+    @flare_cluster.prepare_master_and_slaves(@node_servers[0..0])
+    key = "hoge"
+
+    @localmaster1.set(key, "initial")
+
+    c0 = Client.new(@localmaster1)
+    c1 = Client.new(@remoteproxy1)
+    c2 = Client.new(@remoteproxy1)
+
+    v1,v1version = c1.gets(key)
+    v0,v0version = c0.gets(key)
+    v2,v2version = c2.gets(key)
+    
+    r0 = c0.cas(key, "client0", v0version)
+    r1 = c1.cas(key, "client1", v1version)
+    r2 = c2.cas(key, "client2", v2version)
+
+    assert_equal(true, r0)
+    assert_equal(false, r1)
+    assert_equal(false, r2)
+  end
+  
+end
+

data/test/test/experimental/key_distribution_test.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/util/key_resolver'
+require 'flare/util/hash_function'
+require 'shell'
+require 'test/unit'
+
+class KeyDistributionTest < Test::Unit::TestCase
+  include Flare::Util::HashFunction
+
+  def setup
+    @resolver = Flare::Util::KeyResolver.new
+  end
+
+  def test_key_distribution1
+    proxy_concurrency = 8
+    partition_size = 4
+    prefix = "test::key::distribution"
+    bin = []
+    (0...proxy_concurrency).each do |i|
+      bin[i] = 0
+    end
+    (1..10000).each do |i|
+      key = "#{prefix}::#{i}"
+      hash_of_key = get_key_hash_value(key, :bitshift, 32)
+      target_thread = hash_of_key % proxy_concurrency
+      target_partition = @resolver.resolve(get_key_hash_value(key, :simple, 32), partition_size)
+      if target_partition == 0
+        bin[target_thread] += 1
+      end
+    end
+    p bin
+  end
+
+end

data/test/test/experimental/keychecker_test.rb

@@ -0,0 +1,60 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'shell'
+require 'test/unit'
+
+Shell.verbose = false
+Shell.def_system_command "kc", "../../bin/flare-keychecker"
+
+class KeycheckerTest < Test::Unit::TestCase
+  KeyTxt = "keychecker_test.key.txt"
+  ResultCsv = "keychecker_test.result.csv"
+
+  def setup
+    @basedir = File.dirname(__FILE__)+"/work"
+    @sh = Shell.new
+    @sh.cd(@basedir)
+    @sh.transact {
+      unless exist? KeyTxt
+        keys = (0...1000).inject([]) {|r,v| r << "prefix::#{v}"}
+        echo(*keys) >> KeyTxt
+      end
+    }
+  end
+
+  def test_help1
+    @sh.transact {
+      kc("--help")
+    }
+  end
+
+  def hash(type)
+    @sh.transact {
+      (cat(KeyTxt) | kc("--hash=#{type}")) > ResultCsv
+    }
+  end
+
+  def test_hash1
+    hash("simple")
+    hash("bitshift")
+    hash("crc32")
+  end
+
+  def delimiter(delimiter)
+    @sh.transact {
+      echo(*@keys) >> KeyTxt
+      (cat(KeyTxt) | kc("--delimiter=#{delimiter}")) > ResultCsv
+    }
+  end
+
+  def test_delimiter1
+    delimiter("::")
+  end
+
+end
+
+  
+

data/test/test/experimental/list_test.rb

@@ -0,0 +1,108 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*- 
+
+$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
+
+require 'test/unit'
+require 'flare/tools'
+require 'flare/tools/cli'
+require 'flare/test/cluster'
+require 'subcommands'
+
+class ListTest < Test::Unit::TestCase
+  include Flare::Tools::Common
+  include Subcommands
+
+  S_OK = 0
+  S_NG = 1
+
+  def setup
+    @flare_cluster = Flare::Test::Cluster.new('test')
+    sleep 1 # XXX
+    @node_servers = ['node1', 'node2', 'node3'].map {|name| @flare_cluster.create_node(name)}
+    sleep 1 # XXX
+    @flare_cluster.wait_for_ready
+    @config = {
+      :command => 'dummy',
+      :index_server_hostname => @flare_cluster.indexname,
+      :index_server_port => @flare_cluster.indexport,
+      :dry_run => false,
+      :timeout => 10
+    }
+    @nodes = @node_servers.map do |node|
+      Flare::Tools::Node.open(node.hostname, node.port, 10)
+    end
+  end
+
+  def teardown
+    @nodes.map {|n| n.close}
+  end
+
+  def push_and_get(npush, nget)
+    list if defined? DEBUG
+    key = "k"
+    (0...100).each do |i|
+      value = "v#{i}"
+      npush.x_list_push(key, value)
+      sleep 0.001
+      count = 0
+      nget.x_list_get(key, i, i+1) do |v, k, rel, abs, flag, len, version, expire|
+        assert_equal(key, k)
+        assert_equal(value, v)
+        count += 1
+      end
+      assert_equal(1, count)
+    end
+  end
+
+  def push_and_range_get(npush, nget)
+    list if defined? DEBUG
+    key = "k"
+    size = 100
+    range = (0...size)
+    expected = range.map {|i| "v#{i}"}
+    range.each do |i|
+      npush.x_list_push(key, expected[i])
+    end
+    count = 0
+    nget.x_list_get(key, 0, size) do |v, k, rel, abs, f|
+      value = "v#{count}"
+      assert_equal(key, k)
+      assert_equal(value, v)
+      count += 1
+    end
+    assert_equal(size, count)
+  end
+
+  def push_and_shift(npush, nshift)
+    list if defined? DEBUG
+    key = "k"
+    range = (0...100)
+    expected = range.map {|i| "v#{i}"}
+    range.each do |i|
+      npush.x_list_push(key, expected[i])
+    end
+    values = range.map {|i| nshift.x_list_shift(key)}
+    assert_equal(expected, values)
+  end
+
+  def self.deftest_allpair(name)
+    syms = ["m", "s", "p"]
+    for i in (0...syms.size)
+      for j in (0...syms.size)
+        self.class_eval %{
+          def test_#{name.to_s}_#{syms[i]}#{syms[j]}
+            @flare_cluster.prepare_master_and_slaves(@node_servers[0..1])
+            #{name.to_s}(@nodes[#{i}], @nodes[#{j}])
+          end
+        }
+      end
+    end
+  end
+
+  deftest_allpair :push_and_get
+  deftest_allpair :push_and_range_get
+  deftest_allpair :push_and_shift
+
+end
+

data/test/test/extra/replication_test.rb

@@ -0,0 +1,184 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*- 
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'test/unit'
+require 'flare/tools'
+require 'flare/test/cluster'
+
+class ReplicationTest < Test::Unit::TestCase
+  include Flare::Tools::Common
+
+  def setup
+    @flare_cluster = Flare::Test::Cluster.new('test')
+    sleep 1 # XXX
+    @node_servers = ['node1', 'node2', 'node3', 'node4', 'node5', 'node6'].map {|name| @flare_cluster.create_node(name)}
+    sleep 1 # XXX
+    @flare_cluster.wait_for_ready
+    @config = {
+      :command => 'dummy',
+      :index_server_hostname => @flare_cluster.indexname,
+      :index_server_port => @flare_cluster.indexport,
+      :dry_run => false,
+      :timeout => 10
+    }
+    @nodes = @node_servers.map do |node|
+      Flare::Tools::Node.open(node.hostname, node.port, 10)
+    end
+  end
+
+  def teardown
+    @nodes.map {|n| n.close}
+  end
+
+  def test_replication
+    @flare_cluster.prepare_master_and_slaves(@node_servers)
+    Flare::Tools::Node.open(@flare_cluster.indexname, @flare_cluster.indexport, 10) do |s|
+      puts string_of_nodelist(s.stats_nodes)
+      node = @node_servers[1]
+      puts "throwing requests to #{node.hostname}:#{node.port}."
+      Flare::Tools::Node.open(node.hostname, node.port, 10) do |n|
+        fmt = "key%010.10d"
+        (0...10).each do |i|
+          n.set(fmt % i, "All your base are belong to us.")
+        end
+      end
+      puts string_of_nodelist(s.stats_nodes)
+    end
+  end
+
+  def parallel(*args, &block)
+    args.map do |arg|
+      Thread.new do
+        block.call(arg)
+      end
+    end
+  end
+
+  def test_replication_consistent
+    replication_consistent(false)
+  end
+
+  def test_replication_consistent_noreply
+    replication_consistent(true, 0.1)
+  end
+
+  def replication_consistent(noreply, wait = 0)
+    @flare_cluster.prepare_master_and_slaves(@node_servers)
+    
+    Flare::Tools::Node.open(@flare_cluster.indexname, @flare_cluster.indexport, 10) do |s|
+      puts string_of_nodelist(s.stats_nodes)
+      ntry = 256
+      nparallel = 128
+      nloop = 32
+      entries = (0...nparallel).map { |i|
+        {
+          :index => i,
+          :command_queue => Queue.new,
+          :hostname => @node_servers[0].hostname,
+          :port => @node_servers[0].port,
+          :result_queue => Queue.new,
+          :backlog_queue => Queue.new
+        }
+      }
+      puts "creating threads..."
+      entries.each { |entry| entry[:command_queue].enq("start") }
+      threads = parallel(*entries) do |entry|
+        Flare::Tools::Node.open(entry[:hostname], entry[:port], 10) do |n|
+          while command = entry[:command_queue].deq
+            case command
+            when "start"
+              # puts "#{entry[:index]}:#{entry[:hostname]}:#{entry[:port]}: start"
+              n.set("key", "0")
+              # puts "#{entry[:index]}:#{entry[:hostname]}:#{entry[:port]}: start end"
+            when "end"
+              entry[:result_queue].enq("terminated")
+              break
+            when "execute"
+              # puts "#{entry[:index]}:#{entry[:hostname]}:#{entry[:port]}: begin"
+              (0...nloop).each do |i|
+                if i%10 == 0
+                  print "."
+                  n.delete("key")
+                  n.set("key", "111")
+                elsif i%10 == 1
+                  n.decr("key", "1")
+                else
+                  n.incr("key", "1")
+                end
+                Thread.pass
+              end
+              # puts "#{entry[:index]}:#{entry[:hostname]}:#{entry[:port]}: end"
+              entry[:result_queue].enq("finished")
+            when "execute_noreply"
+              count = 0
+              (0...nloop).each do |i|
+                case rand(8)
+                when 0
+                  n.delete_noreply("key")
+                when 1
+                  n.set_noreply("key", count.to_s)
+                when 2
+                  # n.decr_noreply("key", "1")
+                  n.set_noreply("key", count.to_s)
+                else
+                  n.incr_noreply("key", "1")
+                  # n.set_noreply("key", count.to_s)
+                end
+                count = count+1
+                count = 0 if count > 1000
+              end
+              entry[:result_queue].enq("finished")
+            end
+          end
+        end
+      end
+
+      (0...ntry).each do |n|
+        entries.each {|entry|
+          command = if noreply then "execute_noreply" else "execute" end
+          entry[:command_queue].enq(command)
+        }
+        print "waiting(#{n})..."
+        if noreply && wait > 0
+          while entries[0][:result_queue].empty?
+            @node_servers[0].stop
+            sleep wait
+            @node_servers[0].cont
+            sleep wait if entries[0][:result_queue].empty?
+          end
+        end
+        entries.map {|entry| assert_equal("finished", entry[:result_queue].deq) }
+        for i in 1..5
+          failed = false
+          sleep wait
+          results = @nodes.map do |n|
+            n.get("key")
+          end
+          slave_results = results.dup
+          master_result = slave_results.shift
+          slave_results.each {|r| failed = true if master_result != r}
+          puts results.join(' ')
+          if failed
+            sleep 1
+            if i == 5
+              puts "inconsistent data: master=#{master_result}\n"
+            end
+          else
+            break
+          end
+        end
+        slave_results.each {|r| assert_equal(master_result, r)}
+      end
+      entries.each {|entry| entry[:command_queue].enq("end") }
+      entries.map {|entry| assert_equal("terminated", entry[:result_queue].deq)}
+      threads.each do |t|
+        t.join
+      end
+      puts string_of_nodelist(s.stats_nodes)
+      puts "done."
+    end
+  end # func
+
+end