roma 1.1.0 → 1.2.0

data/lib/roma/tools/check_tc_flag.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'timeout'
+
+module Roma
+  class CheckTc
+    def initialize(storage_path, library_path)
+      @storage_path = storage_path
+      @library_path = library_path
+    end
+
+    def check_flag
+      status = {}
+      timeout(5){
+        Dir.glob("#{@storage_path}/*.tc").each{|f|
+          res = `#{@library_path}/bin/tchmgr inform #{f}`
+          res =~ /additional flags:(.*)\n/
+          status.store(f, $1)
+        }
+      }
+      status
+    end
+
+  end # CheckTc
+end # Roma

data/lib/roma/tools/cpdb.rb

@@ -22,8 +22,8 @@ module Roma
     def check_storage_type
       stats('st_class') do |line|
         storage_type = line.match(/storages\[.+\]\.storage\.st_class\s(.+)/)[1].chomp
-        unless storage_type =~ /^(TCStorage|RubyHashStorage)$/
-          puts "ERROR:cpdb supports just TCStorage or RubyHashStorage system, your storage type is #{storage_type}"
+        unless storage_type =~ /^(TCStorage)$/
+          puts "ERROR:cpdb supports just TCStorage system, your storage type is #{storage_type}"
           exit
         end
       end
@@ -127,6 +127,7 @@ begin
   sc.check_storage_type
   sc.backup_all
   sc.set_gui_last_snapshot
+  puts "finished"
 ensure
   sc.close
 end

data/lib/roma/tools/mkconfig.rb

@@ -50,10 +50,12 @@ module Roma
           choice:
             - Ruby Hash
             - Tokyo Cabinet
+            - Groonga
           default: 1
           next:
             - menu
             - memory
+            - memory
         memory:
           name: memory_size_GB
           path_name:
@@ -83,9 +85,14 @@ module Roma
         plugin:
           name: selected_plugin
           path_name: plugin
-          message: Please select which plugin will you use.(plugin_storage.rb was already set)
+          message: Please select which plugin will you use.(plugin_storage.rb is essential unless you make alternative plugin.)
           choice:
-            #{load_path(PLUGIN_DIR) << "Select all plugins"}
+            #{
+              list = load_path(PLUGIN_DIR) << "Select all plugins"
+              list.delete("plugin_storage.rb")
+              list.unshift("plugin_storage.rb")
+              list
+            }
           default: 1
           next:
             #{
@@ -161,7 +168,6 @@ module Roma
         ret = Array.new
         files = Dir::entries(path)
         files.delete("plugin_stub.rb") if files.include?("plugin_stub.rb")
-        files.delete("plugin_storage.rb") if files.include?("plugin_storage.rb")
 
         files.each do |file|
           ret << file if File::ftype(File.join(path, file)) == "file"
@@ -353,7 +359,7 @@ module Roma
         clear_screen
 
         if @next_hash == "add_plugin"
-          @results["plugin"].value.unshift("plugin_storage.rb")
+          @results["plugin"].value.unshift("plugin_storage.rb") unless @results["plugin"].value.include?("plugin_storage.rb")
           @next_hash = "menu"
         end
 
@@ -504,17 +510,22 @@ module Roma
     #make config.rb based on input data
     def save_data(res)
       if res.key?("storage")
-        if res["storage"].value == "Ruby Hash"
+        case res["storage"].value
+        when "Ruby Hash"
           req = "rh_storage"
           storage = "RubyHashStorage"
-        end
-
-        if res["storage"].value == "Tokyo Cabinet"
+        when "Tokyo Cabinet"
           req = "tc_storage"
           storage = "TCStorage"
           bnum = Calculate.get_bnum(res)
           bnum = 5000000 if bnum < 5000000
           xmsiz = Calculate.get_xmsize_max(res)
+        when "Groonga"
+          req = "groonga_storage"
+          storage = "GroongaStorage"
+          bnum = Calculate.get_bnum(res)
+          bnum = 5000000 if bnum < 5000000
+          xmsiz = Calculate.get_xmsize_max(res)
         end
       end
 
@@ -532,17 +543,15 @@ module Roma
         body = ch_assign(body, "require", " ", "roma\/storage\/#{req}")
         body = ch_assign(body, "STORAGE_CLASS", "Roma::Storage::#{storage}")
 
-        if req == "rh_storage"
+        case req
+        when "rh_storage"
           body = ch_assign(body, "STORAGE_OPTION","")
-        end
-
-        if req == "tc_storage"
+        when /^(tc_storage|groonga_storage)$/
           body = ch_assign(body, "STORAGE_OPTION", "bnum=#{bnum}\#xmsiz=#{xmsiz}\#opts=d#dfunit=10")
         end
       end
 
       if res.key?("plugin")
-        res["plugin"].value.unshift("plugin_storage.rb")
         body = ch_assign(body, "PLUGIN_FILES", res["plugin"].value)
       end
 

data/lib/roma/tools/roma-adm.rb

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'timeout'
+
+module Roma
+  class Adm
+    def initialize(cmd, port)
+      @cmd = cmd.dup #to avoid frozen error
+      @port = port
+    end
+
+    def check_type
+      # waiting value input command
+      require_value_cmd = Regexp.new(/^(set|add|replace|append|prepend|cas|alist_delete|alist_include\?|alist_insert|alist_sized_insert|alist_swap_and_insert|alist_swap_and_sized_insert|alist_join_with_time|alist_join|alist_push|alist_sized_push|alist_swap_and_push|alist_update_at)/)
+
+      case @cmd
+      when "balse", "shutdown" # yes/no check
+        make_command("halt_cmd")
+        @halt_cmd = true
+      when "start" # alias cmd
+        make_command("booting")
+        @alias = true
+      when require_value_cmd
+        make_command("value")
+      else
+        t = Thread.new do
+          loop{
+            print "."
+            sleep 1
+          }
+        end
+      end
+    end
+
+    def send_command(host="localhost")
+      if @alias
+        base_path = Pathname(__FILE__).dirname.parent.parent.parent.expand_path
+        `#{base_path}/#{@cmd}` # bash
+      elsif @halt_cmd
+        return `echo -e "#{@cmd}" | nc -i1 #{host} #{@port}` # bash
+      else
+        timeout(5) {
+          res = []
+          TCPSocket.open(host, @port) do |sock|
+            sock.puts @cmd
+            while select [sock], nil, nil, 0.5
+              res << sock.gets.chomp!
+            end
+          end
+          return res
+        }
+      end
+    end
+
+    private
+
+    def make_command(type)
+      case type
+      when "halt_cmd"
+        puts("Are you sure?(yes/no)\r\n")
+        if STDIN.gets.chomp != "yes"
+          raise "confirmation was rejected"
+        else
+          @cmd.concat("\r\nyes\r\nyes\r\n")
+        end
+      when "booting"
+        puts("Please input hostname or ip address which is used for ROMA.\r\n  Ex.) roma_serverA, 192.168.33.11\r\n")
+        hostname = STDIN.gets.chomp
+        puts("Please input PATH of config.rb.\r\n  Ex.) /home/roma/config.rb\r\n")
+        config_path = STDIN.gets.chomp
+        @cmd = "bin/romad #{hostname} -p #{@port} -d --config #{config_path}"
+      when "value"
+        puts("Please input value.\r\n")
+        value = STDIN.gets.chomp
+        @cmd.concat("\r\n#{value}\r\n")
+      end
+    end
+
+  end #Adm
+end # Roma
+

data/lib/roma/version.rb

@@ -1,4 +1,4 @@
 
 module Roma
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

data/test/config4mhash.rb

@@ -4,6 +4,8 @@ require 'roma/storage/tc_storage'
 module Roma
   
   module Config
+    VERSION = "1.2.0"
+
     DEFAULT_PORT = 12000
     DEFAULT_NAME = 'ROMA'
 

data/test/config4storage_error.rb

@@ -4,6 +4,8 @@ require 'roma/storage/storage_error_storage'
 module Roma
   
   module Config
+    VERSION = "1.2.0"
+
     DEFAULT_PORT = 12000
     DEFAULT_NAME = 'ROMA'
 

data/test/config4test.rb

@@ -3,6 +3,8 @@ require 'roma/storage/rh_storage'
 module Roma
   
   module Config
+    VERSION = "1.2.0"
+
     DEFAULT_PORT = 12000
     DEFAULT_NAME = 'ROMA'
 

data/test/cpdbtest/config4cpdb_base.rb

@@ -0,0 +1,67 @@
+module CpdbBase
+  
+  module Config
+    VERSION = "1.2.0"
+
+    DEFAULT_PORT = 12000
+    DEFAULT_NAME = 'ROMA'
+
+    # :no_action | :auto_assign | :shutdown
+    DEFAULT_LOST_ACTION = :auto_assign
+
+    # log setting
+    LOG_SHIFT_AGE = 10
+    LOG_SHIFT_SIZE = 1024 * 1024 * 10
+    LOG_PATH = '.'
+    # :debug | :info | :warn | :error
+    LOG_LEVEL = :debug
+
+    # routing setting
+    RTTABLE_PATH = '.'
+
+    # connection setting
+    # like a MaxStartups spec in the sshd_config
+    # 'start:rate:full'
+    CONNECTION_CONTINUOUS_LIMIT = '200:30:300'
+
+    # storage setting
+    #STORAGE_CLASS = Roma::Storage::GroongaStorage
+    STORAGE_DIVNUM = 10
+    STORAGE_PATH = '.'
+    STORAGE_DUMP_PATH = '/tmp'
+    STORAGE_OPTION = ''
+
+    # 5 days ago
+    STORAGE_DELMARK_EXPTIME = 60 * 60 * 24 * 5
+
+    # data copy setting
+    DATACOPY_STREAM_COPY_WAIT_PARAM = 0.0001
+
+    # plugin setting
+    PLUGIN_FILES = ['plugin_storage.rb','plugin_alist.rb','plugin_map.rb','plugin_mapcount.rb']
+
+    # write-behind setting
+    WRITEBEHIND_PATH = './wb'
+    WRITEBEHIND_SHIFT_SIZE = 1024 * 1024 * 10
+
+    # redundant setting
+    REDUNDANT_ZREDUNDANT_SIZE = 0
+
+    def self.get_stat
+      ret = {}
+      ret['config.DEFAULT_LOST_ACTION'] = DEFAULT_LOST_ACTION
+      ret['config.LOG_SHIFT_AGE'] = LOG_SHIFT_AGE
+      ret['config.LOG_SHIFT_SIZE'] = LOG_SHIFT_SIZE
+      ret['config.LOG_PATH'] = File.expand_path(LOG_PATH)
+      ret['config.RTTABLE_PATH'] = File.expand_path(RTTABLE_PATH)
+      ret['config.STORAGE_DELMARK_EXPTIME'] = STORAGE_DELMARK_EXPTIME
+      ret['config.DATACOPY_STREAM_COPY_WAIT_PARAM'] = DATACOPY_STREAM_COPY_WAIT_PARAM
+      ret['config.PLUGIN_FILES'] = PLUGIN_FILES.inspect
+      ret['config.WRITEBEHIND_PATH'] = File.expand_path(WRITEBEHIND_PATH)
+      ret['config.WRITEBEHIND_SHIFT_SIZE'] = WRITEBEHIND_SHIFT_SIZE
+      ret
+    end
+
+  end
+
+end

data/test/cpdbtest/config4cpdb_dbm.rb

@@ -0,0 +1,9 @@
+require 'roma/storage/dbm_storage'
+require_relative 'config4cpdb_base'
+
+module Roma
+  module Config
+    include CpdbBase::Config
+    STORAGE_CLASS = Roma::Storage::DbmStorage
+  end
+end

data/test/cpdbtest/config4cpdb_groonga.rb

@@ -0,0 +1,9 @@
+require 'roma/storage/groonga_storage'
+require_relative 'config4cpdb_base'
+
+module Roma
+  module Config
+    include CpdbBase::Config
+    STORAGE_CLASS = Roma::Storage::GroongaStorage
+  end
+end

data/test/cpdbtest/config4cpdb_rh.rb

@@ -0,0 +1,9 @@
+require 'roma/storage/rh_storage'
+require_relative 'config4cpdb_base'
+
+module Roma
+  module Config
+    include CpdbBase::Config
+    STORAGE_CLASS = Roma::Storage::RubyHashStorage
+  end
+end

data/test/cpdbtest/config4cpdb_sqlite3.rb

@@ -0,0 +1,9 @@
+require 'roma/storage/sqlite3_storage'
+require_relative 'config4cpdb_base'
+
+module Roma
+  module Config
+    include CpdbBase::Config
+    STORAGE_CLASS = Roma::Storage::SQLite3Storage
+  end
+end

data/test/cpdbtest/config4cpdb_tc.rb

@@ -0,0 +1,9 @@
+require 'roma/storage/tc_storage'
+require_relative 'config4cpdb_base'
+
+module Roma
+  module Config
+    include CpdbBase::Config
+    STORAGE_CLASS = Roma::Storage::TCStorage
+  end
+end

data/test/cpdbtest/config4cpdb_tcmem.rb

@@ -0,0 +1,9 @@
+require 'roma/storage/tc_storage'
+require_relative 'config4cpdb_base'
+
+module Roma
+  module Config
+    include CpdbBase::Config
+    STORAGE_CLASS = Roma::Storage::TCMemStorage
+  end
+end

data/test/roma-test-utils.rb

@@ -4,71 +4,171 @@ require 'fileutils'
 require 'rbconfig'
 require 'roma/config'
 require 'roma/messaging/con_pool'
+require 'roma/logging/rlogger'
+require 'roma/client/rclient'
+require 'roma/client/client_pool'
+
+Roma::Client::RomaClient.class_eval do
+  def init_sync_routing_proc
+  end
+end
 
 module RomaTestUtils
+  DEFAULT_CONFIG = 'config4test.rb'
+  DEFAULT_HOST = 'localhost'
+  DEFAULT_IP = '127.0.0.1'
+  DEFAULT_PORTS = %w(11211 11212)
+  DEFAULT_NODES = DEFAULT_PORTS.map { |port| "#{DEFAULT_HOST}_#{port}" }
+  DEFAULT_TIMEOUT_SEC = 90
+  SHELL_LOG = 'roma_test_outputs.log'
+
+  Roma::Logging::RLogger.create_singleton_instance("#{Roma::Config::LOG_PATH}/roma_test.log",
+                                                   Roma::Config::LOG_SHIFT_AGE,
+                                                   Roma::Config::LOG_SHIFT_SIZE)
+
   module_function
+
+  def start_roma(conf = DEFAULT_CONFIG, div_bits: 3, replication_in_host: true)
+    FileUtils.rm_rf(Dir.glob("#{Roma::Config::STORAGE_PATH}/#{DEFAULT_NODES[0][0..-2]}?*"))
+    FileUtils.rm_rf(Dir.glob("#{Roma::Config::STORAGE_PATH}/#{DEFAULT_IP}_#{DEFAULT_PORTS[0][0..-2]}?*"))
+    sleep 0.1
+
+    system("#{ruby_path} #{mkroute_path} #{DEFAULT_NODES.join(' ')} -d #{div_bits} --replication_in_host >> #{SHELL_LOG} 2>&1")
+    sleep 0.2
+
+    DEFAULT_NODES.each do |node|
+      do_command_romad(node, conf, replication_in_host)
+    end
+    sleep 1
+  end
+
+  def join_roma(node, conf = DEFAULT_CONFIG, replication_in_host: true)
+    do_command_romad(node, conf, replication_in_host, true)
+  end
+
+  def do_command_romad(node, conf, replication_in_host = true, is_join = false)
+    host, port = node.split('_')
+    romad_command = [
+      ruby_path, romad_path, host,
+      '-p', port,
+      '-d', '--verbose', '--disabled_cmd_protect',
+      '--config', "#{test_dir}/#{conf}"
+    ]
+    romad_command << '--replication_in_host' if replication_in_host
+    romad_command << "-j #{DEFAULT_NODES[0]}" if is_join
+    romad_command << ">> #{SHELL_LOG} 2>&1"
+
+    system(romad_command.join(' '))
+  end
+
+  def get_client
+    client_pool = Roma::Client::ClientPool.instance
+    client_pool.servers = DEFAULT_NODES
+    client_pool.client
+  end
+
+  def wait_join(node)
+    client = get_client
+    wait_count = 0
+    retry_count = 0
+    sleep 5
+
+    until client.rttable.nodes.include?(node)
+      sleep 1
+      client.update_rttable
+      wait_count += 1
+      fail "#{__method__} timeout" if wait_count > DEFAULT_TIMEOUT_SEC
+    end
+
+    while client.stats(node: node)['stats.run_join'] == 'true'
+      sleep 1
+      wait_count += 1
+      fail "#{__method__} timeout" if wait_count > DEFAULT_TIMEOUT_SEC
+    end
+  end
+
+  def wait_failover(down_node)
+    client = get_client
+    stats_node = down_node == DEFAULT_NODES[0] ? DEFAULT_NODES[1] : DEFAULT_NODES[0]
+    wait_count = 0
+    sleep 1
+
+    while client.stats(node: stats_node)['routing.nodes'] =~ /#{down_node}/
+      sleep 1
+      wait_count += 1
+      fail "#{__method__} timeout" if wait_count > DEFAULT_TIMEOUT_SEC
+    end
+  end
+
+  def wait_release(node)
+    client = get_client
+    wait_count = 0
+    sleep 1
+
+    while client.stats(node: node)['stats.run_release'] == 'true'
+      sleep 1
+      wait_count += 1
+      fail "#{__method__} timeout" if wait_count > DEFAULT_TIMEOUT_SEC
+    end
+  end
+
+  def stop_roma
+    balse_message = %w(balse yes)
+    send_message(messages: balse_message)
+    Roma::Client::ConPool.instance.close_all
+    Roma::Messaging::ConPool.instance.close_all
+  end
+
+  def stop_roma_node(node)
+    send_message(messages: 'rbalse', node: node)
+    Roma::Client::ConPool.instance.close_all
+    Roma::Messaging::ConPool.instance.close_all
+  end
+
+  def release_roma_node(node)
+    send_message(messages: 'release', node: node)
+  end
+
+  private
+
   def base_dir
     Pathname(__FILE__).dirname.parent.expand_path
   end
 
   def bin_dir
-    base_dir + "bin"
+    base_dir + 'bin'
   end
 
   def test_dir
-    base_dir + "test"
+    base_dir + 'test'
   end
 
   def mkroute_path
-    (bin_dir + "mkroute").to_s
+    (bin_dir + 'mkroute').to_s
   end
 
   def romad_path
-    (bin_dir + "romad").to_s
+    (bin_dir + 'romad').to_s
   end
 
   def ruby_path
-    File.join(RbConfig::CONFIG["bindir"],
-              RbConfig::CONFIG["ruby_install_name"])
+    File.join(RbConfig::CONFIG['bindir'],
+              RbConfig::CONFIG['ruby_install_name'])
   end
 
-  def start_roma conf='config4test.rb'
-    sh = Shell.new
-    sh.transact do
-      Dir.glob("localhost_1121?.*").each{|f| rm f }
-    end
-    FileUtils.rm_rf("#{Roma::Config::STORAGE_PATH}/localhost_11211")
-    FileUtils.rm_rf("#{Roma::Config::STORAGE_PATH}/localhost_11212")
-    sleep 0.1
-
-    sh.system(ruby_path, mkroute_path,
-              "localhost_11211","localhost_11212",
-              "-d","3",
-              "--replication_in_host")
-    sleep 0.2
-    do_command_romad conf
-    sleep 1
-  end
+  def send_message(messages: [], node: DEFAULT_NODES[0])
+    conn = Roma::Messaging::ConPool.instance.get_connection(node)
+    return false unless conn
 
-  def do_command_romad conf
-    sh = Shell.new
-    sh.system(ruby_path,romad_path,"localhost","-p","11211","-d","--verbose",
-              "--disabled_cmd_protect","--config","#{test_dir}/#{conf}")
-    sh.system(ruby_path,romad_path,"localhost","-p","11212","-d","--verbose",
-              "--disabled_cmd_protect","--config","#{test_dir}/#{conf}")
-  end
+    messages = [messages] if messages.class == String
 
-  def stop_roma
-    conn = Roma::Messaging::ConPool.instance.get_connection("localhost_11211")
-    if conn
-      conn.write "balse\r\n"
-      conn.gets
-      conn.write "yes\r\n"
+    messages.each do |message|
+      conn.write "#{message}\r\n"
       conn.gets
-      conn.close
     end
-    Roma::Client::ConPool.instance.close_all
-  rescue =>e
-    puts "#{e} #{$@}"
+  rescue => e
+    puts "#{e} #{$ERROR_POSITION}"
+  ensure
+    Roma::Messaging::ConPool.instance.return_connection(node, conn)
   end
 end