roma 1.0.0 → 1.1.0

data/{ruby/server/lib → lib}/roma/plugin/plugin_storage.rb

@@ -41,7 +41,7 @@ module Roma
         end
 
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
         data = @storages[hname].get(vn, key, 0)
@@ -64,7 +64,7 @@ module Roma
         end
 
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
         data = @storages[hname].get(vn, key, 0)
@@ -138,7 +138,7 @@ module Roma
           return send_data("#{res}\r\n")
         end
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
 
@@ -180,7 +180,7 @@ module Roma
           return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
         end
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
 
@@ -218,7 +218,7 @@ module Roma
         d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
         vn = @rttable.get_vnode_id(d)
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
         if @storages[hname].rdelete(vn, key, d, s[2].to_i)
@@ -319,7 +319,7 @@ module Roma
         end
         
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
 
@@ -358,7 +358,7 @@ module Roma
         end
         
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
 
@@ -413,7 +413,7 @@ module Roma
           @log.error("forward get failed:nid=#{nid} key=#{k}")
           return nil
         elsif res == "END\r\n"
-          # value dose not found
+          # value does not found
         elsif res.start_with?("ERROR")
           @rttable.proc_succeed(nid)
           con.close_connection
@@ -454,7 +454,7 @@ module Roma
       def store(fnc, hname, vn, k, d, expt, v, nodes)
         expt = chg_time_expt(expt)
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
 
@@ -481,7 +481,7 @@ module Roma
       def store_cas(hname, vn, k, d, clk, expt, v, nodes)
         expt = chg_time_expt(expt)
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
 
@@ -587,7 +587,7 @@ module Roma
 
       def store_incr_decr(fnc, hname, vn, k, d,  v, nodes)
         unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
         end
 

data/{ruby/server/lib → lib}/roma/romad.rb

@@ -156,12 +156,21 @@ module Roma
       if Config.const_defined?(:DATACOPY_STREAM_COPY_WAIT_PARAM)
         @stats.stream_copy_wait_param = Config::DATACOPY_STREAM_COPY_WAIT_PARAM
       end
+      if Config.const_defined?(:LOG_STREAM_SHOW_WAIT_PARAM)
+        @stats.stream_show_wait_param = Config::LOG_STREAM_SHOW_WAIT_PARAM
+      end
       if Config.const_defined?(:WB_COMMAND_MAP)
         @stats.wb_command_map = Config::WB_COMMAND_MAP
       end
       if Config.const_defined?(:STORAGE_CLEAN_UP_INTERVAL)
         @stats.clean_up_interval = Config::STORAGE_CLEAN_UP_INTERVAL
       end
+      if Config.const_defined?(:LOG_SHIFT_SIZE)
+        @stats.log_shift_size = Config::LOG_SHIFT_SIZE
+      end
+      if Config.const_defined?(:LOG_SHIFT_AGE)
+        @stats.log_shift_age = Config::LOG_SHIFT_AGE
+      end
     end
 
     def initialize_connection
@@ -283,9 +292,16 @@ module Roma
 
       opts.on("-n", "--name [name]") { |v| @stats.name = v }
 
+      ##
+      # "--enabled_repeathost" is deplicated. We will rename it to "--replication_in_host"
+      ##
       @stats.enabled_repetition_host_in_routing = false
       opts.on(nil,"--enabled_repeathost", "Allow redundancy to same host"){
         @stats.enabled_repetition_host_in_routing = true
+        puts "Warning: \"--enabled_repeathost\" is deplicated. Please use \"--replication_in_host\""
+      }
+      opts.on(nil,"--replication_in_host", "Allow redundancy to same host"){
+        @stats.enabled_repetition_host_in_routing = true
       }
 
       @stats.disabled_cmd_protect = false
@@ -349,6 +365,7 @@ module Roma
           st = st_class.new
           st.storage_path = "#{path}/#{hname}"
           st.vn_list = @rttable.vnodes
+          st.st_class = st_class
           st.divnum = st_divnum
           st.option = st_option
           @storages[hname] = st
@@ -359,6 +376,7 @@ module Roma
         st = st_class.new
         st.storage_path = "#{path}/#{hname}"
         st.vn_list = @rttable.vnodes
+        st.st_class = st_class
         st.divnum = st_divnum
         st.option = st_option
         @storages[hname] = st
@@ -574,6 +592,12 @@ module Roma
     end
 
     def node_check(nid)
+      if @startup && @rttable.enabled_failover == false
+        unless Roma::Messaging::ConPool.instance.check_connection(nid) 
+          @log.info("I'm wating for booting the #{nid} instance.")
+          return false
+        end
+      end
       name = async_send_cmd(nid,"whoami\r\n",2)
       return false unless name
       if name != @stats.name
@@ -606,7 +630,7 @@ module Roma
 
       idx=nodes.index(@stats.ap_str)
       unless idx
-        @log.error("My node-id(=#{@stats.ap_str}) dose not found in the routingtable.")
+        @log.error("My node-id(=#{@stats.ap_str}) does not found in the routingtable.")
         EventMachine::stop_event_loop
         return
       end

data/{ruby/server/lib → lib}/roma/stats.rb

@@ -37,6 +37,7 @@ module Roma
 
     # proc param
     attr_accessor :stream_copy_wait_param
+    attr_accessor :stream_show_wait_param
     attr_accessor :dcnice
     attr_accessor :clean_up_interval
 
@@ -76,6 +77,10 @@ module Roma
     attr_accessor :gui_run_gather_logs
     attr_accessor :gui_last_snapshot
 
+    # for log rotate
+    attr_accessor :log_shift_size
+    attr_accessor :log_shift_age
+
     def initialize
       @config_path = nil
       @run_recover = false
@@ -92,6 +97,7 @@ module Roma
       @gui_last_snapshot = []
       @spushv_protection = false
       @stream_copy_wait_param = 0.0001
+      @stream_show_wait_param = 0.001
       @dcnice = 3
       @clean_up_interval = 300
       @enabled_vnodes_balance = nil
@@ -113,6 +119,8 @@ module Roma
       @spushv_read_timeout = 100
       @reqpushv_timeout_count = 300 # 0.1 * 300 sec
       @routing_trans_timeout = 3600 * 3 # 3hr
+      @log_shift_size = 1048576
+      @log_shift_age = 0
     end
 
     def ap_str
@@ -141,6 +149,7 @@ module Roma
       ret['stats.gui_last_snapshot'] = @gui_last_snapshot
       ret['stats.spushv_protection'] = @spushv_protection
       ret['stats.stream_copy_wait_param'] = @stream_copy_wait_param
+      ret['stats.stream_show_wait_param'] = @stream_show_wait_param
       ret['stats.dcnice'] = @dcnice
       ret['stats.clean_up_interval'] = @clean_up_interval
       ret['stats.size_of_zredundant'] = @size_of_zredundant
@@ -160,6 +169,8 @@ module Roma
       ret['stats.spushv_read_timeout'] = @spushv_read_timeout
       ret['stats.reqpushv_timeout_count'] = @reqpushv_timeout_count
       ret['stats.routing_trans_timeout'] = @routing_trans_timeout
+      ret['stats.log_shift_size'] = @log_shift_size
+      ret['stats.log_shift_age'] = @log_shift_age
       ret
     end
 

data/{ruby/server/lib → lib}/roma/storage/basic_storage.rb

@@ -18,6 +18,7 @@ module Roma
       attr_writer :storage_path
       attr_writer :option
 
+      attr_accessor :st_class
       attr_accessor :divnum
       attr_accessor :each_vn_dump_sleep
       attr_accessor :each_vn_dump_sleep_count
@@ -42,6 +43,7 @@ module Roma
 
         @ext_name = 'db'
 
+        @st_class = nil
         @divnum = 10
 
         @each_vn_dump_sleep = 0.001
@@ -58,6 +60,7 @@ module Roma
       def get_stat
         ret = {}
         ret['storage.storage_path'] = File.expand_path(@storage_path)
+        ret['storage.st_class'] = @st_class.to_s.match(/Roma::Storage::(.*)/)[1]
         ret['storage.divnum'] = @divnum
         ret['storage.option'] = @option
         ret['storage.each_vn_dump_sleep'] = @each_vn_dump_sleep

data/lib/roma/storage/groonga_storage.rb

@@ -0,0 +1,101 @@
+require 'groonga'
+require 'roma/storage/basic_storage'
+
+module Roma
+  module Storage
+
+    class GroongaStorage < BasicStorage
+      def initialize
+        super
+        @ext_name = 'grn'
+      end
+
+      def get_stat
+        ret = super
+        @hdb.each_with_index do |hdb, i|
+          ret["storage[#{i}].path"] = File.expand_path(hdb.path)
+          ret["storage[#{i}].rnum"] = hdb.rnum
+        end
+        ret
+      end
+
+      private
+      def open_db(fname)
+        hdb = GroongaHash.new(fname)
+        hdb.open
+        hdb
+      end
+
+      def close_db(hdb)
+        hdb.close
+      end
+
+      class GroongaHash
+        def initialize(fname)
+          @fname = fname
+        end
+
+        def path
+          @hash.path
+        end
+
+        def put(key, value)
+          record = @hash.add(key)
+          @value[record.id] = value
+        end
+
+        def get(key)
+          record = @hash[key]
+          return nil if record.nil?
+          @value[record.id]
+        end
+
+        def out(key)
+          record = @hash[key]
+          if record
+            record.delete
+            true
+          else
+            false
+          end
+        end
+
+        def rnum
+          @hash.count
+        end
+
+        def each
+          @hash.each do |record|
+            yield(record.key, @value[record.id])
+          end
+        end
+
+        def open
+          @context = Groonga::Context.new(:encoding => :none)
+          if File.exist?(@fname)
+            @database = Groonga::Database.new(@fname, :context => @context)
+          else
+            @database = Groonga::Database.create(:context => @context,
+                                                 :path => @fname)
+            Groonga::Schema.define(:context => @context) do |schema|
+              schema.create_table("hash",
+                                  :type => :hash,
+                                  :key_type => "ShortText") do |table|
+                table.text("value")
+              end
+            end
+          end
+
+          @hash = @context["hash"]
+          @value = @hash.column("value")
+        end
+
+        def close
+          @database.close
+          @context.close
+          @hash = @value = @database = @context = nil
+        end
+      end
+    end # class GroongaStorage
+  end # module Storage
+end # module Roma

data/{ruby/server/lib → lib}/roma/tools/cpdb.rb

@@ -19,6 +19,16 @@ module Roma
       end
     end
 
+    def check_storage_type
+      stats('st_class') do |line|
+        storage_type = line.match(/storages\[.+\]\.storage\.st_class\s(.+)/)[1].chomp
+        unless storage_type =~ /^(TCStorage|RubyHashStorage)$/
+          puts "ERROR:cpdb supports just TCStorage or RubyHashStorage system, your storage type is #{storage_type}"
+          exit
+        end
+      end
+    end
+
     def backup(hname)
       stat = get_safecopy_stats(hname)
       if stat.uniq != [:normal]
@@ -93,8 +103,8 @@ module Roma
       @con.gets
     end
 
-    def stats
-      @con.puts "stat storage\r\n"
+    def stats(regexp = "storage")
+      @con.puts "stat #{regexp}\r\n"
       yield $_ while @con.gets != "END\r\n"
     end
 
@@ -114,6 +124,7 @@ end
 sc = Roma::SafeCopy.new("localhost", ARGV[0].to_i)
 
 begin
+  sc.check_storage_type
   sc.backup_all
   sc.set_gui_last_snapshot
 ensure

data/{ruby/server/lib → lib}/roma/tools/key_access.rb

@@ -68,7 +68,7 @@ module Roma
 
     def open_storage(path)
       unless File::directory?(path)
-        STDERR.puts "#{path} dose not found."
+        STDERR.puts "#{path} does not found."
         return nil
       end
 

data/{ruby/server/lib → lib}/roma/tools/key_list.rb

@@ -46,7 +46,7 @@ module Roma
 
     def open_storage(path)
       unless File::directory?(path)
-        STDERR.puts "#{path} dose not found."
+        STDERR.puts "#{path} does not found."
         return nil
       end
 

data/{ruby/server/lib → lib}/roma/tools/mkconfig.rb

@@ -359,7 +359,7 @@ module Roma
 
         skip.call if @next_hash == "menu" || @next_hash == "server" || @next_hash == "fd_server" || @next_hash == "check_plugin"
         break if end?(@base[@next_hash])
-        puts "if you dosen't input anything, default value is set."
+        puts "if you doesn't input anything, default value is set."
         Box.print_with_box(@defaults)
         print_status(@results)
         @base.print_question(@next_hash)

data/{ruby/server/lib → lib}/roma/tools/mkrecent.rb

@@ -103,7 +103,7 @@ module Roma
 
     def ropen(path)
       unless File::directory?(path)
-        STDERR.puts "#{path} dose not found."
+        STDERR.puts "#{path} does not found."
         return nil
       end
 

data/{ruby/server/lib → lib}/roma/tools/mkroute.rb

@@ -16,7 +16,11 @@ opts.banner = "usage:#{File.basename($0)} [options] node-id..."
 opts.on("-h","--hash [bits]","(default=32)"){|v| dgst_bits = v.to_i }
 opts.on("-d","--divide [bits]","(default=9)"){|v| div_bits = v.to_i }
 opts.on("-r","--redundant [num]","(default=2)"){|v| rn = v.to_i }
-opts.on(nil,"--enabled_repeathost"){|v| repeathost=true }
+opts.on(nil,"--enabled_repeathost"){|v| 
+  repeathost=true
+  puts "Warning: \"--enabled_repeathost\" is deplicated. Please use \"--replication_in_host\""
+}
+opts.on(nil,"--replication_in_host"){|v| repeathost=true }
 opts.parse!(ARGV)
 
 nodes = ARGV

data/{ruby/server/lib → lib}/roma/tools/recoverlost_lib.rb

@@ -112,7 +112,7 @@ module Roma
 
     def open_storage(path,vn_list)
       unless File::directory?(path)
-        STDERR.puts "#{path} dose not found."
+        STDERR.puts "#{path} does not found."
         return nil
       end
 

data/{ruby/server/lib → lib}/roma/tools/ssroute.rb

@@ -15,7 +15,7 @@ begin
   if Roma::Routing::RoutingData::snapshot("#{ap}.route")
     puts "succeed"
   else
-    puts "Routing-log file dose not found."
+    puts "Routing-log file does not found."
   end
 rescue =>e
   puts "error:#{e}"

data/{ruby/server/lib → lib}/roma/tools/test-scenario.rb

@@ -110,7 +110,7 @@ module Roma
       def init_roma
         @log.debug "begin init_roma"
         exec "rm -f localhost_1121?.*"
-        exec "bin/mkroute -d 7 #{RomaProc.to_str(@roma_procs)} --enabled_repeathost"
+        exec "bin/mkroute -d 7 #{RomaProc.to_str(@roma_procs)} --replication_in_host"
         @log.debug "end init_roma"
       end
 
@@ -124,7 +124,7 @@ module Roma
 
       def start_roma_proc i
         @log.debug "begin start_roma_proc"
-        str = "bin/romad #{@roma_procs[i].addr} -p #{@roma_procs[i].port.to_s} -d --enabled_repeathost"
+        str = "bin/romad #{@roma_procs[i].addr} -p #{@roma_procs[i].port.to_s} -d --replication_in_host"
         exec str
         @roma_procs[i].pid = get_pid(str)
         @log.debug "end start_roma_proc"
@@ -313,7 +313,7 @@ cnf = Roma::Test::Config.new(ARGV)
 # check for a working path
 unless File::exist?("#{cnf.working_path}/bin/romad")
   # in invalid path
-  $stderr.puts "#{cnf.working_path}/bin/romad dose't found"
+  $stderr.puts "#{cnf.working_path}/bin/romad does't found"
   $stderr.puts "You should set to a working path option(-p)."
   exit 1
 end

data/{ruby/server/lib → lib}/roma/tools/tribunus.rb

@@ -6,7 +6,7 @@ UDP_PORT=14329
 MULTICAST_ADDR="225.0.0.123"
 ROMA_LOAD_PATH=File.expand_path(File.join(File.dirname(__FILE__),"../.."))
 RUBY_COMMAND_OPTIONS=["-I",ROMA_LOAD_PATH]
-ROMAD_OPTIONS=["--enabled_repeathost"]
+ROMAD_OPTIONS=["--replication_in_host"]
 bin_dir=File.expand_path(File.join(File.dirname(__FILE__),"../../../bin"))
 ROMAD_PATH= File.expand_path(File.join(bin_dir,"romad"))
 MKROUTE_PATH= File.expand_path(File.join(bin_dir,"mkroute"))

data/lib/roma/version.rb

@@ -0,0 +1,4 @@
+
+module Roma
+  VERSION = "1.1.0"
+end

data/{ruby/server/test → test}/roma-test-utils.rb

@@ -8,27 +8,23 @@ require 'roma/messaging/con_pool'
 module RomaTestUtils
   module_function
   def base_dir
-    Pathname(__FILE__).dirname.parent.parent.expand_path
+    Pathname(__FILE__).dirname.parent.expand_path
   end
 
-  def server_base_dir
-    base_dir + "server"
+  def bin_dir
+    base_dir + "bin"
   end
 
-  def server_bin_dir
-    server_base_dir + "bin"
-  end
-
-  def server_test_dir
-    server_base_dir + "test"
+  def test_dir
+    base_dir + "test"
   end
 
   def mkroute_path
-    (server_bin_dir + "mkroute").to_s
+    (bin_dir + "mkroute").to_s
   end
 
   def romad_path
-    (server_bin_dir + "romad").to_s
+    (bin_dir + "romad").to_s
   end
 
   def ruby_path
@@ -48,7 +44,7 @@ module RomaTestUtils
     sh.system(ruby_path, mkroute_path,
               "localhost_11211","localhost_11212",
               "-d","3",
-              "--enabled_repeathost")
+              "--replication_in_host")
     sleep 0.2
     do_command_romad conf
     sleep 1
@@ -57,9 +53,9 @@ module RomaTestUtils
   def do_command_romad conf
     sh = Shell.new
     sh.system(ruby_path,romad_path,"localhost","-p","11211","-d","--verbose",
-              "--disabled_cmd_protect","--config","#{server_test_dir}/#{conf}")
+              "--disabled_cmd_protect","--config","#{test_dir}/#{conf}")
     sh.system(ruby_path,romad_path,"localhost","-p","11212","-d","--verbose",
-              "--disabled_cmd_protect","--config","#{server_test_dir}/#{conf}")
+              "--disabled_cmd_protect","--config","#{test_dir}/#{conf}")
   end
 
   def stop_roma

data/test/run-test.rb

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+require 'pathname'
+
+base_path = Pathname(__FILE__).dirname.parent.expand_path
+
+$LOAD_PATH.unshift("#{base_path}/lib")
+$LOAD_PATH.unshift("#{base_path}/test")
+
+client_base_path = Pathname(__FILE__).dirname.parent.parent.expand_path
+$LOAD_PATH.unshift("#{client_base_path}/roma-ruby-client/lib")
+
+require 'roma-test-utils'
+
+Dir["#{base_path}/test/t_*.rb"].each do |test_file|
+  require File.basename(test_file, '*.rb')
+end
+
+exit(Test::Unit::AutoRunner.run)