roma 0.8.10 → 0.8.11

data/CHANGELOG

@@ -1,3 +1,14 @@
+*0.8.11 (Aug 8 2012)*
+
+* Allocate cmd_aliases plugin. [firejun] f7555c2
+* Add a tool for log. [firejun] bea6c87
+* Fix timezone of mapcount plugin and add tests. [Hiroki Matsue] 11f36b8
+* Make TCMemStorage available to set options. [Hiroki Matsue] 0fef58b
+* Correct end status of release command. [Hiroki Matsue] fc9a4bc
+* Fix release command feature in a small nodes. [Hiroki Matsue] c765358
+* fix:[can't add a new key into hash during iteration] occurring in leter then ruby 1.9.1 fixed. [junji torii] 2987536
+* Fix result view of mkconfig. [Hiroki Matsue] bfb0f4a
+
 *0.8.10 (Feb 20 2012)*
 
 * Add authors and homepage info to gem. [Hiroki Matsue] 35c5db4

data/ruby/server/bin/log_merger

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+path =  File.dirname(File.expand_path($PROGRAM_NAME))
+$LOAD_PATH << path + "/../lib"
+
+require 'roma/tools/log_merger'

data/ruby/server/lib/roma/async_process.rb

@@ -259,7 +259,7 @@ module Roma
         rescue => e
           @log.error("asyncev_start_release_process:#{e.inspect} #{$@}")
         end
-        @stats.run_relase = false
+        @stats.run_release = false
       }
       t[:name] = __method__
     end
@@ -443,16 +443,20 @@ module Roma
     def release_process
       @log.info("release_process:start.")
       nodes = @rttable.nodes
-      
+      nodes.delete(@stats.ap_str)
+      hosts = []
+
       unless @stats.enabled_repetition_host_in_routing
-        host = @stats.ap_str.split(/[:_]/)[0]
-        nodes.delete_if{|nid| nid.split(/[:_]/)[0] == host }
+        nodes.each{ |node|
+          host = node.split(/[:_]/)[0]
+          hosts << host unless hosts.include?(host)
+        }
       else
-        nodes.delete(@stats.ap_str)
+        hosts = nodes
       end
-      
-      if nodes.length < @rttable.rn
-        @log.error("Physcal node dose not found.")
+           
+      if hosts.length < @rttable.rn
+        @log.error("Sufficient nodes do not found.")
         return
       end
 
@@ -463,9 +467,12 @@ module Roma
           buf = nodes.clone
 
           unless @stats.enabled_repetition_host_in_routing
-            hosts = []
-            nids.each{|nid| hosts << nid.split(/[:_]/)[0]}
-            buf.delete_if{|nid| hosts.include?(nid.split(/[:_]/)[0])}
+            deny_hosts = []
+            nids.each{ |nid|
+              host = nid.split(/[:_]/)[0]
+              deny_hosts << host if host != @stats.ap_str.split(/[:_]/)[0]
+            }
+            buf.delete_if{|nid| deny_hosts.include?(nid.split(/[:_]/)[0])}
           else
             nids.each{|nid| buf.delete(nid) }
           end

data/ruby/server/lib/roma/plugin/plugin_cmd_aliases.rb

@@ -0,0 +1,43 @@
+
+module Roma
+  module CommandPlugin
+
+    module PluginCommandAliases
+      include ::Roma::CommandPlugin
+
+      # shutdown [reason]
+      def ev_shutdown(s)
+        send_data("*** ARE YOU REALLY SURE TO SHUTDOWN? *** (yes/no)\r\n")
+        if gets != "yes\r\n"
+          close_connection_after_writing
+          return
+        end
+
+        if s.length == 2
+          @log.info("Receive a shutdown #{s[1]}")
+        else
+          @log.info("Receive a shutdown command.")
+        end
+        @rttable.enabled_failover = false
+        res = broadcast_cmd("rshutdown\r\n")
+        send_data("#{res.inspect}\r\n")
+        close_connection_after_writing
+        @stop_event_loop = true
+      end
+
+      # rshutdown [reason]
+      def ev_rshutdown(s)
+        if s.length == 2
+          @log.info("Receive a rshutdown #{s[1]}")
+        else
+          @log.info("Receive a rshutdown command.")
+        end
+        @rttable.enabled_failover = false
+        send_data("BYE\r\n")
+        close_connection_after_writing
+        @stop_event_loop = true
+      end
+
+    end
+  end
+end

data/ruby/server/lib/roma/plugin/plugin_mapcount.rb

@@ -100,7 +100,7 @@ module Roma
           end
         end
 
-        v["last_updated_date"] = Time.now.gmtime.strftime(DATE_FORMAT)
+        v["last_updated_date"] = Time.now.strftime(DATE_FORMAT)
         expt = chg_time_expt(ctx.argv[2].to_i)
 
         ret_str = return_str(v, stype)
@@ -116,7 +116,7 @@ module Roma
 
         v = {}
         v = data_load(ctx.stored.value)
-        v["last_updated_date"] = Time.now.gmtime.strftime(DATE_FORMAT)
+        v["last_updated_date"] = Time.now.strftime(DATE_FORMAT)
 
         if v.is_a?(Hash)
           args = ctx.params.value.split(/\s*,\s*/)

data/ruby/server/lib/roma/storage/rh_storage.rb

@@ -19,6 +19,86 @@ module Roma
         }
       end
 
+      if RUBY_VERSION >= '1.9.2'
+        def each_clean_up(t, vnhash)
+          @do_clean_up = true
+          nt = Time.now.to_i
+          @hdb.each{ |hdb|
+            keys = hdb.keys
+            keys.each{ |k|
+              v = hdb[k]
+              return unless @do_clean_up
+              vn, last, clk, expt = unpack_header(v)
+              vn_stat = vnhash[vn]
+              if vn_stat == :primary && ( (expt != 0 && nt > expt) || (expt == 0 && t > last) )
+                yield k, vn
+                hdb.out(k) if hdb.get(k) == v
+              elsif vn_stat == nil && t > last
+                yield k, vn
+                hdb.out(k) if hdb.get(k) == v
+              end
+              sleep @each_clean_up_sleep
+            }
+          }
+        end
+
+        def each_vn_dump(target_vn)
+          count = 0
+          @divnum.times{|i|
+            tn =  Time.now.to_i
+            keys = @hdb[i].keys
+            keys.each{|k|
+              v = @hdb[i][k]
+              vn, last, clk, expt, val = unpack_data(v)
+              if vn != target_vn || (expt != 0 && tn > expt)
+                count += 1              
+                sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
+                next
+              end
+              if val
+                yield [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
+              else
+                yield [vn, last, clk, expt, k.length, k, 0].pack("NNNNNa#{k.length}N")
+              end
+            }
+          }
+        end
+
+        def each_hdb_dump(i,except_vnh = nil)
+          count = 0
+          keys = @hdb[i].keys
+          keys.each{|k|
+            v = @hdb[i][k]
+            vn, last, clk, expt, val = unpack_data(v)
+            if except_vnh && except_vnh.key?(vn) || Time.now.to_i > expt
+              count += 1
+              sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
+            else
+              yield [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
+              sleep @each_vn_dump_sleep
+            end
+          }
+        end
+
+        # Create vnode dump.
+        def get_vnode_hash(vn)
+          buf = {}
+          count = 0
+          hdb = @hdb[@hdiv[vn]]
+          keys = hdb.keys
+          keys.each{ |k|
+            v = hdb[k]
+            count += 1
+            sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
+            dat = unpack_data(v) #v.unpack('NNNN')
+            buf[k] = v if dat[0] == vn
+          }
+          return buf
+        end
+        private :get_vnode_hash
+        
+      end
+
       private
 
       def open_db(fname)

data/ruby/server/lib/roma/storage/tc_storage.rb

@@ -39,6 +39,7 @@ module Roma
         @ext_name = 'tc'
       end
 
+      alias get_stat_org get_stat
       def get_stat
         ret = super
         @hdb.each_with_index{|hdb,idx|
@@ -174,14 +175,47 @@ module Roma
     end # class TCAsyncStorage
 
 
-    class TCMemStorage < BasicStorage
+    class TCMemStorage < TCStorage
       include TokyoCabinet
 
+      def get_stat
+        ret = get_stat_org
+        @hdb.each_with_index{|hdb,idx|
+          ret["storage[#{idx}].rnum"] = hdb.rnum
+          ret["storage[#{idx}].size"] = hdb.size
+        }
+        ret
+      end
+
+      protected
+
+      def get_options(hdb)
+        prop = parse_options
+
+        prop.each_key{|k|
+          unless /^(bnum|capnum|capsiz)$/ =~ k
+            raise RuntimeError.new("Syntax error, unexpected option #{k}")
+          end
+        }
+        
+        opts = ""
+        opts += "#bnum=#{prop['bnum']}" if prop.key?('bnum')
+        opts += "#capnum=#{prop['capnum']}" if prop.key?('capnum')
+        opts += "#capsiz=#{prop['capsiz']}" if prop.key?('capsiz')
+        opts = nil unless opts.length > 0
+        opts
+      end
+
       private
 
       def open_db(fname)
         hdb = ADB::new
-        hdb.open("*")
+
+        options = get_options(hdb)
+        dbname = "*"
+        dbname += options if options
+
+        hdb.open(dbname)
         hdb
       end
 

data/ruby/server/lib/roma/tools/log_merger.rb

@@ -0,0 +1,150 @@
+#!/usr/bin/env ruby
+require 'optparse'
+
+class ReadLog
+  def initialize(fn)
+    @f = open(fn)
+    @buff = ''
+    @prev = @f.gets
+    @s_date = ''
+  end
+
+  def read_line
+    begin
+      if /^#\sLogfile\screated\son.+/ =~ @prev then
+        @prev = @f.gets
+      end
+
+      @buff = @prev
+      @prev = @f.gets
+
+      while !(match_line(@prev)) && !(@prev.nil?) do
+        @buff << @prev
+        @prev = @f.gets
+      end
+
+      match_line(@buff)
+    rescue
+      unless @f.closed?
+        @f.close
+      end
+    end
+  end
+
+  def get_line
+    @buff
+  end
+
+  def get_date
+    @s_date
+  end
+
+  def match_line(l)
+    if /^[TDIWEFU],\s\[(\d{4})\-(\d{2})\-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d+)\s#\d+\].+/ =~ l then
+      @s_date = $1 << $2 << $3 << $4 << $5 << $6 << $7
+      true
+    else
+      @s_date = nil
+      false
+    end
+  end
+end
+
+class WriteLog
+  def initialize(fn)
+    if fn == $stdout then
+      @f = $stdout
+    else
+      @f = open(fn, "w")
+    end
+  end
+
+  def write_line(th, l)
+    buff = th << ',' << l
+    @f.write(buff)
+  end
+
+  def close_file
+    unless @f.closed?
+      @f.close
+    end
+  end
+end
+
+$save_fn = $stdout
+$log_l = 'TRACE'
+$start_date = '00000000000000000000'
+$end_date = '99999999999999999999'
+o = OptionParser.new
+o.banner = "Usage : #{__FILE__} [log_file1]..[log_fileN] [-o output_file] [-l log_level]"
+o.on('-o output_file', '[output file name]', '(default=STDOUT)') {|v| $save_fn = v }
+o.on('-l log_level', '[TRACE | DEBUG | INFO | WARN | ERROR | FATAL | UNKNOWN]', '(default=TRACE)') {|v| $log_l = v}
+o.on('-s start_date', '[YYYYMMDDHHMMSS]') {|v| $start_date = v.ljust(20, "0").to_s}
+o.on('-e end_date', '[YYYYMMDDHHMMSS]') {|v| $end_date = v.ljust(20, "9").to_s}
+begin
+  o.parse!
+rescue
+  STDERR.puts o.help
+  puts  "ERROR : unrecognized option"
+  exit
+end
+
+if ARGV.length == 0 then
+  STDERR.puts o.help
+  exit
+end
+
+r = Array.new()
+w = WriteLog.new($save_fn)
+h = Hash.new()
+tmp = Hash.new()
+e_lv = nil
+
+case $log_l
+  when 'DEBUG' then
+    e_lv = 'DIWEFU'
+  when 'INFO' then
+    e_lv = 'IWEFU'
+  when 'WARN' then
+    e_lv = 'WEFU'
+  when 'ERROR' then
+    e_lv = 'EFU'
+  when 'FATAL' then
+    e_lv = 'FU'
+  when 'UNKNOWN' then
+    e_lv = 'U'
+  else
+    e_lv = 'TDIWEFU'
+end
+
+ARGV.length.times do |i|
+  r[i] = ReadLog.new(ARGV[i])
+  r[i].read_line
+  if $start_date != '00000000000000000000' then
+    while !(r[i].get_date.nil?) && $start_date > r[i].get_date do
+      r[i].read_line
+    end
+  end
+end
+
+begin
+  h.clear
+  tmp.clear
+  ARGV.length.times do |i|
+    unless r[i].get_line.nil? then
+      tmp[i] = r[i].get_date
+    end
+  end
+  if tmp.length > 0 then
+    h = tmp.sort_by{|a, b| b}
+    if /^[#{e_lv}],.+/ =~ r[h[0][0]].get_line then
+      if ($start_date <= h[0][1]) && ($end_date >= h[0][1]) then
+        w.write_line(File.basename(ARGV[h[0][0]].dup), r[h[0][0]].get_line)
+      end
+    end
+    r[h[0][0]].read_line
+  end
+end while tmp.length > 0
+
+w.close_file
+exit

data/ruby/server/lib/roma/tools/mkconfig.rb

@@ -500,7 +500,7 @@ module Roma
       puts "Before"
       Box.print_with_box(@defaults)
 
-      re_require(CONFIG_PATH, CONFIG_FULL_PATH, Config)
+      re_require(CONFIG_OUT_PATH, Config)
       results = load_config([:STORAGE_CLASS, :STORAGE_OPTION, :PLUGIN_FILES])
       print "\r\nAfter\r\n"
       Box.print_with_box(results)
@@ -523,12 +523,12 @@ module Roma
       end
     end
 
-    def re_require(lib_path, path, c_obj)
+    def re_require(path, c_obj)
       $".delete(File.expand_path(path))
       c_obj.constants.each do |cnst|
         c_obj.class_eval { remove_const cnst }
       end
-      require lib_path
+      require path
     end
 
   end # Mkconfig

data/ruby/server/lib/roma/version.rb

@@ -1,4 +1,4 @@
 
 module Roma
-  VERSION = "0.8.10"
+  VERSION = "0.8.11"
 end