roma 0.8.14 → 1.0.0

data/ruby/server/lib/roma/config.rb

@@ -26,6 +26,7 @@ module Roma
 
     # routing setting
     RTTABLE_PATH = '.'
+    ROUTING_EVENT_LIMIT_LINE = 1000
 
     # connection setting
 

data/ruby/server/lib/roma/plugin/plugin_gui.rb

@@ -0,0 +1,66 @@
+require 'json'
+
+module Roma
+  module CommandPlugin
+
+    module PluginGui
+      include ::Roma::CommandPlugin
+
+      # get_routing_history
+      def ev_get_routing_history(s)
+        routing_path  = get_config_stat["config.RTTABLE_PATH"]
+        f_list = Dir.glob("#{routing_path}/*")
+        contents = ""
+        f_list.each{|fname|
+          contents << File.read(fname)
+        }
+        routing_list = contents.scan(/[-\.a-zA-Z\d]+_[\d]+/).uniq.sort
+        routing_list.each{|routing|
+          send_data("#{routing}\r\n")
+        }
+        send_data("END\r\n")
+      end
+
+      # get_logs [line count]
+      def ev_gather_logs(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments (#{s.length-1} for 1)\r\n")
+        end
+
+        line_count = s[1].to_i
+        if line_count < 1 || line_count > 100
+          return send_data("CLIENT_ERROR line counts is restricted to between 1-100 lines\r\n")
+        end
+
+        if @stats.gui_run_gather_logs
+          return send_data("CLIENT_ERROR gathering process is already going\r\n")
+        end
+
+        begin
+          @stats.gui_run_gather_logs = true
+          Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_get_logs', [line_count]))
+
+          send_data("STARTED\r\n")
+        rescue
+          @stats.gui_run_gather_logs = false
+          @rttable.logs = []
+          send_data("CLIENT_ERROR\r\n")
+        end
+      end
+
+      # show_logs
+      def ev_show_logs(s)
+        if @stats.gui_run_gather_logs
+          send_data("Not finished gathering\r\n")
+        else
+          @rttable.logs.each{|log|
+            send_data(log)
+          }
+          send_data("END\r\n")
+          @rttable.logs.clear
+        end
+      end
+
+    end # end of module PluginGui
+  end # end of module CommandPlugin
+end # end of modlue Roma

data/ruby/server/lib/roma/plugin/plugin_storage.rb

@@ -141,6 +141,12 @@ module Roma
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
           return
         end
+
+        if @stats.wb_command_map.key?(:delete__prev)
+          data = @storages[hname].get(vn, key, d)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:delete__prev], key, data) if data
+        end
+        
         res = @storages[hname].delete(vn, key, d)
         @stats.delete_count += 1
 
@@ -177,6 +183,12 @@ module Roma
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
           return
         end
+
+        if @stats.wb_command_map.key?(:delete__prev)
+          data = @storages[hname].get(vn, key, d)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:delete__prev], key, data) if data
+        end
+        
         res = @storages[hname].delete(vn, key, d)
         @stats.delete_count += 1
 
@@ -311,12 +323,20 @@ module Roma
           return
         end
 
+        if @stats.wb_command_map.key?(:set_expt__prev)
+@log.debug(":set_export__prev")
+          # [vn, t, clk, expt, val]
+          data = @storages[hname].get_raw(vn, key, d)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt__prev], key, data[3].to_s) if data
+        end
+
         expt = chg_time_expt(s[2].to_i)
         ret = @storages[hname].set_expt(vn, key, d, expt)
 
         if ret
-          if @stats.wb_command_map.key?(:set_export)
-            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt], k, expt.to_s)
+          if @stats.wb_command_map.key?(:set_expt)
+@log.debug(":set_export")
+            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt], key, expt.to_s)
           end
           redundant(nodes[1..-1], hname, key, d, ret[2], ret[3], ret[4])
           send_data("STORED\r\n")
@@ -342,12 +362,18 @@ module Roma
           return
         end
 
+        if @stats.wb_command_map.key?(:set_expt__prev)
+          # [vn, t, clk, expt, val]
+          data = @storages[hname].get_raw(vn, key, d)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt__prev], key, data[3].to_s) if data
+        end
+
         expt = chg_time_expt(s[2].to_i)
         ret = @storages[hname].set_expt(vn, key, d, expt)
 
         if ret
-          if @stats.wb_command_map.key?(:set_export)
-            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt], k, expt.to_s)
+          if @stats.wb_command_map.key?(:set_expt)
+            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt], key, expt.to_s)
           end
           redundant(nodes[1..-1], hname, key, d, ret[2], ret[3], ret[4])
           send_data("STORED\r\n")
@@ -431,6 +457,12 @@ module Roma
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
           return
         end
+
+        if @stats.wb_command_map.key?("#{fnc.to_s}__prev".to_sym)
+          data = @storages[hname].get(vn, k, d)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map["#{fnc.to_s}__prev".to_sym], k, data) if data
+        end
+
         ret = @storages[hname].send(fnc, vn, k, d, expt ,v)
         @stats.write_count += 1
 
@@ -453,6 +485,11 @@ module Roma
           return
         end
 
+        if @stats.wb_command_map.key?(:cas__prev)
+          data = @storages[hname].get(vn, k, d)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:cas__prev], k, data) if data
+        end
+
         ret = @storages[hname].cas(vn, k, d, clk, expt ,v)
         @stats.write_count += 1
 
@@ -507,8 +544,8 @@ module Roma
 
         bytes = s[4].to_i
         if bytes < 0
-          @log.error("set:wrong key size(#{s})")
-          return send_data("CLIENT_ERROR Wrong key size.\r\n")
+          @log.error("set:wrong value size(#{s})")
+          return send_data("CLIENT_ERROR Wrong value size.\r\n")
         end
 
         key,hname = s[1].split("\e")
@@ -553,6 +590,12 @@ module Roma
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
           return
         end
+
+        if @stats.wb_command_map.key?("#{fnc.to_s}__prev".to_sym)
+          data = @storages[hname].get(vn, k, d)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map["#{fnc.to_s}__prev".to_sym], k, data) if data
+        end
+
         res = @storages[hname].send(fnc, vn, k, d, v)
         @stats.write_count += 1
 

data/ruby/server/lib/roma/romad.rb

@@ -13,7 +13,7 @@ require 'roma/routing/cb_rttable'
 require 'timeout'
 
 module Roma
-  
+
   class Romad
     include AsyncProcess
     include WriteBehindProcess
@@ -49,7 +49,7 @@ module Roma
       @storages.each{|hashname,st|
         st.opendb
       }
-      
+
       start_async_process
       start_wb_process
       timer
@@ -84,7 +84,7 @@ module Roma
           Event::Handler::connections.clear
 
           EventMachine::run do
-            EventMachine.start_server('0.0.0.0', @stats.port, 
+            EventMachine.start_server('0.0.0.0', @stats.port,
                                       Roma::Command::Receiver,
                                       @storages, @rttable)
             # a management of connections lives
@@ -109,7 +109,7 @@ module Roma
                     end
                   end
                 }
-                dellist.each{|k| 
+                dellist.each{|k|
                   @log.info("delete connection lastcmd = #{k.lastcmd}")
                   Event::Handler::connections.delete(k)
                 }
@@ -194,7 +194,7 @@ module Roma
 
     def initialize_wb_writer
       @wb_writer = Roma::WriteBehind::FileWriter.new(
-                                                     Roma::Config::WRITEBEHIND_PATH, 
+                                                     Roma::Config::WRITEBEHIND_PATH,
                                                      Roma::Config::WRITEBEHIND_SHIFT_SIZE,
                                                      @log)
     end
@@ -223,7 +223,7 @@ module Roma
         Event::Handler.class_eval{
           alias gets2 gets
           undef gets
-          
+
           def gets
             ret = gets2
             @log.info("command log:#{ret.chomp}") if ret
@@ -231,7 +231,7 @@ module Roma
           end
         }
       end
-      
+
       if @stats.join_ap
         Command::Receiver::mk_evlist
       else
@@ -280,7 +280,7 @@ module Roma
       opts.on_tail("-v", "--version", "Show version") {
         puts "romad.rb #{Roma::VERSION}"; exit
       }
-      
+
       opts.on("-n", "--name [name]") { |v| @stats.name = v }
 
       @stats.enabled_repetition_host_in_routing = false
@@ -311,7 +311,7 @@ module Roma
       unless @stats.port =~ /^\d+$/
         raise OptionParser::ParseError.new('Port number is not numeric.')
       end
-      
+
       @stats.join_ap.sub!(':','_') if @stats.join_ap
       if @stats.join_ap && !(@stats.join_ap =~ /^.+_\d+$/)
         raise OptionParser::ParseError.new('[address:port] can not parse.')
@@ -327,7 +327,7 @@ module Roma
       if Config.const_defined? :STORAGE_PATH
         path = "#{Roma::Config::STORAGE_PATH}/#{@stats.ap_str}"
       end
-      
+
       if Config.const_defined? :STORAGE_CLASS
         st_class = Config::STORAGE_CLASS
       end
@@ -379,11 +379,11 @@ module Roma
           @rttable = Roma::Routing::ChurnbasedRoutingTable.new(rd,fname)
         end
       end
-      
+
       if Roma::Config.const_defined?(:RTTABLE_SUB_NID)
         @rttable.sub_nid = Roma::Config::RTTABLE_SUB_NID
       end
-      
+
       if Roma::Config.const_defined?(:ROUTING_FAIL_CNT_THRESHOLD)
         @rttable.fail_cnt_threshold = Roma::Config::ROUTING_FAIL_CNT_THRESHOLD
       end
@@ -413,6 +413,11 @@ module Roma
           @log.error("AUTO_RECOVER is off or Unavailable value is set to [DEFAULT_LOST_ACTION] => #{@rttable.lost_action}")
         end
       }
+
+      if Roma::Config.const_defined?(:ROUTING_EVENT_LIMIT_LINE)
+        @rttable.event_limit_line = Roma::Config::ROUTING_EVENT_LIMIT_LINE
+      end
+      Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_get_routing_event'))
     end
 
     def initialize_rttable_join
@@ -490,7 +495,7 @@ module Roma
       Messaging::ConPool.instance.return_connection(nid,con)
       rcv
     rescue Exception
-      nil      
+      nil
     end
 
     def timer
@@ -631,11 +636,11 @@ module Roma
         @stats.run_sync_routing = false
       }
       t[:name] = 'sync_routing'
-    end    
+    end
 
     def routing_hash_comparison(nid,id='0')
       return :skip if @stats.run_join || @stats.run_recover || @stats.run_balance
-      
+
       h = async_send_cmd(nid,"mklhash #{id}\r\n")
       if h && h.start_with?("ERROR") == false && @rttable.mtree.get(id) != h
         if (id.length - 1) == @rttable.div_bits
@@ -652,12 +657,12 @@ module Roma
     def sync_routing(nid,id)
       vn = @rttable.mtree.to_vn(id)
       @log.warn("vn=#{vn} inconsistent")
-      
+
       res = async_send_cmd(nid,"getroute #{vn}\r\n")
       return if res == nil || res.start_with?("ERROR")
       clk,*nids = res.split(' ')
       clk = @rttable.set_route(vn, clk.to_i, nids)
-      
+
       if clk.is_a?(Integer) == false
         clk,nids = @rttable.search_nodes_with_clk(vn)
         cmd = "setroute #{vn} #{clk-1}"
@@ -715,7 +720,7 @@ module Roma
       @log.error("#{e}\n#{$@}")
       nil
     end
-    
+
     def stop
       @storages.each_value{|st|
         st.closedb

data/ruby/server/lib/roma/routing/cb_rttable.rb

@@ -19,6 +19,9 @@ module Roma
       attr_accessor :auto_recover
       attr_accessor :auto_recover_status
       attr_accessor :auto_recover_time
+      attr_accessor :event
+      attr_accessor :event_limit_line
+      attr_accessor :logs
       attr_reader :version_of_nodes
       attr_reader :min_version
 
@@ -34,6 +37,9 @@ module Roma
         @auto_recover=false
         @auto_recover_status="waiting"
         @auto_recover_time=1800
+        @event = []
+        @event_limit_line = 1000
+        @logs = []
         @enabled_failover=false
         @lock = Mutex.new
         @version_of_nodes = Hash.new(0)
@@ -47,11 +53,13 @@ module Roma
         ret['routing.auto_recover'] = @auto_recover.to_s
         ret['routing.auto_recover_status'] = @auto_recover_status.to_s
         ret['routing.auto_recover_time'] = @auto_recover_time
+        ret['routing.event'] = @event
+        ret['routing.event_limit_line'] = @event_limit_line
         ret['routing.version_of_nodes'] = @version_of_nodes.inspect
         ret['routing.min_version'] = @min_version
         ret
       end
-      
+
       def set_version(nid,ver)
         @version_of_nodes[nid] = ver
         @min_version = find_min_version
@@ -64,11 +72,11 @@ module Roma
       end
 
       def set_leave_proc(&block)
-        @leave_proc=block        
+        @leave_proc=block
       end
 
       def set_lost_proc(&block)
-        @lost_proc=block        
+        @lost_proc=block
       end
 
       def set_recover_proc(&block)
@@ -86,13 +94,13 @@ module Roma
             @log_name="#{@fname}.#{log_list.last[0]+1}"
           end
         end
-        @log_fd=File.open(@log_name,"a")      
+        @log_fd=File.open(@log_name,"a")
       end
 
       def write_log_setroute(vn, clk, nids)
         log="setroute #{vn} #{clk}"
         nids.each{ |nid| log << " #{nid}" }
-        write_log(log)        
+        write_log(log)
       end
 
       def write_log(line)
@@ -119,7 +127,7 @@ module Roma
         buf = self.nodes
         buf.delete(ap_str)
         hosts = []
-        
+
         unless rep_host
           buf.each{ |node|
             host = node.split(/[:_]/)[0]
@@ -128,7 +136,7 @@ module Roma
         else
           hosts = buf
         end
-        
+
         hosts.length < @rd.rn
       end
 
@@ -168,7 +176,7 @@ module Roma
           end
         }
         idx = short_idx if short_idx.length > 0
-          
+
         ks = idx.keys
         return nil if ks.length == 0
         vn = ks[rand(ks.length)]
@@ -205,6 +213,7 @@ module Roma
           @rd.nodes << nid
           @rd.nodes.sort!
           write_log("join #{nid}")
+          set_event(nid, 'join')
         end
       end
 
@@ -227,10 +236,11 @@ module Roma
         @rd.nodes.delete(nid)
         @version_of_nodes.delete(nid)
         @min_version = find_min_version
-        
+
         @log.warn("#{nid} just failed.")
         write_log("leave #{nid}")
-        
+        set_event(nid, __method__)
+
         lost_vnodes=[]
         short_vnodes=[]
         @lock.synchronize {
@@ -261,6 +271,14 @@ module Roma
         @fail_cnt.delete(nid)
       end
 
+      def set_event(nid, process)
+        t = Time.now
+        tstr = "#{t.strftime('%Y-%m-%dT%H:%M:%S')}.#{t.usec}"
+        @event.shift if @event.size >= @event_limit_line
+        @event << ("#{tstr} #{process} #{nid}")
+      end
+      private :set_event
+
       def set_route_and_inc_clk_inside_sync(vn, nodes)
         @rd.v_idx[vn] = nodes
         clk = @rd.v_clk[vn] + 1