roma 0.8.14 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eaf529b1a3ec8c250e906ce4b360cc694455159d
-  data.tar.gz: f566d3a0c29055ac384683ec5c7859723b9fc091
+  metadata.gz: 463ee73649102c0de8e1b6a7fa8e42f5130f2332
+  data.tar.gz: d8f360a9690da7729b3b939486ac9c12a3eb32bf
 SHA512:
-  metadata.gz: 0d259866eb0fc71effe35310966a706bc2517959ed8d37fe793bc3bbf77ccb55b24815df765c1f90945ce397f4e8f63df8be28342290c97fe69aa9190db63af4
-  data.tar.gz: e7ac00f6f5a1db6e4ebfdc17e1170ae431d0a15211ec92c3b978fd1003d33b479822f0dcb0e0aff38e949f2aab8ca1e4155d4c75f074b38934d62a7908dbca1f
+  metadata.gz: ec235f905b6bf987aaa157b2c360dfe0b8adf683597cff10fb1149a27aff25a1e9d976a842b7c91907d78e4a2b0dafa281f5af97ce21d408571bbbad99911178
+  data.tar.gz: 6e9eda8b8885efc140708c2adc549b633263ed29e9bac65ccd5492f3d002c3bf0a8cc3556fc72220bca1ccbeda3111c0dc1c67365136a910a974b51530486742

data/CHANGELOG

@@ -1,3 +1,22 @@
+*1.0.0 (Sep 30 2014)*
+
+* Add new function to get vnodes information of each data [hiroaki-iwase] ad25c6e
+* Add new function to get key list without stopping ROMA [hiroaki-iwase] f36e907
+* Add new GUI control system(Gladiator) [hiroaki-iwase] 66dfd58
+* Add new function to get log data via ROMA console [hiroaki-iwase] b1995e4
+* change some snapshot and storage logic [hiroaki-iwase] 257b0df
+* Add new function to check routing consistecy [hiroaki-iwase] dcd1ace
+* Add new function of getting routing event [hiroaki-iwase] e297215
+* fix : no data copy in the join process when 2 hosts, 2 processes [junji torii] 10eae74
+* Merge branch 'master' of git@github.com:roma/roma into fix_join [junji torii] 41e0269
+* refacort:error case for the release command. [junji torii] 0144b63
+* change release logic [hiroaki-iwase] f950fa1
+* bugfix:end condition of recover process in no_action [junji torii] 9b41a3c
+* {func}__prev support for DSL [junji torii] d772c56
+* Add new function to get first instance information [hiroaki-iwase] e4b465f
+* {func}__prev support in write-behind [junji torii] 96af508
+* each_vn_dump supported in normal status [junji torii] 546ee7d
+
 *0.8.14 (Mar 26 2014)*
 
 * add timeout parameter for a vnode copy [junji torii] 0522750

data/ruby/server/lib/roma/async_process.rb

@@ -305,7 +305,7 @@ module Roma
       t = Thread::new do
         begin
           timeout(@rttable.auto_recover_time){
-            loop{ 
+            loop{
               sleep 1
               break if @rttable.auto_recover_status != "preparing"
               #break if @stats.run_join #run_join don't have possibility to be true in this case.
@@ -331,7 +331,7 @@ module Roma
             when :no_action
               @log.debug("auto recover NOT start. Because lost action is [no_action]")
           end
-        end      
+        end
       end
       t[:name] = __method__
     end
@@ -366,6 +366,7 @@ module Roma
       @do_acquired_recover_process = true
       loop do
         break unless @do_acquired_recover_process
+        break if @rttable.num_of_vn(@stats.ap_str)[2] == 0 # short vnodes
 
         vn, nodes, is_primary = @rttable.select_vn_for_recover(exclude_nodes)
         break unless vn
@@ -487,12 +488,18 @@ module Roma
 
       @do_release_process = true
       while(@rttable.has_node?(@stats.ap_str)) do
+        break unless @do_release_process
         @rttable.each_vnode do |vn, nids|
           break unless @do_release_process
           if nids.include?(@stats.ap_str)
 
             to_nid, new_nids = @rttable.select_node_for_release(@stats.ap_str, @stats.rep_host, nids)
-            unless sync_a_vnode_for_release(vn, to_nid, new_nids)
+            res = sync_a_vnode_for_release(vn, to_nid, new_nids)
+            if res == :abort
+              @log.error("#{__method__}:release_process aborted due to SERVER_ERROR received.")
+              @do_release_process = false
+            end
+            if res == false
               @log.warn("#{__method__}:error at vn=#{vn} to_nid=#{to_nid} new_nid=#{new_nids}")
               redo
             end
@@ -510,7 +517,7 @@ module Roma
     def sync_a_vnode_for_release(vn, to_nid, new_nids)
       nids = @rttable.search_nodes(vn)
 
-      if nids.include?(to_nid)==false || (is_primary && nids[0]!=to_nid)
+      if nids.include?(to_nid)==false
         @log.debug("#{__method__}:#{vn} #{to_nid}")
         # change routing data at the vnode and synchronize a data
         nids << to_nid
@@ -523,6 +530,7 @@ module Roma
           if res != "STORED"
             @rttable.rollback(vn)
             @log.error("#{__method__}:push_a_vnode was failed:hname=#{hname} vn=#{vn}:#{res}")
+            return :abort if res.start_with?("SERVER_ERROR")
             return false
           end
         }
@@ -607,14 +615,31 @@ module Roma
       if @rttable.rn == 1
         return [new_nid]
       end
-      if nodes.length > @rttable.rn
-        nodes.delete(new_nid)
-        nodes.delete(nodes.last)
-        nodes << new_nid
+      # [node_a, node_b, new_nid]
+      nodes.delete(new_nid)
+      # [node_a, node_b]
+
+      if nodes.length >= @rttable.rn
+        host = new_nid.split(/[:_]/)[0]
+        buf = [] # list of a same host
+        nodes.each do |nid|
+          buf << nid if nid.split(/[:_]/)[0] == host
+        end
+        if buf.length > 0
+          # include same host
+          # delete a last one, due to save a primary node
+          nodes.delete(buf.last)
+        else
+          nodes.delete(nodes.last)
+        end
       end
+
       if is_primary
-        nodes.delete(new_nid)
+        # [new_nid, node_a]
         nodes.insert(0,new_nid)
+      else
+        # [node_a, new_nid]
+        nodes << new_nid
       end
       nodes
     end
@@ -636,7 +661,7 @@ module Roma
         return res.chomp
       end
 
-      @storages[hname].each_vn_dump(vn){|data|
+      res_dump = @storages[hname].each_vn_dump(vn) do |data|
 
         unless @do_push_a_vnode_stream
           con.close
@@ -646,12 +671,16 @@ module Roma
 
         con.write(data)
         sleep @stats.stream_copy_wait_param
-      }
+      end
       con.write("\0"*20) # end of steram
 
       res = con.gets # STORED\r\n or error string
       Roma::Messaging::ConPool.instance.return_connection(nid,con)
       res.chomp! if res
+      if res_dump == false
+        @log.error("#{__method__}:each_vn_dump in hname=#{hname} vn=#{vn} nid=#{nid}")
+        return "CANCELED"
+      end
       res
     rescue =>e
       @log.error("#{e}\n#{$@}")
@@ -773,7 +802,7 @@ module Roma
     def asyncev_start_storage_flush_process(args)
       hname, dn = args
       @log.debug("#{__method__} #{args.inspect}")
-      
+
       st = @storages[hname]
       if st.dbs[dn] != :safecopy_flushing
         @log.error("Can not flush storage. stat = #{st.dbs[dn]}")
@@ -796,7 +825,7 @@ module Roma
     def asyncev_start_storage_cachecleaning_process(args)
       hname, dn = args
       @log.debug("#{__method__} #{args.inspect}")
-      
+
       st = @storages[hname]
       if st.dbs[dn] != :cachecleaning
         @log.error("Can not start cachecleaning process. stat = #{st.dbs[dn]}")
@@ -818,16 +847,16 @@ module Roma
       count = 0
       rcount = 0
       st = @storages[hname]
-      
+
       @do_storage_cachecleaning_process = true
       loop do
         # get keys in a cache up to 100 kyes
         keys = st.get_keys_in_cache(dn)
         break if keys == nil || keys.length == 0
         break unless @do_storage_cachecleaning_process
-        
+
         # @log.debug("#{__method__}:#{keys.length} keys found")
-        
+
         # copy cache -> db
         st.each_cache_by_keys(dn, keys) do |vn, last, clk, expt, k, v|
           break unless @do_storage_cachecleaning_process
@@ -854,6 +883,91 @@ module Roma
       @do_storage_cachecleaning_process = false
     end
 
+    def asyncev_start_get_routing_event(args)
+      @log.debug("#{__method__} #{args}")
+      t = Thread::new do
+        begin
+          get_routing_event
+        rescue => e
+          @log.error("#{__method__}:#{e.inspect} #{$@}")
+        ensure
+        end
+      end
+      t[:name] = __method__
+    end
+
+    def get_routing_event
+      @log.info("#{__method__}:start.")
+
+      routing_path = Config::RTTABLE_PATH
+      f_list = Dir.glob("#{routing_path}/#{@stats.ap_str}*")
+
+      f_list.each{|fname|
+        IO.foreach(fname){|line|
+          if line =~ /join|leave/
+            @rttable.event.shift if @rttable.event.size >= @rttable.event_limit_line
+            @rttable.event << line.chomp 
+          end
+        }
+      }
+
+      @log.info("#{__method__} has done.")
+    rescue =>e
+      @log.error("#{e}\n#{$@}")
+    end
+
+    def asyncev_start_get_logs(args)
+      @log.debug("#{__method__} #{args}")
+      t = Thread::new do
+        begin
+          get_logs(args)
+        rescue => e
+          @log.error("#{__method__}:#{e.inspect} #{$@}")
+        ensure
+          @stats.gui_run_gather_logs = false
+        end
+      end
+      t[:name] = __method__
+    end
+
+    def get_logs(args)
+      @log.debug("#{__method__}:start.")
+
+      log_path =  Config::LOG_PATH
+      log_file = "#{log_path}/#{@stats.ap_str}.log"
+
+      raw_logs = []
+      start_time = Time.now
+      File.open(log_file){|f|
+        f.each_line{|line|
+          # hilatency check
+          ps = Time.now - start_time
+          if ps > 5
+            @log.warn("gather_logs process was failed.")
+            raise
+          end
+
+          raw_logs << line unless line.chomp == '.'
+        }
+      }
+
+      sliced_logs = []
+      if raw_logs.size > args[0]
+        sliced_logs = raw_logs.slice(-args[0]..-1)
+      else
+        raw_logs.shift # remove first line(date of log file was created)
+        sliced_logs = raw_logs
+      end
+
+      @rttable.logs = sliced_logs
+      @log.info("#{__method__} has done.")
+    rescue =>e
+      @rttable.logs = []
+      @log.error("#{e}\n#{$@}")
+    ensure
+      @stats.gui_run_gather_logs = false
+    end
+
   end # module AsyncProcess
 
 end # module Roma

data/ruby/server/lib/roma/command/command_definition.rb

@@ -119,6 +119,11 @@ module Roma
               stored.vn, stored.last, stored.clk, stored.expt, stored.value =
                 @storages[params.hash_name].get_raw(params.vn, params.key, params.digest)
               stored = nil if stored.vn == nil || Time.now.to_i > stored.expt
+              if stored && @stats.wb_command_map.key?("#{cmd}__prev".to_sym)
+                Roma::WriteBehindProcess::push(params.hash_name,
+                                               @stats.wb_command_map["#{cmd}__prev".to_sym],
+                                               params.key, stored.value)
+              end
               ctx = CommandContext.new(s, params, stored)
 
               ret = instance_exec(ctx, &block)
@@ -137,7 +142,9 @@ module Roma
 
                 if ret
                   if @stats.wb_command_map.key?(cmd.to_sym)
-                    Roma::WriteBehindProcess::push(ctx.params.hash_name, @stats.wb_command_map[cmd.to_sym], ctx.params.key, ret[4])
+                    Roma::WriteBehindProcess::push(ctx.params.hash_name,
+                                                   @stats.wb_command_map[cmd.to_sym],
+                                                   ctx.params.key, ret[4])
                   end
                   redundant(ctx.params.nodes[1..-1], ctx.params.hash_name,
                             ctx.params.key, ctx.params.digest, ret[2],
@@ -223,8 +230,15 @@ module Roma
               stored.vn, stored.last, stored.clk, stored.expt, stored.value =
                 @storages[params.hash_name].get_raw(params.vn, params.key, params.digest)
               stored = nil if stored.vn == nil || Time.now.to_i > stored.expt
-              ctx = CommandContext.new(s, params, stored)
 
+              if stored && @stats.wb_command_map.key?("#{cmd}__prev".to_sym)
+                Roma::WriteBehindProcess::push(params.hash_name,
+                                               @stats.wb_command_map["#{cmd}__prev".to_sym],
+                                               params.key, stored.value)
+              end
+
+              ctx = CommandContext.new(s, params, stored)
+              
               ret = instance_exec(ctx, &block)
               if ret.instance_of? Array
                 flg, expt, value, count, msg = ret
@@ -241,7 +255,9 @@ module Roma
 
                 if ret
                   if @stats.wb_command_map.key?(cmd.to_sym)
-                    Roma::WriteBehindProcess::push(ctx.params.hash_name, @stats.wb_command_map[cmd.to_sym], ctx.params.key, ctx.params.value)
+                    Roma::WriteBehindProcess::push(ctx.params.hash_name,
+                                                   @stats.wb_command_map[cmd.to_sym],
+                                                   ctx.params.key, ret[4])
                   end
                   redundant(ctx.params.nodes[1..-1], ctx.params.hash_name, 
                             ctx.params.key, ctx.params.digest, ret[2], 

data/ruby/server/lib/roma/command/rt_command_receiver.rb

@@ -60,6 +60,19 @@ module Roma
         end
       end
 
+      def ev_enabled_repetition_in_routing?(s)
+        rt = @rttable
+        rd = @rttable.sub_nid_rd(@addr)
+        rt = Roma::Routing::RoutingTable.new(rd) if rd
+
+        if s.length == 1
+          repetition = rt.check_repetition_in_routing
+          send_data("#{repetition}\r\n")
+        else
+          send_data("CLIENT_ERROR\r\n")
+        end
+      end
+
       # setroute <vnode-id> <clock> <node-id> ...
       def ev_setroute(s)
         if s.length < 4
@@ -275,6 +288,21 @@ module Roma
         end
       end
 
+      # get_key_info <key>
+      def ev_get_key_info(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments(0 for 1)\r\n")
+        end
+
+        d = Digest::SHA1.hexdigest(s[1]).hex % @rttable.hbits
+        vn = @rttable.get_vnode_id(d)
+        nodes = @rttable.search_nodes_for_write(vn)
+        send_data(sprintf("d = %s 0x%x\r\n",d,d))
+        send_data(sprintf("vn = %s 0x%x\r\n",vn,vn))
+        send_data("nodes = #{nodes.inspect}\r\n")
+        send_data("END\r\n")
+      end
+
     end # module RoutingCommandReceiver
   end # module Command
 end # module Roma

data/ruby/server/lib/roma/command/sys_command_receiver.rb

@@ -1052,7 +1052,7 @@ module Roma
             return send_data("CLIENT_ERROR storage[#{dn}] != :normal status\r\n")
           end
           if st.set_db_stat(dn, :safecopy_flushing) == false
-            return send_data("CLIENT_ERROR storage[#{dn}] != :normal status\r\n")
+            return send_data("CLIENT_ERROR storage[#{dn}] status can't changed\r\n")
           end
           Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_storage_flush_process',[hname, dn]))
         elsif s[2] ==  'normal'
@@ -1060,7 +1060,7 @@ module Roma
             return send_data("CLIENT_ERROR storage[#{dn}] != :safecopy_flushed status\r\n")
           end
           if st.set_db_stat(dn, :cachecleaning) == false
-            return send_data("CLIENT_ERROR storage[#{dn}] != :safecopy_flushed status\r\n")
+            return send_data("CLIENT_ERROR storage[#{dn}] status can't changed\r\n")
           end
           Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_storage_cachecleaning_process',[hname, dn]))
         else
@@ -1070,6 +1070,68 @@ module Roma
         send_data("PUSHED\r\n")
       end
 
+      # set_gui_run_snapshot [true|false]
+      def ev_set_gui_run_snapshot(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+
+        case s[1]
+        when 'true'
+          @stats.gui_run_snapshot = true
+          send_data("STORED\r\n")
+        when 'false'
+          @stats.gui_run_snapshot = false
+          send_data("STORED\r\n")
+        else
+          return send_data("CLIENT_ERROR value must be true or false\r\n")
+        end
+      end
+
+      # set_gui_last_snapshot_date [%Y/%m/%d %H:%M:%S]
+      def ev_set_gui_last_snapshot(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1] !~ /^\d+\/\d+\/\d+T\d+:\d+:\d+$/
+          return send_data("CLIENT_ERROR format is [%Y/%m/%dT%H:%M:%S]\r\n")
+        end
+        res = broadcast_cmd("rset_gui_last_snapshot #{s[1]}\r\n")
+        @stats.gui_last_snapshot = s[1]
+        res[@stats.ap_str] = "PUSHED"
+        send_data("#{res}\r\n")
+      end
+
+      # rset_gui_last_snapshot(s)
+      def ev_rset_gui_last_snapshot(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1] !~ /^\d+\/\d+\/\d+T\d+:\d+:\d+$/
+          return send_data("CLIENT_ERROR format is [%Y/%m/%dT%H:%M:%S]\r\n")
+        end
+        @stats.gui_last_snapshot = s[1]
+        send_data("PUSHED\r\n")
+      end
+
+      # set_cleanup_regexp <regexp>
+      def ev_set_cleanup_regexp(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments #{s.length-1} to 1\r\n")
+        end
+
+        # failover check
+        unless @rttable.enabled_failover
+          return send_data("CLIENT_ERROR failover disable now!!\r\n")
+        end
+
+        @storages.each{|hname,st|
+          st.cleanup_regexp = s[1]
+          st.stop_clean_up
+          send_data("STORED\r\n")
+        }
+      end
+
       private 
 
       def dcnice(p)