roma 0.8.2

data/lib/roma/routing/cb_rttable.rb

@@ -0,0 +1,326 @@
+require 'roma/routing/rttable'
+
+module Roma
+  module Routing
+
+    class ChurnbasedRoutingTable < RoutingTable
+
+      attr :fname
+      attr :log_fd
+      attr :log_name
+      attr :trans # transaction
+      attr :leave_proc
+      attr :lost_proc
+      attr_accessor :lost_action
+      attr_reader :version_of_nodes
+      attr_reader :min_version
+
+      def initialize(rd,fname)
+        super(rd)
+        @rd.nodes.sort!
+        @trans={}
+        @fname=fname
+        @leave_proc=nil
+        @lost_proc=nil
+        @lost_action=:no_action
+        @enabled_failover=false
+        @lock = Mutex.new
+        @version_of_nodes = Hash.new(0)
+        @min_version = nil
+        open_log
+      end
+
+      def get_stat(ap)
+        ret = super(ap)
+        ret['routing.lost_action'] = @lost_action.to_s
+        ret['routing.version_of_nodes'] = @version_of_nodes.inspect
+        ret['routing.min_version'] = @min_version
+        ret
+      end
+      
+      def set_version(nid,ver)
+        @version_of_nodes[nid] = ver
+        if @min_version == nil || @min_version > ver
+          @min_version = ver
+        end
+      end
+
+      def find_min_version
+        ret = 0xffffff
+        @version_of_nodes.each_value{|ver| ret = ver if ret > ver}
+        ret
+      end
+
+      def set_leave_proc(&block)
+        @leave_proc=block        
+      end
+
+      def set_lost_proc(&block)
+        @lost_proc=block        
+      end
+
+      def open_log
+        log_list=@rd.get_file_list(@fname)
+        if log_list.length==0
+          @log_name="#{@fname}.1"
+        else
+          if File::stat("#{@fname}.#{log_list.last[0]}").size == 0
+            @log_name="#{@fname}.#{log_list.last[0]}"
+          else
+            @log_name="#{@fname}.#{log_list.last[0]+1}"
+          end
+        end
+        @log_fd=File.open(@log_name,"a")      
+      end
+
+      def write_log_setroute(vn, clk, nids)
+        log="setroute #{vn} #{clk}"
+        nids.each{ |nid| log << " #{nid}" }
+        write_log(log)        
+      end
+
+      def write_log(line)
+        # log rotation
+        if File::stat(@log_name).size > 1000 * 1024
+          close_log
+          open_log
+        end
+        t = Time.now
+        tstr = "#{t.strftime('%Y-%m-%dT%H:%M:%S')}.#{t.usec}"
+        @log_fd.write("#{tstr} #{line}\n")
+        @log_fd.flush
+      end
+
+      def close_log
+        @log_fd.close
+      end
+
+      def can_i_recover?
+        @rd.nodes.length >= @rd.rn
+      end
+
+      # Retuens the list of losted-data vnode newer than argument time.
+      def search_lost_vnodes(t)
+        ret = []
+        @rd.each_log_all(@fname){|log_t,line|
+          next if t > log_t
+          s = line.split(/ /)
+          if s[0] == 'setroute' && s.length == 3
+            # vnode has a no pnode. therefor this vnode was lost.
+            ret << s[1].to_i
+          end
+        }
+        ret
+      end
+
+      # select a vnodes where short of redundancy.
+      def select_a_short_vnodes(exclued_nodes)
+        ret = []
+        @rd.v_idx.each_pair{|vn, nids|
+          if nids.length < @rd.rn && list_include?(nids,exclued_nodes) == false
+            ret << [vn,nids]
+          end
+        }
+        ret
+      end
+
+      # vnode sampling without +without_nodes+
+      def sample_vnode(without_nodes)
+        short_idx = {}
+        idx = {}
+        @rd.v_idx.each_pair{|vn, nids|
+          unless list_include?(nids, without_nodes)
+            idx[vn] = nids
+            short_idx[vn] = nids if nids.length < @rd.rn
+          end
+        }
+        idx = short_idx if short_idx.length > 0
+          
+        ks = idx.keys
+        return nil if ks.length == 0
+        vn = ks[rand(ks.length)]
+        nids = idx[vn]
+        [vn, nids]
+      end
+
+      def list_include?(list,nodes)
+        nodes.each{|nid|
+          return true if list.include?(nid)
+        }
+        false
+      end
+      private :list_include?
+
+      def set_route(vn, clk, nids)
+        return "#{vn} is not found." unless @rd.v_idx.key?(vn)
+        @lock.synchronize {
+          return "It's old table." if @rd.v_clk[vn] > clk
+          nids.each{ |nid|
+            add_node(nid) unless @rd.nodes.include?(nid)
+          }
+          @rd.v_idx[vn] = nids.clone
+          clk += 1
+          @rd.v_clk[vn] = clk
+          @mtree.set(vn, nids)
+          write_log_setroute(vn, clk, nids)
+          return clk
+        }
+      end
+
+      def add_node(nid)
+        unless @rd.nodes.include?(nid)
+          @rd.nodes << nid
+          @rd.nodes.sort!
+          write_log("join #{nid}")
+        end
+      end
+
+      def enabled_failover=(b)
+        @enabled_failover=b
+        @fail_cnt.clear
+      end
+
+      def enabled_failover
+        @enabled_failover
+      end
+
+      def leave(nid)
+        unless @enabled_failover
+          return
+        end
+        return unless @rd.nodes.include?(nid)
+
+        @leave_proc.call(nid) if @leave_proc
+        @rd.nodes.delete(nid)
+        @version_of_nodes.delete(nid)
+        @min_version = find_min_version
+        
+        @log.warn("#{nid} just failed.")
+        write_log("leave #{nid}")
+        
+        lost_vnodes=[]
+        @lock.synchronize {
+          @rd.v_idx.each_pair{ |vn, nids|
+            buf = nids.clone
+            if buf.delete(nid)
+              set_route_and_inc_clk_inside_sync(vn, buf)
+              if buf.length == 0
+                lost_vnodes << vn
+                @log.error("Vnode data is lost.(Vnode=#{vn})")
+               end
+            end
+          }
+        }
+        if lost_vnodes.length > 0
+          @lost_proc.call if @lost_proc
+          if @lost_action == :auto_assign
+            lost_vnodes.each{ |vn|
+              set_route_and_inc_clk_inside_sync( vn, next_alive_vnode(vn) )
+            }
+          end
+        end
+        @fail_cnt.delete(nid)
+      end
+
+      def set_route_and_inc_clk_inside_sync(vn, nodes)
+        @rd.v_idx[vn] = nodes
+        clk = @rd.v_clk[vn] + 1
+        @rd.v_clk[vn] = clk
+        @mtree.set(vn, nodes)
+        write_log_setroute(vn, clk, nodes)
+        clk
+      end
+      private :set_route_and_inc_clk_inside_sync
+
+      def next_alive_vnode(vn)
+        svn = vn
+        while( (vn = @rd.next_vnode(vn)) != svn )
+          return @rd.v_idx[vn].clone if @rd.v_idx[vn].length != 0
+        end
+        []
+      end
+      private :next_alive_vnode
+
+      def each_vnode
+        @rd.v_idx.each_pair{ |k, v| yield(k, v) }
+      end
+
+      def v_idx
+        @rd.v_idx.clone
+      end
+
+      def search_nodes_for_write(vn)
+        return @trans[vn][0].clone if @trans.key?(vn)
+        @rd.v_idx[vn].clone
+      rescue
+        nil
+      end
+
+      def search_nodes_with_clk(vn)
+        @lock.synchronize {
+          return [@rd.v_clk[vn], @rd.v_idx[vn].clone]
+        }
+      rescue
+        nil
+      end
+
+      # +vn+: vnode-id
+      # +nids+: node-id list
+      def transaction(vn, nids)
+        return false if @trans.key?(vn)
+        @trans[vn]=[nids.clone, Time.now]
+        true
+      end
+
+      def commit(vn)
+        return false unless @trans.key?(vn)
+        @lock.synchronize {
+          @rd.v_idx[vn]=@trans[vn][0]
+          @trans.delete(vn)
+          clk = @rd.v_clk[vn] + 1
+          @rd.v_clk[vn] = clk
+          @mtree.set(vn, @rd.v_idx[vn])
+          write_log_setroute(vn, clk, @rd.v_idx[vn])
+          return clk
+        }
+      end
+
+      def rollback(vn)
+        @trans.delete(vn)
+      end
+
+      # +sec+: elapsed-time
+      def delete_old_trans(sec=3600)
+        @trans.delete_if{|vn,val| val[1] < Time.now-sec }
+      end
+
+      # Returns the status of vnode balance.
+      # +ap+: my address_port string(ex."roma0_11211")
+      def vnode_balance(ap)
+        # amount of primary at node = amount of vnode / amount of node
+        n = (2**div_bits) / nodes.length
+
+        pcount = scount = 0
+        @rd.v_idx.each_pair{ |vn, nids|
+          next if nids == nil or nids.length == 0
+          if nids[0] == ap
+            pcount += 1
+          elsif nids.include?(ap)
+            scount += 1
+          end
+        }
+
+        @log.debug("#{__FILE__}:#{__LINE__}:n=#{n} pcount=#{pcount} scount=#{scount}")
+
+        if pcount > n*1.1
+          return :over
+        elsif pcount < n*0.9
+          return :less
+        end
+        :even
+      end
+
+    end # class ChurnbasedRoutingTable
+
+  end # module Routing
+end # module Roma

data/lib/roma/routing/merkle_tree.rb

@@ -0,0 +1,54 @@
+require 'digest/sha1'
+
+module Roma
+  module Routing
+
+    class MerkleTree
+      
+      attr :tree
+      attr :div_bits
+      attr :dgst_bits
+
+      def initialize(dgst_bits,div_bits)
+        @tree = {}
+        @dgst_bits = dgst_bits
+        @div_bits = div_bits
+        create_tree_instance('0')
+      end
+
+      def set(vn,nodes)
+        id = '0' + (vn >> (@dgst_bits-@div_bits)).to_s(2).rjust(@div_bits,'0')
+        @tree[id] = Digest::SHA1.hexdigest(nodes.to_s)
+        update(parent(id))
+      end
+
+      def get(id)
+        @tree[id]
+      end
+
+      def to_vn(id)
+        id[1,id.length].to_i(2) << (@dgst_bits-@div_bits)
+      end
+
+      private
+
+      def create_tree_instance(id)
+        @tree[id] = 0
+        return if id.length > @div_bits
+        create_tree_instance("#{id}0")
+        create_tree_instance("#{id}1")
+      end
+
+      def update(id)
+        @tree[id] = Digest::SHA1.hexdigest("#{@tree[id+'0']}:#{@tree[id+'1']}")
+        update(parent(id)) if id.length != 1
+      end
+
+      def parent(id)
+        id.chop
+      end
+
+    end # class MerkleTree
+
+  end # module Routing
+end # module Roma

data/lib/roma/routing/rttable.rb

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+require 'roma/logging/rlogger'
+require 'roma/routing/routing_data'
+require 'roma/routing/merkle_tree'
+require 'yaml'
+require 'json'
+
+module Roma
+  module Routing
+
+    class RoutingTable
+
+      attr :rd
+      attr :search_mask
+      attr :fail_cnt
+      attr :mtree
+      attr_reader :hbits
+      attr_reader :rn
+      attr_reader :div_bits
+      attr_accessor :fail_cnt_threshold
+      attr_accessor :fail_cnt_gap
+
+      def initialize(rd)
+        @log = Roma::Logging::RLogger.instance
+        @rd = rd
+        @rn = @rd.rn
+        @div_bits=@rd.div_bits
+        @hbits = 2**@rd.dgst_bits
+        @search_mask = @rd.search_mask
+        @fail_cnt = Hash.new(0)
+        @fail_cnt_threshold = 5
+        @fail_cnt_gap = 0
+        @fail_time = Time.now
+        init_mtree
+      end
+
+      def get_stat(ap)
+
+        pn = sn = short = lost = 0
+        @rd.v_idx.each_pair{|vn, nids|
+          if nids == nil || nids.length == 0
+            lost += 1
+            next
+          elsif nids[0] == ap
+            pn += 1
+          elsif nids.include?(ap)
+            sn += 1
+          end
+          short += 1 if nids.length < @rd.rn 
+        }
+
+        ret = {}
+        ret['routing.redundant'] = @rn
+        ret['routing.nodes.length'] = nodes.length
+        ret['routing.nodes'] = nodes.inspect
+        ret['routing.dgst_bits'] = @rd.dgst_bits
+        ret['routing.div_bits'] = @div_bits
+        ret['routing.vnodes.length'] = vnodes.length
+        ret['routing.primary'] = pn
+        ret['routing.secondary'] = sn
+        ret['routing.short_vnodes'] = short
+        ret['routing.lost_vnodes'] = lost
+        ret['routing.fail_cnt_threshold'] = @fail_cnt_threshold
+        ret['routing.fail_cnt_gap'] = @fail_cnt_gap
+        ret
+      end
+
+      def init_mtree
+        @mtree = MerkleTree.new(@rd.dgst_bits,@rd.div_bits)
+        @rd.v_idx.each_pair{ |vn, nids|
+          @mtree.set(vn,nids)
+        }
+      end
+
+      def nodes
+        @rd.nodes.clone
+      end
+
+      def vnodes
+        @rd.v_idx.keys
+      end
+
+      # ハッシュ値からvnode idを返す
+      def get_vnode_id(d)
+        d & @search_mask
+      end
+
+      # vnode があるノードIDの配列を返す
+      # +vn+: vnode id
+      def search_nodes(vn)
+        @rd.v_idx[vn].clone
+      rescue
+        nil
+      end
+
+      # 離脱ノードを検索リストから削除する
+      # +nid+: 離脱ノード
+      def leave(nid)
+        @rd.nodes.delete(nid)
+        # リストから nid を消す
+        @rd.v_idx.each_pair{ |vn, nids|
+          nids.delete_if{ |nid2| nid2 == nid}
+          if nids.length == 0
+            @log.error("Vnode data is lost.(Vnode=#{vn})")
+          end
+          @mtree.set(vn,nids)
+        }
+        @fail_cnt.delete(nid)
+      end
+
+      def dump
+        Marshal.dump(@rd)
+      end
+
+      def dump_yaml
+        YAML.dump(@rd)
+      end
+
+      def dump_json
+        JSON.generate(
+                      [{:dgst_bits=>@rd.dgst_bits,:div_bits=>@rd.div_bits,:rn=>@rd.rn},
+                       @rd.nodes,@rd.v_idx])
+      end
+
+      def proc_failed(nid)
+        t = Time.now
+        if t - @fail_time > @fail_cnt_gap
+          @fail_cnt[nid] += 1
+          if @fail_cnt[nid] >= @fail_cnt_threshold
+            leave(nid)
+          end
+        end
+        @fail_time = t
+      end
+
+      def proc_succeed(nid)
+        @fail_cnt.delete(nid)
+      end
+
+      # v_idx から nodes を再構築する
+      def create_nodes_from_v_idx
+        @rd.create_nodes_from_v_idx
+      end
+
+    end # class RoutingTable
+
+  end # module Routing
+end # module Roma

data/lib/roma/stats.rb

@@ -0,0 +1,112 @@
+require 'singleton'
+require 'roma/config'
+
+module Roma
+
+  class Stats
+    include Singleton
+
+    # command options
+    attr_accessor :address, :port
+    attr_accessor :daemon
+    attr_accessor :join_ap
+    attr_accessor :start_with_failover
+    attr_accessor :name
+    attr_accessor :verbose
+    attr_accessor :enabled_repetition_host_in_routing
+
+    # proc mode
+    attr_accessor :enabled_vnodes_balance
+
+    # proc status
+    attr_accessor :run_acquire_vnodes
+    attr_accessor :run_recover
+    attr_accessor :run_sync_routing
+    attr_accessor :run_iterate_storage
+    attr_accessor :run_storage_clean_up
+    attr_accessor :run_receive_a_vnode
+    attr_accessor :run_release
+
+    # proc param
+    attr_accessor :stream_copy_wait_param
+
+    # compressed redundant param
+    attr_accessor :size_of_zredundant
+
+    # performance counter
+    attr_accessor :write_count
+    attr_accessor :read_count
+    attr_accessor :delete_count
+    attr_accessor :out_count
+    attr_accessor :out_message_count
+    attr_accessor :redundant_count
+
+    def initialize
+      @run_acquire_vnodes = false
+      @run_recover = false
+      @run_sync_routing = false
+      @run_iterate_storage = false
+      @run_storage_clean_up = false
+      @run_receive_a_vnode = false
+      @run_release = false
+      @stream_copy_wait_param = 
+        Roma::Config::DATACOPY_STREAM_COPY_WAIT_PARAM
+      @enabled_vnodes_balance = nil
+      @write_count = 0
+      @read_count = 0
+      @delete_count = 0
+      @out_count = 0
+      @out_message_count = 0
+      @redundant_count = 0
+      @size_of_zredundant = 0
+    end
+
+    def ap_str
+      "#{@address}_#{port}"
+    end
+
+    def get_stat
+      ret = {}
+      ret['stats.address'] = @address
+      ret['stats.port'] = @port
+      ret['stats.daemon'] = @daemon
+      ret['stats.name'] = @name
+      ret['stats.verbose'] = @verbose
+      ret['stats.enabled_repetition_host_in_routing'] = @enabled_repetition_host_in_routing
+      ret['stats.run_acquire_vnodes'] = @run_acquire_vnodes
+      ret['stats.run_recover'] = @run_recover
+      ret['stats.run_sync_routing'] = @run_sync_routing
+      ret['stats.run_iterate_storage'] = @run_iterate_storage
+      ret['stats.run_storage_clean_up'] = @run_storage_clean_up
+      ret['stats.run_release'] = @run_release
+      ret['stats.stream_copy_wait_param'] = @stream_copy_wait_param
+      ret['stats.size_of_zredundant'] = @size_of_zredundant
+      ret['stats.write_count'] = @write_count
+      ret['stats.read_count'] = @read_count
+      ret['stats.delete_count'] = @delete_count
+      ret['stats.out_count'] = @out_count
+      ret['stats.out_message_count'] = @out_message_count
+      ret['stats.redundant_count'] = @redundant_count
+      ret
+    end
+
+    def clear_counters
+      clear_count(:@write_count)
+      clear_count(:@read_count)
+      clear_count(:@delete_count)
+      clear_count(:@out_count)
+      clear_count(:@out_message_count)
+      clear_count(:@redundant_count)
+    end
+
+    private
+
+    def clear_count(var)
+      if self.instance_variable_get(var) > 0xffffffff
+        self.instance_variable_set(var,0)
+      end
+    end
+
+  end # class Stats
+
+end # module Roma