roma-client 0.3.7

data/lib/roma/client/routing/routing_data.rb

@@ -0,0 +1,241 @@
+require 'yaml'
+
+module Roma
+  module Routing
+
+    class RoutingData
+      attr_accessor :dgst_bits
+      attr_accessor :div_bits
+      attr_accessor :rn
+      attr_accessor :nodes
+      attr_accessor :v_idx
+      attr_accessor :v_clk
+
+      def initialize(dgst_bits,div_bits,rn)
+        @dgst_bits=dgst_bits
+        @div_bits=div_bits
+        @rn=rn
+        @nodes=[]
+        @v_idx={}
+        @v_clk={}
+      end
+
+      def save(fname)
+        @nodes.sort!
+        open(fname,'wb'){|io|
+          io.write(YAML.dump(self))
+        }
+      end
+
+      def self.load(fname)
+        rd=load_snapshot(fname)
+        rd.load_log_all(fname)
+        rd
+      end
+
+      def self.load_snapshot(fname)
+        rd=nil
+        open(fname,'rb'){|io|
+          rd = YAML.load(io.read)
+        }
+        rd
+      end
+
+      def self.snapshot(fname)
+        rd=load_snapshot(fname)
+        loglist=rd.get_file_list(fname)
+        if loglist.length<2
+          return false
+        end
+        loglist.delete(loglist.last)
+        loglist.each{|i,f|
+          rd.load_log_one(f)
+          File.rename(f,"#{f}~")
+        }
+        File.rename(fname,"#{fname}~")
+        rd.save(fname)
+        true
+      end
+
+      def each_log_all(fname)
+        loglist=get_file_list(fname)
+        loglist.each{|i,f|
+          each_log_one(f){|t,l| yield t,l}
+        }
+      end
+
+      def each_log_one(fname)
+        File.open(fname,"r"){|f|
+          while((line=f.gets)!=nil)
+            line.chomp!
+            next if line[0]=="#" || line.length==0
+            if line =~ /(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.\d+\s(.+)/
+              yield Time.mktime($1, $2, $3, $4, $5, $6), $7 
+            end
+          end
+        }
+      end
+
+      def load_log_all(fname)
+        each_log_all(fname){|t,line|
+          parse_log(t,line)
+        }
+        @nodes.sort!
+      end
+
+      def load_log_one(fname)
+        each_log_one(fname){|t,line|
+          parse_log(t,line)
+        }
+        @nodes.sort!
+      end
+
+      def parse_log(t,line)
+        s=line.split(' ')
+        case s[0]
+        when 'setroute'
+          # setroute <vnode-id> <clock> <node-id> ...
+          nids=[]
+          s[3..-1].each{ |nid| nids << nid }
+          @v_idx[s[1].to_i]=nids
+          @v_clk[s[1].to_i]=s[2].to_i
+        when 'join'
+          # join <node-id>
+          @nodes << s[1] unless @nodes.include?(s[1])
+        when 'leave'
+          # leave <node-id>
+          @nodes.delete(s[1])
+        else
+          raise "RoutingData.parse_log:parse error #{line}"
+        end
+      end
+
+      def search_mask
+        2**@div_bits-1<<(@dgst_bits-@div_bits)
+      end
+
+      def next_vnode(vn)
+        n = (vn >> (@dgst_bits-@div_bits)) + 1
+        n = 0 if n == (2**@div_bits)
+        n << (@dgst_bits-@div_bits)
+      end
+
+      def create_nodes_from_v_idx
+        buf_nodes={}
+        v_idx.each_value{|nids|
+          nids.each{|nid| buf_nodes[nid]=nid }
+        }
+        @nodes=buf_nodes.values.sort
+      end
+
+      # Returns the losted vnode-id list.
+      def get_lost_vnodes
+        ret=[]
+        v_idx.each_pair{|vn,nids|
+          ret << vn if nids.length == 0
+        }
+        ret
+      end
+
+      def self.create(dgst_bits,div_bits,rn,nodes,repethost=false)
+        ret=RoutingData.new(dgst_bits,div_bits,rn)
+        ret.nodes=nodes.clone
+
+        rnlm=RandomNodeListMaker.new(nodes,repethost)
+
+        (2**div_bits).times{|i|
+          vn=i<<(dgst_bits-div_bits)
+          ret.v_clk[vn]=0
+          ret.v_idx[vn]=rnlm.list(rn)
+        }
+        ret
+      end
+
+      # Returns the log file list by old ordered.
+      # +fname+:: Prefix of a log file.(ex.roma0_3300.route)
+      # One of the following example:
+      #   [[1, "roma0_3300.route.1"], [2, "roma0_3300.route.2"]]
+      def get_file_list(fname)
+        l={}
+        files=Dir.glob("#{fname}*")
+        files.each{ |file|
+          if /#{fname}\.(\d+)$/=~file
+            l[$1.to_i]=$&
+          end
+        }
+        # sorted by old order
+        l.to_a.sort{|a,b| a[0]<=>b[0]}
+      end
+
+      def get_histgram
+        ret = {}
+        nodes.each{|nid|
+          ret[nid] = Array.new(rn,0)
+        }
+        v_idx.each_pair{|vn,nids|
+          nids.each_with_index{|nid,i|
+            ret[nid][i] += 1
+          }
+        }
+        ret
+      end
+
+      private
+
+      class RandomNodeListMaker
+        def initialize(nodes,repethost)
+          @repethost=repethost
+          @nodes=nodes
+          @host_idx={}
+          nodes.each{|nid|
+            h,p=nid.split('_')
+            if @host_idx.key?(h)
+              @host_idx[h] << nid
+            else
+              @host_idx[h]=[nid]
+            end
+          }          
+        end
+
+        # Returns the random node-list without repetition.
+        # +n+:: list length
+        def list(n)
+          ret=[]
+          hosts=[]
+          proc_other_one = :get_other_one
+          proc_other_one = :get_other_one_repethost if @repethost
+          n.times{
+            nid=nil
+            nid=send(proc_other_one,hosts,ret)
+            break unless nid
+            hosts << nid.split('_')[0]
+            ret << nid
+          }
+          ret
+        end
+        
+        # +exp_hosts+:: ignore
+        # +exp_nodes+:: exceptional nodes(ex.['roma0_11211'])
+        def get_other_one_repethost(exp_hosts,exp_nodes)
+          buf=@nodes.clone
+          buf.delete_if{|nid| exp_nodes.include?(nid)}
+          buf[rand(buf.length)]
+        end
+
+        # +exp_hosts+:: exceptional hosts(ex.['roma0','roma1'])
+        # +exp_nodes+:: ignore
+        def get_other_one(exp_hosts,exp_nodes)
+          hidx=@host_idx.clone
+          exp_hosts.each{|h| hidx.delete(h) }
+          return nil if hidx.length == 0
+          
+          rh=hidx.keys[rand(hidx.keys.length)]
+          nodes=hidx[rh]
+          nodes[rand(nodes.length)]
+        end
+      end # class RandomNodeListMaker
+
+    end # class RoutingData
+
+  end # module Routing
+end # module Roma

data/lib/roma/client/sender.rb

@@ -0,0 +1,155 @@
+require 'timeout'
+require 'yaml'
+require 'roma/client/con_pool'
+
+module Roma
+  module Client
+
+    class Sender
+
+      def initialize
+      end
+
+      def send_route_mklhash_command(node_id)
+        timeout(1){
+          conn = ConPool.instance.get_connection(node_id)
+          conn.write "mklhash 0\r\n"
+          ret = conn.gets
+          ConPool.instance.return_connection(node_id, conn)
+          return ret.chomp if ret
+        }
+      rescue =>e
+        STDERR.puts "#{node_id} #{e.inspect}"
+        return nil
+      end
+
+      def send_routedump_command(node_id)
+        timeout(1){
+          buf = RUBY_VERSION.split('.')
+          if buf[0].to_i == 1 && buf[1].to_i == 8
+            return send_routedump_yaml_command(node_id)
+          end
+          conn = ConPool.instance.get_connection(node_id)
+          conn.write "routingdump\r\n"
+          routes_length = conn.gets.to_i
+          if (routes_length <= 0)
+            conn.close
+            return :error if routes_length < 0
+            return nil
+          end
+          
+          routes = ''
+          while (routes.length != routes_length)
+            routes = routes + conn.read(routes_length - routes.length)
+          end
+          conn.read(2) # "\r\n"
+          conn.gets
+          rd = Marshal.load(routes)
+          ConPool.instance.return_connection(node_id, conn)
+          return rd
+        }
+      rescue =>e
+        STDERR.puts "#{node_id} #{e.inspect}"
+        nil
+      end
+
+      def send_routedump_yaml_command(node_id)
+        conn = ConPool.instance.get_connection(node_id)
+        conn.write "routingdump yaml\r\n"
+        
+        yaml = ''
+        while( (line = conn.gets) != "END\r\n" )
+          yaml << line
+        end
+
+        rd = YAML.load(yaml)
+        ConPool.instance.return_connection(node_id, conn)
+        return rd        
+      end
+
+      def send_stats_command
+        # TODO
+      end
+
+      def send_version_command(ap)
+        conn = ConPool.instance.get_connection(ap)
+        conn.write("version\r\n")
+        res = conn.gets.chomp
+        ConPool.instance.return_connection(ap, conn)
+        raise unless res
+        return res
+      end
+
+      def send_verbosity_command(ap)
+        conn = ConPool.instance.get_connection(ap)
+        # TODO
+        ConPool.instance.return_connection(ap, conn)
+      end
+
+      def send_command(nid, cmd, value = nil, receiver = :oneline_receiver)
+        con = ConPool.instance.get_connection(nid)
+        raise unless con
+        if value
+          con.write "#{cmd}\r\n#{value}\r\n"
+        else
+          con.write "#{cmd}\r\n"
+        end
+        ret = send(receiver, con)
+        ConPool.instance.return_connection(nid, con)
+        if ret && ret.instance_of?(String) &&
+            (ret =~ /^SERVER_ERROR/ || ret =~ /^CLIENT_ERROR/)
+          raise ret
+        end
+        ret
+      end
+
+      private
+
+      def oneline_receiver(con)
+        ret = con.gets
+        raise "connection closed" if ret.nil?
+        ret.chomp 
+      end
+
+      def value_list_receiver(con)
+        ret = []
+        while (line = con.gets) != "END\r\n"
+          s = line.split(' ')
+          return line.chomp if s[0] == 'SERVER_ERROR' || s[0] == 'CLIENT_ERROR'
+          ret << read_bytes(con, s[3].to_i)
+          read_bytes(con, 2)
+        end
+        ret
+      end
+
+      def value_hash_receiver(con)
+        ret = {}
+        while (line = con.gets) != "END\r\n"
+          s = line.split(' ')
+          return line.chomp if s[0] == 'SERVER_ERROR' || s[0] == 'CLIENT_ERROR'
+          ret[s[1]] = read_bytes(con, s[3].to_i)
+          read_bytes(con, 2)
+        end
+        ret
+      end      
+
+      def multiplelines_receiver(con) 
+        ret = []
+        while (line = con.gets) != "END\r\n"
+          ret << line.chomp
+        end
+        ret
+      end
+
+      def read_bytes(con, len)
+        ret = ''
+        until (ret.length == len)
+          ret = ret + con.read(len - ret.length)
+        end
+        ret
+      end
+
+    end
+    
+  end
+end

data/lib/roma/client/tools/showbalance.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'roma/client/sender'
+require 'roma/client/routing/routing_data'
+
+if ARGV.length < 1
+  STDERR.puts "usage:#{$0} node-id"
+  exit
+end
+
+sender = Roma::Client::Sender.new
+rd = sender.send_routedump_command(ARGV[0])
+hist = rd.get_histgram
+
+# max length of node-id string
+len_nid = 0
+rd.nodes.each{|nid|
+  len_nid = nid.length if nid.length > len_nid
+}
+
+csum = Array.new(rd.rn, 0)
+rd.nodes.each{|nid|
+  puts sprintf("%-#{len_nid}s %s %d",
+               nid,
+               hist[nid].inspect,
+               hist[nid].inject{|r,n| r + n})
+  hist[nid].each_with_index{|n,i|
+    csum[i] += n
+  }
+}
+
+puts sprintf("%#{len_nid}s %s","sum",csum.inspect)
+

data/lib/roma/client/version.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# = roma/client/version.rb
+# This file is derived from roma client.
+#
+module Roma #:nodoc:
+    module Client #:nodoc:
+      # == What Is This Library?
+      # ROMA クライアントバージョン情報モジュール
+      #
+      module VERSION
+        # メジャーバージョン
+        MAJOR = 0
+
+        # マイナバージョン
+        MINOR = 3
+
+        # TINY version
+        TINY  = 7
+
+        # バージョン文字列
+        STRING = [MAJOR, MINOR, TINY].join('.')
+      end
+    end
+end