roma 0.8.2

data/lib/roma/storage/sqlite3_storage.rb

@@ -0,0 +1,73 @@
+require 'sqlite3'
+require 'roma/storage/basic_storage'
+
+module Roma
+  module Storage
+
+    module SQLite3_Ext
+      def put(k,v)
+        if self.execute("select count(*) from t_roma where key=?",k)[0][0].to_i==0
+          self.execute("insert into t_roma values (?,?)",k,SQLite3::Blob.new(v))
+        else
+          self.execute("update t_roma set val=? where key=?",SQLite3::Blob.new(v),k)
+        end
+      end
+
+      def get(k)
+        r = self.execute("select * from t_roma where key=?",k)
+        return nil if r.length==0
+        r[0][1]
+      end
+
+      def out(k)
+        return nil if get(k) == nil
+        self.execute("delete from t_roma where key=?",k)
+      end
+
+      def rnum
+        self.execute("select count(*) from t_roma")[0][0].to_i
+      end
+
+      def each
+        self.execute("select * from t_roma"){ |r|
+          yield r[0],r[1]
+        }
+      end
+
+      def create_table
+        sql = "create table t_roma ( " + 
+          "key TEXT PRIMARY KEY," +
+          "val BLOB);"
+        self.execute( sql )
+      end
+
+      def tables
+        sql = "SELECT name FROM " +
+          "sqlite_master WHERE type='table' UNION ALL SELECT name FROM sqlite_temp_master " +
+          "WHERE type='table' ORDER BY name;"
+        self.execute( sql ).flatten
+      end
+    end
+
+    class SQLite3Storage < BasicStorage
+
+      def initialize
+        super
+        @ext_name = 'sql3'
+      end
+
+      private
+
+      def open_db(fname)
+        hdb = SQLite3::Database.new(fname)
+        hdb.extend(Roma::Storage::SQLite3_Ext)
+        hdb.create_table if hdb.tables.length == 0
+        hdb
+      end
+
+      def close_db(hdb); hdb.close; end
+
+    end # class SQLite3Storage
+
+  end # module Storage
+end # module Roma

data/lib/roma/storage/tc_storage.rb

@@ -0,0 +1,133 @@
+require 'tokyocabinet'
+require 'roma/storage/basic_storage'
+
+module Roma
+  module Storage
+
+    class TCStorage < BasicStorage
+      include TokyoCabinet
+
+      def initialize
+        super
+        @ext_name = 'tc'
+      end
+
+      def get_stat
+        ret = super
+        @hdb.each_with_index{|hdb,idx|
+          ret["storage[#{idx}].path"] = File.expand_path(hdb.path)
+          ret["storage[#{idx}].rnum"] = hdb.rnum
+          ret["storage[#{idx}].fsiz"] = hdb.fsiz
+        }
+        ret
+      end
+
+      protected
+
+      def set_options(hdb)
+        prop = parse_options
+
+        prop.each_key{|k|
+          unless /^(bnum|apow|fpow|opts|xmsiz|rcnum|dfunit)$/ =~ k
+            raise RuntimeError.new("Syntax error, unexpected option #{k}")
+          end
+        }
+        
+        opts = 0
+        if prop.key?('opts')
+          opts |= HDB::TLARGE if prop['opts'].include?('l')
+          opts |= HDB::TDEFLATE if prop['opts'].include?('d')
+          opts |= HDB::TBZIP if prop['opts'].include?('b')
+          opts |= HDB::TTCBS if prop['opts'].include?('t')
+        end
+
+        hdb.tune(prop['bnum'].to_i,prop['apow'].to_i,prop['fpow'].to_i,opts)
+
+        hdb.setxmsiz(prop['xmsiz'].to_i) if prop.key?('xmsiz')
+        hdb.setcache(prop['rcnum'].to_i) if prop.key?('rcnum')
+        hdb.setdfunit(prop['dfunit'].to_i) if prop.key?('dfunit')
+      end
+
+      private
+
+      def parse_options
+        return Hash.new(-1) unless @option
+        buf = @option.split('#')
+        prop = Hash.new(-1)
+        buf.each{|equ|
+          if /(\S+)\s*=\s*(\S+)/ =~ equ
+            prop[$1] = $2
+          else
+            raise RuntimeError.new("Option string parse error.")
+          end
+        }
+        prop
+      end
+
+      def open_db(fname)
+        hdb = HDB::new
+
+        set_options(hdb)
+
+        if !hdb.open(fname, HDB::OWRITER | HDB::OCREAT | HDB::ONOLCK)
+          ecode = hdb.ecode
+          raise RuntimeError.new("tcdb open error #{hdb.errmsg(ecode)}")
+        end
+        hdb
+      end
+
+      def close_db(hdb)
+        if !hdb.close
+          ecode = hdb.ecode
+          raise RuntimeError.new("tcdb close error #{hdb.errmsg(ecode)}")
+        end        
+      end
+
+    end # class TCStorage
+
+    class TCAsyncStorage < TCStorage
+      private
+
+      def open_db(fname)
+        hdb = HDB::new
+
+        set_options(hdb)
+
+        hdb.instance_eval{
+          alias put putasync
+        }
+
+        if !hdb.open(fname, HDB::OWRITER | HDB::OCREAT)
+          ecode = hdb.ecode
+          raise RuntimeError.new("tcdb open error #{hdb.errmsg(ecode)}")
+        end
+
+        Thread.new {
+          loop{
+            sleep 10
+            hdb.sync
+          }
+        }
+
+        hdb
+      end
+    end # class TCAsyncStorage
+
+
+    class TCMemStorage < BasicStorage
+      include TokyoCabinet
+
+      private
+
+      def open_db(fname)
+        hdb = ADB::new
+        hdb.open("*")
+        hdb
+      end
+
+      def close_db(hdb); end
+
+    end # class TCMemStorage
+
+  end # module Storage
+end # module Roma

data/lib/roma/tools/mkrecent.rb

@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# usage:mkrecent dgst-bits div-bits divnum storage-path1 storage-path2 recent-storage-path
+#
+require 'roma/routing/routing_data'
+
+module Roma
+  module Storage
+  end
+  Storage::autoload(:TCStorage,'roma/storage/tc_storage')
+  Storage::autoload(:DbmStorage,'roma/storage/dbm_storage')
+  Storage::autoload(:SQLite3Storage,'roma/storage/sqlite3_storage')
+
+  class MakeRecentData
+    
+    def initialize(argv = nil)
+      if argv.length != 6
+        STDERR.puts "usage:mkrecent dgst-bits div-bits divnum storage-path1 storage-path2 recent-storage-path"
+        exit
+      end
+
+      dgst_bits = argv[0].to_i
+      div_bits = argv[1].to_i
+      @divnum = argv[2].to_i
+      @strgpath1 = argv[3]
+      @strgpath2 = argv[4]
+      @recentpath = argv[5]
+
+      @vnodes = []
+      (2**div_bits).times{|i|
+        @vnodes << ( i<<(dgst_bits-div_bits) )
+      }
+    end
+
+    def suite
+      if File::directory?(@recentpath)
+        STDERR.puts "#{@recentpath} exists."
+        exit
+      end
+      
+      Dir::mkdir(@recentpath)
+
+      Dir::glob("#{@strgpath1}/*").each{|dir|
+        next unless File::directory?(dir)
+        hname = dir[dir.rindex('/')+1..-1]
+        open_storage(dir,
+                     "#{@strgpath2}/#{hname}",
+                     "#{@recentpath}/#{hname}")
+        exec(hname)
+
+        close_storage
+      }
+    end
+
+    def open_storage(p1,p2,rp)
+      puts "Open #{p1}"
+      @st1 = ropen(p1)
+      @st1.each_vn_dump_sleep = 0
+      exit unless @st1
+      puts "Open #{p2}"
+      @st2 = ropen(p2)
+      @st2.each_vn_dump_sleep = 0
+      unless @st2
+        STDERR.puts ""
+        @st1.closedb
+        exit
+      end
+ 
+      if @st1.class != @st2.class
+        STDERR.puts "#{p1} and #{p2} that file type is different."
+        @st1.closedb
+        @st2.closedb
+        exit
+      end
+
+      puts "Open #{rp}"
+      @rst = @st1.class.new
+      @rst.divnum = @divnum
+      @rst.vn_list = @vnodes
+      @rst.storage_path = rp
+      @rst.opendb
+    end
+
+    def close_storage
+      @st1.closedb
+      @st2.closedb
+      @rst.closedb
+    end
+
+    def exec(hname)
+      n = 0
+      @vnodes.each{|vn|
+        printf "#{hname}:#{n}/#{@vnodes.length}\r"
+        n+=1
+        buf = @st1.dump(vn)
+        @rst.load( buf ) if buf
+        buf = @st2.dump(vn)
+        @rst.load( buf ) if buf
+      }
+    end
+
+    private
+
+    def ropen(path)
+      unless File::directory?(path)
+        STDERR.puts "#{path} dose not found."
+        return nil
+      end
+
+      ext = File::extname(Dir::glob("#{path}/0.*")[0])[1..-1]
+
+      storage = new_storage(ext)
+      storage.divnum = @divnum
+      storage.vn_list = @vnodes
+      storage.storage_path = path
+      storage.opendb
+      storage
+    end
+
+    def new_storage(ext)
+      case(ext)
+      when 'tc'
+        return ::Roma::Storage::TCStorage.new
+      when 'dbm'
+        return Roma::Storage::DbmStorage.new
+      when 'sql3'
+        return Roma::Storage::SQLite3Storage.new
+      else
+        return nil
+      end
+    end
+
+  end # MakeRecentData
+end # module Roma
+
+Roma::MakeRecentData.new(ARGV).suite
+puts "Make recent data process has succeed."

data/lib/roma/tools/mkroute.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'optparse'
+require 'roma/routing/routing_data'
+
+# ダイジェストのビット数
+dgst_bits=32
+# 分割ビット数 (dgst_bits >= div_bits)
+div_bits=9
+# 冗長度 (nodes.length >= rn)
+rn=2
+# ホスト名の重複許可
+repeathost=false
+
+opts = OptionParser.new
+opts.banner = "usage:#{File.basename($0)} [options] node-id..."
+opts.on("-h","--hash [bits]","(default=32)"){|v| dgst_bits = v.to_i }
+opts.on("-d","--divide [bits]","(default=9)"){|v| div_bits = v.to_i }
+opts.on("-r","--redundant [num]","(default=2)"){|v| rn = v.to_i }
+opts.on(nil,"--enabled_repeathost"){|v| repeathost=true }
+opts.parse!(ARGV)
+
+nodes = ARGV
+nodes.map!{|n| n.sub(':','_')}
+
+if nodes.length == 0
+  STDERR.puts opts.help
+  exit!
+end
+
+if dgst_bits < div_bits
+  STDERR.puts "The hash bits should be divide bits or more."
+  exit!  
+end
+
+if div_bits > 32
+  STDERR.puts "The upper bound of divide bits is 32."
+  exit!    
+end
+
+if nodes.length < rn
+  STDERR.puts "The node-id number should be redundant number or more."
+  exit!
+end
+
+rt = Roma::Routing::RoutingData::create(dgst_bits,div_bits,rn,nodes,repeathost)
+
+nodes.each{|nid|
+  rt.save("#{nid}.route")
+}
+puts "nodes => #{nodes}"
+puts "Routing table has created."

data/lib/roma/tools/recoverlost.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# usage:recoverlost address port storage-path [yyyymmddhhmmss]
+#
+require 'roma/tools/recoverlost_lib'
+
+Roma::RecoverLost.new('recoverlost', 'spushv', ARGV).suite
+puts "Recover process has succeed."

data/lib/roma/tools/recoverlost_alist.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# usage:recoverlost_alist address port storage-path [yyyymmddhhmmss]
+#
+require 'roma/tools/recoverlost_lib'
+
+Roma::RecoverLost.new('recoverlost_alist', 'alist_spushv', ARGV).suite
+puts "Recover process has succeed."

data/lib/roma/tools/recoverlost_lib.rb

@@ -0,0 +1,217 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# usage:recoverlost address port storage-path [yyyymmddhhmmss]
+#
+require 'roma/client/sender'
+require 'roma/messaging/con_pool'
+require 'roma/routing/routing_data'
+
+module Roma
+  module Storage
+  end
+  Storage::autoload(:TCStorage,'roma/storage/tc_storage')
+  Storage::autoload(:DbmStorage,'roma/storage/dbm_storage')
+  Storage::autoload(:SQLite3Storage,'roma/storage/sqlite3_storage')
+
+  class RecoverLost
+    
+    def initialize(pname, pushv_cmd, argv)
+      if argv.length < 3
+        puts "usage:#{pname} address port storage-path [yyyymmddhhmmss]"
+        exit
+      end
+
+      @addr = argv[0]
+      @port = argv[1]
+      @strgpath = argv[2]
+      @ymdhms = argv[3]
+
+      if @port =~ /\D/
+        STDERR.puts "port was not numeric."
+        exit
+      end
+      
+      if @ymdhms && (@ymdhms.length != 14 || @ymdhms =~ /\D/)
+        STDERR.puts "yyyymmddhhmmss format mismatch."
+        exit
+      end
+      @pushv_cmd = pushv_cmd
+      @nodeid = "#{@addr}_#{@port}"
+      @stream_copy_wait_param = 0.0001
+    end
+
+    def suite
+      @rd = get_routing_data(@nodeid)
+      @lost_vnodes = get_lost_vnodes(@rd,@ymdhms)
+      puts "#{@lost_vnodes.length} vnodes where data was lost."
+
+      exit if @lost_vnodes.length == 0
+
+      each_hash(@strgpath){|hname,dir|
+        puts "#{hname} #{dir}"
+        @storage = open_storage(dir,@lost_vnodes)
+        start_recover(hname)
+        @storage.closedb
+      }
+    end
+
+    def each_hash(path)
+      Dir::glob("#{path}/*").each{|dir|
+        next unless File::directory?(dir)
+        hname = dir[dir.rindex('/')+1..-1]
+        yield hname,dir
+      }      
+    end
+
+    def get_routing_data(nid)
+      sender = Roma::Client::Sender.new
+      sender.send_routedump_command(nid)
+    end
+
+    def get_lost_vnodes(rd,ymdhms)
+      ret = rd.get_lost_vnodes
+      if ymdhms
+        ret |= get_history_of_lost(rd.nodes[0],ymdhms)
+      end
+      ret
+    end
+
+    def get_history_of_lost(nid,ymdhms)
+      ret = []
+      con = Roma::Messaging::ConPool.instance.get_connection(nid)
+      con.write("history_of_lost #{ymdhms}\r\n")
+      while((buf = con.gets) != "END\r\n")
+        ret << buf.chomp.to_i
+      end
+      Roma::Messaging::ConPool.instance.return_connection(nid, con)
+      ret
+    end
+
+    def open_storage(path,vn_list)
+      unless File::directory?(path)
+        STDERR.puts "#{path} dose not found."
+        return nil
+      end
+
+      # get a file extension
+      ext = File::extname(Dir::glob("#{path}/0.*")[0])[1..-1]
+      # count a number of divided files
+      divnum = Dir::glob("#{path}/*.#{ext}").length
+        
+      st = new_storage(ext)
+      st.divnum = divnum
+      st.vn_list = vn_list
+      st.storage_path = path
+      st.opendb
+      st
+    end
+
+    def new_storage(ext)
+      case(ext)
+      when 'tc'
+        return ::Roma::Storage::TCStorage.new
+      when 'dbm'
+        return Roma::Storage::DbmStorage.new
+      when 'sql3'
+        return Roma::Storage::SQLite3Storage.new
+      else
+        return nil
+      end
+    end
+
+    def start_recover(hname)
+      @lost_vnodes.each{|vn|
+        nodes = @rd.v_idx[vn]
+        if nodes == nil || nodes.length == 0
+          nid = @rd.nodes[rand(@rd.nodes.length)]
+          puts "#{vn} assign to #{nid}"
+        else
+          nid = nodes[0]
+          puts "#{vn} was auto assirned at #{nid}"
+        end
+
+        if push_a_vnode_stream(hname, vn, nid)!="STORED"
+          STDERR.puts "push_a_vnode_stream aborted in #{vn}"
+          exit
+        end
+
+        if nodes == nil || nodes.length == 0
+          cmd = "setroute #{vn} #{@rd.v_clk[vn]} #{nid}\r\n"
+          exit unless send_cmd(nid ,cmd)
+          broadcast_cmd(cmd, nid)
+        end
+      }
+    end
+
+    def push_a_vnode(hname, vn, nid)
+      dmp = @storage.dump(vn)
+      return true unless dmp
+      con = Roma::Messaging::ConPool.instance.get_connection(nid) unless con
+      con.write("pushv #{hname} #{vn}\r\n")
+      res = con.gets
+      con.write("#{dmp.length}\r\n#{dmp}\r\nEND\r\n")
+      res = con.gets
+      con.close
+      res.chomp! if res
+      res
+    rescue =>e
+      STDERR.puts "#{e}\n#{$@}"
+      nil
+    end
+
+    def push_a_vnode_stream(hname, vn, nid)
+      con = Roma::Messaging::ConPool.instance.get_connection(nid)
+
+#      con.write("spushv #{hname} #{vn}\r\n")
+      con.write("#{@pushv_cmd} #{hname} #{vn}\r\n")
+
+      res = con.gets # READY\r\n or error string
+      if res != "READY\r\n"
+        con.close
+        return res.chomp
+      end
+
+      @storage.each_vn_dump(vn){|data|
+        con.write(data)
+        sleep @stream_copy_wait_param
+      }
+      con.write("\0"*20) # end of steram
+
+      res = con.gets # STORED\r\n or error string
+      Roma::Messaging::ConPool.instance.return_connection(nid,con)
+      res.chomp! if res
+      res
+    rescue =>e
+      STDERR.puts "#{e}\n#{$@}"
+      nil
+    end
+
+    def broadcast_cmd(cmd,without_nids=nil)
+      without_nids=[] unless without_nids
+      res = {}
+      @rd.nodes.each{ |nid|
+        res[nid] = send_cmd(nid,cmd) unless without_nids.include?(nid)
+      }
+      res
+    rescue => e
+      STDERR.puts("#{e}\n#{$@}")
+      nil
+    end
+
+    def send_cmd(nid, cmd)
+      con = Roma::Messaging::ConPool.instance.get_connection(nid)
+      con.write(cmd)
+      res = con.gets
+      Roma::Messaging::ConPool.instance.return_connection(nid, con)
+      if res
+        res.chomp!
+      end
+      res
+    rescue => e
+      STDERR.puts("#{__FILE__}:#{__LINE__}:Send command failed that node-id is #{nid},command is #{cmd}.")
+      nil
+    end
+
+  end # class RecoverLost
+end # module Roma

data/lib/roma/tools/sample_watcher.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'roma/commons'
+require 'roma/client/sender'
+
+module Roma
+  
+  class Watcher
+
+    def initialize
+      @sender = Roma::Client::Sender.new
+    end
+
+    def get_node_list(nid)
+      @sender.send_command(nid, "nodelist").split(' ')
+    end
+
+  end # class Watcher
+  
+end # module Roma
+
+
+w = Roma::Watcher.new
+
+# 監視ノードの定義
+nodes = ['roma0:11211','roma0:11212','roma0:11213','roma0:11214']
+
+nodes.each{|nid|
+  begin
+    if w.get_node_list(nid).length != nodes.length
+      STDERR.puts "fail over occurred in #{nid}."
+    end
+  rescue
+    STDERR.puts "command error in #{nid}."
+  end
+}
+
+puts "#{Time.now} ROMA watcher has done."

data/lib/roma/tools/sample_watcher2.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'roma/commons'
+require 'roma/client/sender'
+
+module Roma
+  
+  class Watcher
+
+    def initialize
+      @sender = Roma::Client::Sender.new
+    end
+
+    def watch(nid, command)
+      ret = @sender.send_command(nid, command, value = nil, :multiplelines_receiver)
+    end
+  end # class Watcher
+  
+end # module Roma
+
+w = Roma::Watcher.new
+
+# the definition of all nodes 
+nodes = ['roma0:11211','roma0:11212']
+
+nodes.each{|nid|
+  begin
+    ret = w.watch(nid, "stat #{ARGV[0]}")
+    puts "/********** #{nid} **********/"
+    ret.each{|l|
+      puts l
+    }
+  rescue
+    STDERR.puts "command error in #{nid}."
+  end
+}
+
+puts "#{Time.now} ROMA watcher2 has done."