flare-tools 0.1.4 → 0.4.5.1

data/lib/flare/tools/cli/remove.rb

@@ -0,0 +1,119 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/index_server'
+require 'flare/util/conversion'
+require 'flare/tools/common'
+require 'flare/tools/cli/sub_command'
+
+# 
+module Flare
+  module Tools
+    module Cli
+
+      # == Description
+      # 
+      class Remove < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Tools::Common
+
+        myname :remove
+        desc   "remove a node. (experimental)"
+        usage  "remove"
+  
+        def setup(opt)
+          opt.on('--force',          "commit changes without confirmation")                  {@force = true}
+          opt.on('--wait=SECOND',    "specify the time to wait node for getting ready (default:#{@wait})") {|v| @wait = v.to_i}
+          opt.on('--retry=COUNT',    "retry count(default:#{@retry})")                        {|v| @retry = v.to_i}
+          opt.on('--connection-threshold=[COUNT]', "specify connection threashold (default:#{@connection_threshold})") {|v| @connection_threshold = v.to_i}
+        end
+
+        def initialize
+          super
+          @force = false
+          @wait = 30
+          @retry = 5
+          @connection_threshold = 2
+        end
+
+        def execute(config, *args)
+          hosts = args.map {|x| x.split(':')}
+          hosts.each do |x|
+            if x.size != 2
+              error "invalid argument '#{x.join(':')}'. it must be hostname:port."
+              return S_NG
+            end
+          end
+          
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            cluster = fetch_cluster(s)
+
+            hosts.each do |hostname,port|
+              nodekey = nodekey_of hostname, port
+              unless cluster.has_nodekey? nodekey
+                error "unknown node name: #{nodekey}"
+                return S_NG
+              end
+            end
+
+            hosts.each do |hostname,port|
+              exec = false
+              Flare::Tools::Node.open(hostname, port, config[:timeout]) do |n|
+                nwait = @wait
+                node = n.stats
+                cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
+                while nwait > 0
+                  conn = node['curr_connections'].to_i
+                  cluster = fetch_cluster(s)
+                  role = cluster.node_stat("#{hostname}:#{port}")['role']
+                  info "waiting until #{hostname}:#{port} (role=#{role}, connections=#{conn}) is inactive..."
+                  if conn <= @connection_threshold && role == 'proxy'
+                    exec = true
+                    break
+                  end
+                  interruptible {sleep 1}
+                  nwait -= 1
+                  node = n.stats
+                end
+                unless @force
+                  node_stat = cluster.node_stat("#{hostname}:#{port}")
+                  role = node_stat['role']
+                  state = node_stat['state']
+                  STDERR.print "please shutdown the daemon and continue (node=#{hostname}:#{port}, role=#{role}, state=#{state}) (y/n): "
+                  interruptible {
+                    exec = false if gets.chomp.upcase != "Y"
+                  }
+                end
+              end
+              
+              if exec
+                suc = false
+                nretry = @retry
+                while nretry > 0
+                  resp = false
+                  info "removing #{hostname}:#{port}."
+                  resp = s.node_remove(hostname, port) unless config[:dry_run]
+                  if resp
+                    suc = true
+                    break
+                  end
+                  nretry -= 1
+                end
+                info "done." if suc
+                info "failed." unless suc
+              else
+                info "skipped."
+              end
+            end
+            puts string_of_nodelist(s.stats_nodes)
+          end
+          
+          S_OK
+        end
+        
+      end
+    end
+  end
+end

data/lib/flare/tools/cli/restore.rb

@@ -0,0 +1,180 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/node'
+require 'flare/tools/index_server'
+require 'flare/tools/common'
+require 'flare/util/conversion'
+require 'flare/util/constant'
+require 'flare/tools/cli/sub_command'
+require 'csv'
+
+begin
+  require 'tokyocabinet'
+rescue LoadError => e
+end
+
+
+module Flare
+  module Tools
+    module Cli
+      
+      class Restore < SubCommand
+
+        class Restorer
+          attr_reader :name
+          def iterate &block
+            raise "internal error"
+          end
+          def close
+            raise "internal error"
+          end
+        end
+
+        class TchRestorer < Restorer
+          # uint32_t flag -> L    // uint32_t
+          # time_t   expire -> Q  // unsigned long
+          # uint64_t size -> Q    // uint64_t
+          # uint64_t version -> Q // uint64_t
+          # uint32_t option -> L  // uint32_t
+          def self.myname
+            "tch"
+          end
+          def initialize filepath
+            raise "output file not specified." if filepath.nil?
+            raise "#{filepath} isn't a path." unless filepath.kind_of?(String)
+            @hdb = TokyoCabinet::HDB.new
+            @hdb.open(filepath, TokyoCabinet::HDB::OCREAT|TokyoCabinet::HDB::OREADER)
+          end
+          def iterate &block
+            @hdb.iterinit
+            while (key = @hdb.iternext)
+              value = @hdb.get(key)
+              a = value.unpack("LQQQC*")
+              flag, expire, size, version = a.shift(4)
+              data = a.pack("C*")
+              block.call(key, data, flag, expire)
+            end
+          end
+          def close
+            @hdb.close
+          end
+        end
+
+        Restorers = []
+        Restorers << TchRestorer if defined? TokyoCabinet
+        Formats = Restorers.map {|n| n.myname}
+
+        include Flare::Util::Conversion
+        include Flare::Util::Constant
+        include Flare::Tools::Common
+        
+        myname :restore
+        desc   "restore data to nodes. (experimental)"
+        usage  "restore [hostname:port]"
+        
+        def setup(opt)
+          opt.on('-i', '--input=FILE',             "input from file") {|v| @input = v}
+          opt.on('-f', '--format=FORMAT',          "input format [#{Formats.join(',')}]") {|v|
+            @format = v
+          }
+          opt.on('--bwlimit=BANDWIDTH',            "bandwidth limit (bps)") {|v| @bwlimit = v}
+          opt.on('--include=PATTERN',              "include pattern") {|v|
+            begin
+              @include = Regexp.new(v)
+            rescue RegexpError => e
+              raise "#{v} isn't a valid regular expression."
+            end
+          }
+          opt.on('--prefix-include=STRING',        "prefix string") {|v|
+            @prefix_include = Regexp.new("^"+Regexp.escape(v))
+          }          
+          opt.on('--exclude=PATTERN',              "exclude pattern") {|v|
+            begin
+              @exclude = Regexp.new(v)
+            rescue RegexpError => e
+              raise "#{v} isn't a valid regular expression."
+            end
+          }
+          opt.on('--print-keys',                     "enables key dump to console") {@print_key = true}
+        end
+
+        def initialize
+          super
+          @input = nil
+          @format = nil
+          @wait = 0
+          @part = 0
+          @partsize = 1
+          @bwlimit = 0
+          @include = nil
+          @prefix_include = nil
+          @exclude = nil
+          @print_key = false
+        end
+
+        def execute(config, *args)
+          STDERR.puts "please install tokyocabinet via gem command." unless defined? TokyoCabinet
+
+          dry_run = config[:dry_run]
+
+          unless @format.nil? || Formats.include?(@format)
+            STDERR.puts "unknown format: #{@format}"
+            return S_NG
+          end
+          
+          if @prefix_include
+            if @include
+              STDERR.puts "--include option is specified."
+              return S_NG
+            end
+            @include = @prefix_include
+          end
+
+          hosts = args.map {|x| x.split(':')}
+          hosts.each do |x|
+            if x.size != 2
+              STDERR.puts "invalid argument '#{x.join(':')}'."
+              return S_NG
+            end
+          end
+          
+          restorer = case @format
+                     when TchRestorer.myname
+                       TchRestorer.new(@input)
+                     else
+                       raise "invalid format"
+                     end
+
+          nodes = hosts.map do |hostname,port|
+            Flare::Tools::Node.open(hostname, port.to_i, config[:timeout], @bwlimit, @bwlimit)
+          end
+
+          count = 0
+          restorer.iterate do |key,data,flag,expire|
+            if @include.nil? || @include =~ key
+              next if @exclude && @exclude =~ key
+              STDOUT.puts key if @print_key
+              nodes[0].set(key, data, flag, expire) unless dry_run
+              count += 1
+            end
+          end
+          STDERR.puts "#{count} entries have been restored."
+
+          nodes.each do |n|
+            n.close
+          end
+
+          restorer.close
+          
+          S_OK
+        end # execute()
+
+      end
+    end
+  end
+end
+
+

data/lib/flare/tools/cli/slave.rb

@@ -0,0 +1,125 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/stats'
+require 'flare/tools/node'
+require 'flare/tools/index_server'
+require 'flare/tools/cluster'
+require 'flare/tools/common'
+require 'flare/util/conversion'
+require 'flare/util/constant'
+require 'flare/tools/cli/sub_command'
+
+module Flare
+  module Tools
+    module Cli
+      class Slave < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::Constant
+        include Flare::Tools::Common
+
+        DefaultRetry = 10
+        
+        myname :slave
+        desc   "construct slaves from proxy nodes."
+        usage  "slave [hostname:port:balance:partition] ..."
+
+        def setup(opt)
+          opt.on('--force',          "commit changes without confirmation") {@force = true}
+          opt.on('--retry=COUNT',    "specify retry count(default:#{@retry})") {|v| @retry = v.to_i}
+          opt.on('--clean',          "clear datastore before construction") {@clean = true}
+        end
+
+        def initialize
+          super
+          @force = false
+          @retry = DefaultRetry
+          @clean = false
+        end
+
+        def execute(config, *args)
+          return S_NG if args.empty?
+
+          hosts = args.map do |arg|
+            hostname, port, balance, partition, rest = arg.split(':', 5)
+            if rest != nil || balance.nil?
+              error "invalid argument '#{arg}'. it must be hostname:port:balance:partition."
+              return S_NG
+            end
+            [hostname, port, balance.to_i, partition.to_i]
+          end
+          
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            cluster = fetch_cluster(s)
+          
+            hosts.each do |hostname,port,balance,partition|
+              nodekey = nodekey_of hostname, port
+
+              unless cluster.has_nodekey? nodekey
+                error "invalid 'hostname:port' pair: #{nodekey}"
+                return S_NG
+              end
+
+              node = cluster.node_stat(nodekey)
+
+              if node['role'] != 'proxy'
+                puts "#{nodekey} is not a proxy."
+                next
+              end
+              
+              exec = @force
+              unless exec
+                STDERR.print "making node slave (node=#{nodekey}, role=#{node['role']} -> slave) (y/n): "
+                interruptible do
+                  exec = true if gets.chomp.upcase == "Y"
+                end
+              end
+              if exec && !config[:dry_run]
+                if @clean
+                  Flare::Tools::Node.open(hostname, port, config[:timeout]) do |n|
+                    n.flush_all
+                  end
+                end
+
+                nretry = 0
+                resp = false
+                while resp == false && nretry < @retry
+                  resp = s.set_role(hostname, port, 'slave', 0, partition)
+                  if resp
+                    puts "started constructing slave node..."
+                  else
+                    nretry += 1
+                    puts "waiting #{nretry} sec..."
+                    sleep nretry
+                    puts "retrying..."
+                  end
+                end
+                if resp
+                  wait_for_slave_construction(s, nodekey, config[:timeout])
+                  if balance > 0
+                    unless @force
+                      STDERR.print "changing node's balance (node=#{nodekey}, balance=0 -> #{balance}) (y/n): "
+                      exec = interruptible {(gets.chomp.upcase == "Y")}
+                    end
+                    if exec
+                      s.set_role(hostname, port, 'slave', balance, partition)
+                    end
+                  end
+                else
+                  error "failed to change the state."
+                  return S_NG
+                end
+              end
+            end
+            STDOUT.puts string_of_nodelist(s.stats_nodes, hosts.map {|x| x[0..1].join(':')})
+          end
+          
+          return S_OK
+        end # execute()
+
+      end
+    end
+  end
+end

data/lib/flare/tools/cli/stats.rb

@@ -1,130 +1,237 @@
 # -*- coding: utf-8; -*-
-# Author::    kgws  (http://d.hatena.ne.jp/kgws/)
-# Copyright:: Copyright (c) 2010- kgws.
-# License::   This program is licenced under the same licence as kgws.
-#
-# $--- flare-stats - [ by Ruby ] $
-# vim: foldmethod=marker tabstop=2 shiftwidth=2
-require 'flare/tools'
-
-module FlareTools
-class Stats < Core
-  # {{{ constractor
-  def initialize()
-    # {{{ @format
-    @format = "%20.20s:%5.5s"   # hostname:port
-    @format += " %6s"           # state
-    @format += " %6s"           # role
-    @format += " %9s"           # partition
-    @format += " %7s"           # balance
-    @format += " %8.8s"         # items
-    @format += " %4s"           # connection
-    @format += " %6.6s"         # behind
-    @format += " %3.3s"         # hit
-    @format += " %4.4s"         # size
-    @format += " %6.6s"         # uptime
-    @format += " %7s"           # version
-    @format += "\n"
-    # }}}
-    # {{{ @label
-    @label = @format % [
-      "hostname",
-      "port",
-      "state",
-      "role",
-      "partition",
-      "balance",
-      "items",
-      "conn",
-      "behind",
-      "hit",
-      "size",
-      "uptime",
-      "version",
-    ]
-    # }}}
-    super
-  end
-  # }}}
-  # {{{ option_on
-  def option_on
-    super
-    @option.on(             '--index-server=[HOSTNAME]',          "index server hostname(default:#{@index_server_hostname})") {|v| @index_server_hostname = v}
-    @option.on(             '--index-server-port=[PORT]',         "index server port(default:#{@index_server_port})") {|v| @index_server_port = v.to_i}
-  end
-  # }}}
-  # {{{ sort_node
-  def sort_node(nodes)
-    # sort partition role hostname
-    nodes.sort_by{|key, val| [val['partition'], val['role'], key]}
-  end
-  # }}}
-  # {{{ str_date
-  def str_date(date)
-    date = date.to_i
-    res = ""
-    # sec
-    if date >= 60
-      date = date / 60
-    else
-      return "#{date}s"
-    end
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
 
-    # min
-    if date >= 60
-      date = date / 60
-    else
-      return date + "m"
-    end
+require 'thread'
+require 'flare/tools/index_server'
+require 'flare/tools/cli/sub_command'
+require 'flare/tools/common'
+require 'flare/util/conversion'
 
-    # hour
-    if date >= 24
-      date = date / 24
-    else
-      return date + "h"
-    end
+# 
+module Flare
+  module Tools
+    module Cli
 
-    # day
-    "#{date}d"
-  end
-  # }}}
-  # {{{ execute
-  def execute
-    str = ""
-    nodes = self.get_stats_nodes
-    nodes = self.sort_node(nodes)
-    threads  = self.get_stats_threads
-    nodes.each do |hostname_port,data|
-      ipaddr, port = hostname_port.split(":", 2)
-      begin
-        hostname = Resolv.getname(ipaddr).to_s
-      rescue Resolv::ResolvError
-        hostname = ipaddr
+      # == Description
+      #
+      class Stats < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::Logging
+        include Flare::Tools::Common
+        
+        myname :stats
+        desc   "show the statistics of a flare cluster."
+        usage  "stats [hostname:port] ..."
+
+        HeaderConfig = [ ['%-25.25s', 'hostname:port'],
+                         ['%7s',      'state'],
+                         ['%6s',      'role'],
+                         ['%9s',      'partition'],
+                         ['%7s',      'balance'],
+                         ['%8.8s',    'items'],
+                         ['%4s',      'conn'],
+                         ['%6.6s',    'behind'],
+                         ['%3.3s',    'hit'],
+                         ['%4.4s',    'size'],
+                         ['%6.6s',    'uptime'],
+                         ['%7s',      'version'] ]
+
+        def setup(opt)
+          opt.on("-q", '--qps',              "show qps")                             {|v| @qps = v}
+          opt.on("-w", '--wait=SECOND',      "specify wait time for repeat(second)") {|v| @wait = v.to_i}
+          opt.on("-c", '--count=REPEATTIME', "specify repeat count")                 {|v| @count = v.to_i}
+          opt.on("-d", '--delimiter=CHAR',   "spedify delimiter")                    {|v| @delimiter = v}
+        end
+
+        def initialize
+          super
+          @qps = false
+          @wait = 1
+          @count = 1
+          @cont = true
+          @delimiter = ' '
+        end
+  
+        def interrupt
+          puts "INTERRUPTED"
+          @cont = false
+        end
+
+        def execute(config, *args)
+          nodes = {}
+          threads = {}
+          header = Marshal.load(Marshal.dump(HeaderConfig))
+          header << ['%5.5s', 'qps'] << ['%5.5s', 'qps-r'] << ['%5.5s', 'qps-w'] if @qps
+
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            nodes = s.stats_nodes
+            unless nodes
+              error "Invalid index server."
+              return S_NG
+            end
+            nodes = nodes.sort_by{|key,val| [val['partition'].to_i, val['role'], key]}
+            threads = s.stats_threads_by_peer
+          end
+
+          worker_threads = []
+          queue = {}
+
+          nodes.each do |hostname_port,data|
+            hostname, port = hostname_port.split(":", 2)
+            queue[hostname_port] = SizedQueue.new(1)
+            worker_threads << Thread.new(queue[hostname_port]) do |q|
+              s = nil
+              while @cont
+                stats_data = nil
+                begin
+                  s = Flare::Tools::Stats.open(hostname, data['port'], config[:timeout])
+                  stats = s.stats
+                  time = Time.now
+                  behind = threads[hostname_port].has_key?('behind') ? threads[hostname_port]['behind'] : "-"
+                  uptime_short = short_desc_of_second(stats['uptime'])
+                  hit_rate = if stats.has_key?('cmd_get') && stats['cmd_get'] != "0"
+                               cmd_get = stats['cmd_get'].to_f
+                               get_hits = stats['get_hits'].to_f
+                               (get_hits / cmd_get * 100.0).round
+                             else
+                               "-"
+                             end
+                  size =  stats['bytes'] == "0" ? "-" : (stats['bytes'].to_i / 1024 / 1024 / 1024) # gigabyte
+                  stats_data = {
+                    :hostname   => hostname,
+                    :port       => port,
+                    :hostname_port => "#{hostname}:#{port}",
+                    :state      => data['state'],
+                    :role       => data['role'],
+                    :partition  => data['partition'] == "-1" ? "-" : data['partition'],
+                    :balance    => data['balance'],
+                    :items      => stats['curr_items'],
+                    :conn       => stats['curr_connections'],
+                    :behind     => behind,
+                    :hit_rate   => hit_rate,
+                    :size       => size,
+                    :uptime     => stats['uptime'],
+                    :uptime_short => uptime_short,
+                    :version    => stats['version'],
+                    :cmd_get    => stats['cmd_get'],
+                    :cmd_set    => stats['cmd_set'],
+                    :time       => time,
+                  }
+                rescue Errno::ECONNREFUSED => e
+                rescue => e
+                  begin
+                    s.close unless s.nil?
+                  rescue => close_error
+                  end
+                  s = nil
+                end
+                if stats_data.nil?
+                  stats_data = {
+                    :hostname_port => "#{hostname}:#{port}",
+                  }
+                end
+                q.push stats_data
+              end
+              s.close unless s.nil?
+            end # Thread.new
+          end # nodes.each
+
+          query_prev = {} if @qps
+
+          if @count > 1 || @qps
+            interruptible {sleep 1}
+          end
+
+          s = Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout])
+          unless s
+            error "Couldn't connect to the index server."
+            return S_NG
+          end
+
+          (0...@count).each do |i|
+            nodes = s.stats_nodes
+            unless nodes
+              error "Invalid index server."
+              exit 1
+            end
+            nodes = nodes.sort_by{|key,val| [val['partition'].to_i, val['role'], key]}
+            threads = s.stats_threads_by_peer
+
+            break unless @cont
+            max_nodekey_length = 25
+            nodes.each do |k, n|
+              max_nodekey_length = k.length if k.length > max_nodekey_length
+            end
+            header[0][0] = "%-#{max_nodekey_length}.#{max_nodekey_length}s"
+            format = header.map {|x| x[0]}.join(@delimiter)
+            label = format % header.map{|x| x[1]}.flatten
+            puts label
+            nodes.each do |k, n|
+              stats_data = queue[k].pop
+              next if (args.size > 0 && !args.include?(k))
+              stats_data[:state] = n['state']
+              stats_data[:role] = n['role']
+              stats_data[:partition] = n['partition']
+              stats_data[:balance] = n['balance']
+              stats_data[:behind] = (threads.has_key?(k) || threads[k].has_key?('behind')) ? threads[k]['behind'] : "-"
+              output = [:hostname_port, :state, :role, :partition, :balance, :items,
+                        :conn, :behind, :hit_rate, :size, :uptime_short, :version].map {|x| stats_data[x]}
+              if @qps
+                query = {}
+                query[:query] = stats_data[:cmd_get].to_i+stats_data[:cmd_set].to_i
+                query[:query_r] = stats_data[:cmd_get].to_i
+                query[:query_w] = stats_data[:cmd_set].to_i
+                query[:time] = time = stats_data[:time]
+                if query_prev.has_key?(k)
+                  duration = (time-query_prev[k][:time]).to_f
+                  [:query, :query_r, :query_w].each do |x|
+                    diff = (query[x]-query_prev[k][x]).to_f
+                    qps = if diff > 0 then diff/duration else 0 end
+                    output << qps
+                  end
+                else
+                  output << 0 << 0 << 0
+                end
+                query_prev[k] = query.dup
+              end
+
+              puts format % output
+            end
+            interruptible {
+              wait_for_stats
+            }
+          end
+          s.close
+
+          @cont = false
+
+          queue.each do |k,q|
+            q.clear
+          end
+          
+          interruptible {
+            worker_threads.each do |t|
+              t.join
+            end
+          }
+          
+          S_OK
+        end
+
+        def wait_for_stats
+          if @qps || @count > 1
+            wait = @wait
+            while wait > 0 && @cont
+              sleep 1
+              wait -= 1
+            end
+          end
+        end
+        
       end
-      stats = self.get_stats(ipaddr, data['port'])
-      partition = data['partition'] == "-1" ? "-" : data['partition']
-      behind = threads[hostname_port].key?('behind') ? threads[hostname_port]['behind'] : "-"
-      uptime = self.str_date(stats['uptime'])
-      hit_rate = stats['cmd_get'] == "0" ?  "-" : (stats['get_hits'].to_f / stats['cmd_get'].to_f * 100.0).round
-      size =  stats['bytes'] == "0" ? "-" : (stats['bytes'].to_i / 1024 / 1024 / 1024)
-      str += @format % [
-        hostname,
-        port,
-        data['state'],
-        data['role'],
-        partition,
-        data['balance'],
-        stats['curr_items'],
-        stats['curr_connections'],
-        behind,
-        hit_rate,
-        size,
-        uptime,
-        stats["version"],
-      ]
     end
-    puts @label + str
   end
-  # }}}
-end
 end
+