flare-tools 0.1.4 → 0.4.5.1

data/lib/flare/tools/common.rb

@@ -0,0 +1,196 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'resolv'
+require 'flare/tools/cluster'
+require 'flare/tools/stats'
+require 'flare/util/constant'
+require 'flare/util/logging'
+
+# 
+module Flare
+  module Tools
+    module Common
+      include Flare::Util::Logging
+
+      def fetch_cluster s
+        Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
+      end
+
+      def user_confirmed opt = { true => /^Y$/, false => /^N$/ }, &block
+        ret = nil
+        while ret.nil?
+          line = gets.chomp.upcase
+          opt.each do |key,pattern|
+            if patterns =~ line
+              ret = key
+              break
+            end
+          end
+        end
+        if block.nil?
+          ret
+        else
+          block.call(ret)
+        end
+      end
+
+      def nodekey_of *args
+        if args.size == 1
+          args = if args[0].kind_of?(Array)
+                   args[0]
+                 elsif args[0].kind_of?(String)
+                   args[0].split(':')
+                 end
+        end
+        if args.size >= 2
+          hostname, port = args
+          if hostname.kind_of?(String) && port.kind_of?(String)
+            if port.empty?
+              port = Flare::Util::Constant::DefaultNodePort
+              return "#{hostname}:#{port}"
+            elsif /^\d+$/ =~ port
+              return "#{hostname}:#{port}"
+            end
+          elsif hostname.kind_of?(String) && port.kind_of?(Integer)
+            return "#{hostname}:#{port}"
+          end
+        end
+        nil
+      end
+
+      def address_of_hostname(hostname)
+        Resolv.getaddress(hostname)
+      rescue Resolv::ResolvError
+        hostname
+      end
+
+      def hostname_of_address(ipaddr)
+        Resolv.getname(ipaddr)
+      rescue Resolv::ResolvError
+        ipaddr
+      end
+      
+      NodeListHeader = [ ['%-32s', 'node'],
+                         ['%9s', 'partition'],
+                         ['%6s', 'role'],
+                         ['%6s', 'state'],
+                         ['%7s', 'balance'] ]
+      NodeListFormat = (NodeListHeader.map {|x| x[0]}.join(' '))
+
+      def string_of_nodelist(nodes, opt = {})
+        format = NodeListFormat+"\n"
+        ret = format % NodeListHeader.map{|x| x[1]}.flatten
+        nodes.each do |nodekey, node|
+          if opt.empty? || opt.include?(nodekey)
+            partition = if node['partition'] == "-1"
+                          "-"
+                        else
+                          node['partition']
+                        end
+            ret += format % [
+                             nodekey,
+                             partition,
+                             node['role'],
+                             node['state'],
+                             node['balance'],
+                            ]
+          end
+        end
+        ret
+      end
+
+      # s:IndexServer, nodekey:"hostname:port", timeout(second):Integer -> state:String
+      def wait_for_slave_construction(index_server, nodekey, timeout, silent = false)
+        cluster = Flare::Tools::Cluster.new(index_server.host, index_server.port, index_server.stats_nodes)
+        slave = cluster.node_stat(nodekey)
+        partition = slave['partition'].to_i
+        m_hostname, m_port = cluster.master_in_partition(partition).split(':')
+        s_hostname, s_port = nodekey.split(':')
+        m = Flare::Tools::Stats.open(m_hostname, m_port.to_i, timeout)
+        s = Flare::Tools::Stats.open(s_hostname, s_port.to_i, timeout)
+        start = Time.now
+        while true
+          cluster = Flare::Tools::Cluster.new(index_server.host, index_server.port, index_server.stats_nodes)
+          slave = cluster.node_stat(nodekey)
+          stats_master = m.stats
+          stats_slave = s.stats
+          ts_diff = Time.now-start
+          state_slave = slave['state'];
+          role_slave = slave['role'];
+          item_m = stats_master['curr_items'].to_i;
+          item_s = stats_slave['curr_items'].to_i;
+          eta = if ts_diff > 0 && item_s > 0
+                  ((item_m - item_s) / (item_s / ts_diff)).to_i;
+                  else
+                  "n/a";
+                end
+          unless silent
+            STDERR.puts "%d/%d (role = %s, state = %s) [ETA: %s sec (elapsed = %d sec)]" % [item_s, item_m, role_slave, state_slave, eta, ts_diff]
+          end
+          break if role_slave == "slave" && state_slave == "active"
+          sleep 1
+        end
+        info "state is active -> stop waiting" unless silent
+        m.close
+        s.close
+        state_slave
+      end
+
+      # s:IndexServer, nodekey:"hostname:port", timeout(second):Integer -> state:String
+      def wait_for_master_construction(index_server, nodekey, timeout, silent = false)
+        cluster = Flare::Tools::Cluster.new(index_server.host, index_server.port, index_server.stats_nodes)
+        master = cluster.node_stat(nodekey)
+        partition = master['partition'].to_i
+        m_hostname, m_port = cluster.master_in_partition(partition).split(':')
+        m = Flare::Tools::Stats.open(m_hostname, m_port.to_i, timeout)
+        start = Time.now
+        while true
+          cluster = Flare::Tools::Cluster.new(index_server.host, index_server.port, index_server.stats_nodes)
+          master = cluster.node_stat(nodekey)
+          stats_master = m.stats
+          ts_diff = Time.now-start
+          state_master = master['state'];
+          role_master = master['role'];
+          item_m = stats_master['curr_items'].to_i;
+          eta = "n/a";
+          unless silent
+            STDERR.puts "%d (role = %s, state = %s) [ETA: %s sec (elapsed = %d sec)]" % [item_m, role_master, state_master, eta, ts_diff]
+          end
+          if role_master == "master" && state_master == "active"
+            if partition != 0
+              warn "The master should be ready after the reconstruction but it became active."
+            end
+            break
+          end
+          break if role_master == "master" && state_master == "ready"
+          sleep 1
+        end
+        info "state is ready -> stop waiting" unless silent
+        m.close
+        state_master
+      end
+
+      def wait_for_servers(index_server, timeout = Flare::Util::Constant::DefaultTimeout, silent = false)
+        index_server.stats_nodes.each do |nodekey, v|
+          hostname, port = nodekey.split(':')
+          is_alive = false
+          while is_alive
+            begin
+              Flare::Tools::Node.open(hostname, port.to_i, 2) do |n|
+                n.ping
+                is_alize = true
+              end
+            rescue Errno::ECONNREFUSED
+            rescue SocketError
+            end
+            sleep 1 unless is_alive
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/flare/tools/index_server.rb

@@ -0,0 +1,51 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/stats'
+
+# 
+module Flare
+  module Tools
+
+    # == Description
+    # 
+    class IndexServer < Stats
+
+      def set_role(host, port, role, balance, partition)
+        set_role_(host, port, role, balance, partition)
+      end
+      defcmd :set_role_, 'node role %s %d %s %d %d\r\n' do |resp|
+        resp
+      end
+
+      def set_state(host, port, state)
+        set_state_(host, port, state)
+      end
+      defcmd :set_state_, 'node state %s %s %s\r\n' do |resp|
+        resp
+      end
+
+      def node_remove(host, port)
+        node_remove_(host, port)
+      end
+      defcmd :node_remove_, 'node remove %s %s\r\n' do |resp|
+        resp
+      end
+
+      def meta()
+        meta_()
+      end
+      defcmd :meta_, 'meta\r\n' do |resp|
+        result = {}
+        resp.gsub(/META /, '').split("\r\n").each do |x|
+          key, val = x.split(" ", 2)
+          result[key] = val
+        end
+        result
+      end
+
+    end
+  end
+end

data/lib/flare/tools/node.rb

@@ -0,0 +1,162 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/stats'
+
+# 
+module Flare
+  module Tools
+
+    # == Description
+    # 
+    class Node < Stats
+      
+      def set_state(host, port, state)
+        set_state_(host, port, state)
+      end
+      defcmd :set_state_, 'node state %s %s %s\r\n' do |resp| resp end
+
+      def flush_all
+        flush_all_
+      end
+      defcmd :flush_all_, 'flush_all\r\n' do |resp| resp end
+
+      def x_list_push(k, v, flag = 0, expire = 0)
+        x_list_push_(k.chomp, flag, expire, v.size, v)
+      end
+      defcmd :x_list_push_, 'list_push %s %d %d %d\r\n%s\r\n' do |resp| resp end
+
+      def x_list_unshift(k, v, flag = 0, expire = 0)
+        x_list_unshift_(k.chomp, flag, expire, v.size, v)
+      end
+      defcmd :x_list_unshift_, 'list_unshift %s %d %d %d\r\n%s\r\n' do |resp| resp end
+
+      def set(k, v, flag = 0, expire = 0)
+        set_(k.chomp, flag, expire, v.size, v)
+      end
+      defcmd :set_, 'set %s %d %d %d\r\n%s\r\n' do |resp| resp end
+
+      def set_noreply(k, v, flag = 0, expire = 0)
+        set_noreply_(k.chomp, flag, expire, v.size, v)
+      end
+      defcmd_noreply :set_noreply_, 'set %s %d %d %d noreply\r\n%s\r\n'
+
+      def cas(k, v, casunique, flag = 0, expire = 0)
+        r = cas_(k.chomp, flag, expire, v.size, casunique, v)
+        r = true if r == ""
+        r
+      end
+      defcmd :cas_, 'cas %s %d %d %d %d\r\n%s\r\n' do |resp| resp end
+
+      def delete(k)
+        delete_(k.chomp)
+      end
+      defcmd :delete_, 'delete %s\r\n' do |resp| resp end
+
+      def delete_noreply(k)
+        delete_noreply_(k.chomp)
+      end
+      defcmd_noreply :delete_noreply_, 'delete %s noreply\r\n'
+
+      def x_list_pop(k)
+        r = nil
+        x_list_pop_(k.chomp) do |data, key, flag, len, version, expire|
+          r = data
+        end
+        r
+      end
+      defcmd_value :x_list_pop_, 'list_pop %s\r\n'
+
+      def x_list_shift(k)
+        r = nil
+        x_list_shift_(k.chomp) do |data, key, flag, len, version, expire|
+          r = data
+        end
+        r
+      end
+      defcmd_value :x_list_shift_, 'list_shift %s\r\n'
+
+      def x_list_get(k, b, e, &block)
+        x_list_get_(k.chomp, b, e, &block)
+      end
+      defcmd_listelement :x_list_get_, 'list_get %s %d %d\r\n' do |data, key, rel, abs, flag, len, version, expire|
+        data
+      end
+
+      def get(*keys, &block)
+        return false if keys.size == 0
+        r = get_(keys.map{|x|x.chomp}.join(' '), &block)
+        if r && keys.size == 1
+          return false if r.size == 0
+          return r[0] if r.size == 1
+        end
+        r
+      end
+      defcmd_value :get_, 'get %s\r\n' do |data, key, flag, len, version, expire|
+        data
+      end
+
+      def gets(*keys, &block)
+        return false if keys.size == 0
+        r = gets_(keys.map{|x|x.chomp}.join(' '), &block)
+        if r && keys.size == 1
+          return false if r.size == 0
+          return r[0] if r.size == 1
+        end
+        r
+      end
+      defcmd_value :gets_, 'gets %s\r\n' do |data, key, flag, len, version, expire|
+        [data, version]
+      end
+
+      def dump(wait = 0, part = 0, partsize = 1, bwlimit = 0, &block)
+        dump_(wait, part, partsize, bwlimit.to_i, &block)
+      end
+      defcmd_value :dump_, 'dump %d %d %d %d\r\n' do |data, key, flag, len, version, expire|
+        false
+      end
+
+      def dumpkey(part = nil, partsize = nil, &block)
+        return dumpkey_0_(&block) if part.nil?
+        return dumpkey_1_(part, &block) if partsize.nil?
+        return dumpkey_2_(part, partsize, &block)
+      end
+      defcmd_key :dumpkey_0_, 'dump_key\r\n' do |key|
+        false
+      end
+      defcmd_key :dumpkey_1_, 'dump_key %d\r\n' do |key|
+        false
+      end
+      defcmd_key :dumpkey_2_, 'dump_key %d %d\r\n' do |key|
+        false
+      end
+
+      def incr_noreply(k, v)
+        incr_noreply_(k.chomp, v.to_s)
+      end
+      defcmd_noreply :incr_noreply_, 'incr %s %s noreply\r\n'
+
+      def incr(k, v)
+        incr_(k.chomp, v.to_s)
+      end
+      defcmd_oneline :incr_, 'incr %s %s\r\n' do |resp|
+        resp.chomp
+      end
+
+      def decr_noreply(k, v)
+        decr_noreply_(k.chomp, v.to_s)
+      end
+      defcmd_noreply :decr_noreply_, 'decr %s %s noreply\r\n'
+
+      def decr(k, v)
+        decr_(k.chomp, v.to_s)
+      end
+      defcmd_oneline :decr_, 'decr %s %s\r\n' do |resp|
+        resp.chomp
+      end
+
+    end
+  end
+end

data/lib/flare/tools/stats.rb

@@ -0,0 +1,75 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'timeout'
+require 'flare/tools/client'
+
+# 
+module Flare
+  module Tools
+
+    # == Description
+    # 
+    class Stats < Client
+
+      defcmd :stats_nodes, 'stats nodes\r\n' do |resp|
+        result = {}
+        resp.gsub(/STAT /, '').split("\r\n").each do |x|
+          ip, port, stat = x.split(":", 3)
+          key, val = stat.split(" ")
+          result["#{ip}:#{port}"] = {} if result["#{ip}:#{port}"].nil?
+          result["#{ip}:#{port}"]['port'] = port
+          result["#{ip}:#{port}"][key] = val
+        end
+        result
+      end
+
+      defcmd :stats, 'stats\r\n' do |resp|
+        result = {}
+        resp.gsub(/STAT /, '').split("\r\n").each do |x|
+          key, val = x.split(" ", 2)
+          result[key] = val
+        end
+        result
+      end
+
+      def stats_threads_by_peer
+        result = {}
+        stats_threads.each do |thread_id, stat|
+          k = stat['peer']
+          result[k] = {} if result[k].nil?
+          result[k]['thread_id'] = thread_id
+          result[k].merge!(stat)
+        end
+        result
+      end
+
+      def stats_threads
+        return {} unless required_version?([1, 0, 10])
+        stats_threads_
+      end
+
+      defcmd :stats_threads_, 'stats threads\r\n' do |resp|
+        threads = {}
+        resp.gsub(/STAT /, '').split("\r\n").each do |x|
+          thread_id, stat = x.split(":", 2)
+          key, val = stat.split(" ")
+          threads[thread_id] = {} unless threads.has_key?(thread_id)
+          threads[thread_id][key] = val
+        end
+        threads
+      end
+
+      defcmd :ping, 'ping\r\n' do |resp|
+        true if resp
+      end
+
+      defcmd :quit, 'quit\r\n' do |resp|
+        true
+      end
+
+    end
+  end
+end