flare-tools 0.1.4 → 0.4.5.1

data/lib/flare/tools/cli/sub_command.rb

@@ -0,0 +1,73 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/util/logging'
+require 'flare/util/interruption'
+
+module Flare
+  module Tools
+    module Cli
+      class SubCommand
+        @@myname = {}
+        @@desc = {}
+        @@usage = {}
+
+        S_OK = 0
+        S_NG = 1
+
+        def self.to_sym
+          myname
+        end
+
+        def self.myname(myname = nil)
+          if myname.nil? then @@myname[name] else @@myname[name] = myname end
+        end
+
+        def self.usage(usage = nil)
+          if usage.nil?
+            @@usage[name] = "" unless @@usage.has_key?(name)
+            @@usage[name]
+          else
+            @@usage[name] = usage
+          end
+        end
+
+        def self.desc(desc = nil)
+          if desc.nil?
+            @@desc[name] = "" unless @@desc.has_key?(name)
+            @@desc[name]
+          else
+            @@desc[name] = desc
+          end
+        end
+
+        def myname
+          myname = @@myname[self.class.name]
+          if myname.nil? then "" else myname end
+        end
+
+        def self.to_s
+          self.to_sym.to_s
+        end
+
+        def initalize
+        end
+
+        def setup(opt)
+          raise "setup"
+        end
+        
+        def execute(config, *args)
+          raise "execute"
+        end
+
+        include Flare::Util::Logging
+        include Flare::Util::Interruption
+      end
+    end
+  end
+end
+
+

data/lib/flare/tools/cli/summary.rb

@@ -0,0 +1,97 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'thread'
+require 'flare/tools/index_server'
+require 'flare/tools/cli/sub_command'
+require 'flare/tools/common'
+require 'flare/util/conversion'
+
+# 
+module Flare
+  module Tools
+    module Cli
+
+      # == Description
+      #
+      class Summary < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::Logging
+        include Flare::Tools::Common
+        
+        myname :summary
+        desc   "show the summary of a flare cluster."
+        usage  "summary [hostname:port] ..."
+
+        HeaderConfig = [ ['%-30.30s', '#cluster'],
+                         ['%6s',      'part'],
+                         ['%6s',      'node'],
+                         ['%7s',      'master'],
+                         ['%6s',      'slave'],
+                         ['%16s',     'size'],
+                         ['%13s',    'items'] ]
+
+        def setup(opt)
+        end
+
+        def initialize
+          super
+        end
+  
+        def execute(config, *args)
+          nodes = {}
+          threads = {}
+          header = HeaderConfig
+
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            nodes = s.stats_nodes
+            unless nodes
+              error "Invalid index server."
+              return S_NG
+            end
+            nodes = nodes.sort_by{|key,val| [val['partition'].to_i, val['role'], key]}
+          end
+
+          name = if config[:cluster].nil? then
+                   "#{config[:index_server_hostname]}:#{config[:index_server_port]}"
+                 else
+                   config[:cluster]
+                 end
+
+          total_bytes = 0
+          total_parts = 0
+          total_nodes = 0
+          total_items = 0
+          total_masters = 0
+          total_slaves = 0
+
+          nodes.each do |hostname_port,data|
+            hostname, port = hostname_port.split(":", 2)
+            Flare::Tools::Stats.open(hostname, data['port'], config[:timeout]) do |s|
+              stats = s.stats
+              p = data['partition'].to_i
+              total_parts = p+1 if p+1 > total_parts if data['role'] == 'master'
+              total_nodes += 1
+              total_masters += 1 if data['role'] == 'master'
+              total_slaves += 1 if data['role'] == 'slave'
+              total_bytes += stats['bytes'].to_i if data['role'] == 'master'
+              total_items += stats['curr_items'].to_i if data['role'] == 'master'
+            end
+          end
+
+          format = header.map {|x| x[0]}.join(@delimiter)
+          label = format % header.map{|x| x[1]}.flatten
+          puts label
+          puts(format % [name, total_parts, total_nodes, total_masters, total_slaves, total_bytes, total_items])
+          
+          S_OK
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/flare/tools/cli/threads.rb

@@ -0,0 +1,78 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/index_server'
+require 'flare/util/conversion'
+require 'flare/tools/common'
+require 'flare/tools/cli/sub_command'
+
+module Flare
+  module Tools
+    module Cli
+      class Threads < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Tools::Common
+
+        myname :threads
+        desc   "show the list of threads in a flare cluster."
+        usage  "threads [hostname:port]"
+  
+        def setup(opt)
+        end
+
+        def initialize
+        end
+
+        def execute(config, *args)
+          header = [
+                    ['%5s', 'id'],
+                    ['%-32s', 'peer'],
+                    ['%-13s', 'operation'],
+                    ['%4s', 'type'],
+                    ['%8s', 'queue']
+                   ]
+          format = header.map {|x| x[0]}.join(' ')
+
+          hostname = config[:index_server_hostname]
+          port = config[:index_server_port]
+
+          if args.size == 1
+            nodekey = nodekey_of args[0]
+            if nodekey.nil?
+              error "invalid nodekey: "+args[0]
+              return S_NG
+            end
+            hostname, port = nodekey.split(':')
+          elsif args.size > 1
+            error "invalid arguments: "+args.join(' ')
+            return S_NG
+          end
+          
+          threads = []
+          
+          Flare::Tools::Stats.open(hostname, port, config[:timeout]) do |s|
+            threads = s.stats_threads
+            threads = threads.sort_by{|key,val| [val['peer'], key]}
+          end
+          
+          puts format % header.map{|x| x[1]}.flatten
+
+          threads.each do |thread_id, data|
+            puts format % [
+                           thread_id,
+                           data['peer'],
+                           if data['op'].nil? then "-" else data['op'] end,
+                           data['type'],
+                           data['queue'],
+                          ]
+          end
+          
+          S_OK
+        end
+        
+      end
+    end
+  end
+end

data/lib/flare/tools/cli/verify.rb

@@ -0,0 +1,202 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/index_server'
+require 'flare/tools/node'
+require 'flare/util/conversion'
+require 'flare/util/key_resolver'
+require 'flare/util/hash_function'
+require 'flare/tools/cli/sub_command'
+require 'digest/md5'
+
+
+# 
+module Flare
+  module Tools
+    module Cli
+
+      # == Description
+      # 
+      class Verify < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::HashFunction
+
+        myname :verify
+        desc   "verify the cluster. (experimental)"
+        usage  "verify"
+  
+        def setup(opt)
+          opt.on('--key-hash-algorithm=TYPE',              "key hash algorithm") do |v|
+            case @key_hash_algorithm = v.to_sym
+            when :simple, :crc32
+            else
+              puts "unknown type: #{v}"
+              exit
+            end
+          end
+          opt.on('--use-test-data',                        "store test data")           {|v| @use_test_data = true}
+          opt.on('--debug',                                "use debug mode")            {|v| @debug = true}
+          opt.on('--64bit',                                "(experimental) 64bit mode") {|v| @word_size = 64}
+          opt.on('--verbose',                              "use verbose mode")          {|v| @verbose = true}
+          opt.on('--meta',                                 "use meta command")          {|v| @meta = true}
+          opt.on('--quiet',                                "use quiet mode")            {|v| @quiet = true}
+        end
+
+        def initialize
+          super
+          @numeric_hosts = false
+          @key_hash_algorithm = :simple
+          @use_test_data = false
+          @debug = false
+          @word_size = 32
+          @bwlimit = 0
+          @verbose = false
+          @meta = false
+          @quiet = false
+        end
+
+        def execute(config, *args)
+          keys = {}
+          cout = STDERR
+          status = S_OK
+          info "connecting to index ..."
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            nodes = s.stats_nodes.sort_by{|key, val| [val['partition'].to_i, val['role'], key]}
+
+            # meta
+            info "setting up key resolver ..."
+            resolver = nil
+            if @meta
+              meta = s.meta
+              stats = s.stats
+              kha = meta['key-hash-algorithm']
+              if kha
+                @key_hash_algorithm = :crc32 if kha == 'crc32'
+                @key_hash_algorithm = :simple if kha == 'simple'
+              else
+                @key_hash_algorithm = :simple
+              end
+              pointer_size = stats['pointer_size']
+              option = {
+                :partition_size => meta['partition-size'].to_i,
+                :virtual => meta['partition-modular-virtual'].to_i,
+                :hint => meta['partition-modular-hint'].to_i
+              }
+              resolver = Util::KeyResolver.new(:modular, option)
+            else
+              resolver = Util::KeyResolver.new
+            end
+            info "key_hash_algorithm = #{@key_hash_algorithm.to_s}"
+
+            # check node list size
+            if nodes.size == 0
+              cout.puts "no nodes"
+              return S_NG
+            end
+            hostname0, port0 = nodes[0][0].split(":", 2)
+
+            # partition size
+            partition_size = 1+nodes.inject(-1) do |r,entry|
+              node, val = entry
+              i = val['partition'].to_i
+              if i >= r then i else r end
+            end
+            if partition_size <= 0
+              info "no need to verify."
+              return S_NG
+            end
+            info "partition_size: #{partition_size}"
+
+            if @use_test_data
+              info "storing test data ..."
+              Flare::Tools::Node.open(hostname0, port0.to_i, config[:timeout]) do |n|
+                (1..10000).each do |i|
+                  key = ".test."+Digest::MD5.new.update(i.to_s).to_s
+                  n.set(key, i.to_s)
+                  keys[key] = :not_found
+                end
+              end
+            end
+
+            nodes.each do |nodekey,val|
+              hostname, port = nodekey.split(":", 2)
+              partition = val['partition'].to_i
+              Flare::Tools::Node.open(hostname, port.to_i, config[:timeout]) do |n|
+                cout.write "checking #{nodekey} ... "
+                msg = "OK"
+                interruptible do
+                  count = 0
+                  cout.write "keydump ... "
+                  n.dumpkey(partition, partition_size) do |key|
+                    next if key.nil?
+                    type = @key_hash_algorithm
+                    hash = get_key_hash_value(key, type, @word_size)
+                    p = resolver.resolve(hash, partition_size)
+                    count += 1
+                    if p != partition then
+                      cout.puts "keydump failed: the partition for #{key}(#{hash}) is #{p} but it was dumpped from #{partition}." if @debug
+                      status = S_NG
+                      msg = "NG"
+                    else
+                      keys[key] = :found
+                    end
+                    false
+                  end
+                  cout.write "#{count} entries. "
+                  count = 0
+                  cout.write "dump ... "
+                  n.dump(0, partition, partition_size, @bwlimit) do |data, key, flag, len, version, expire|
+                    next if key.nil?
+                    type = @key_hash_algorithm
+                    hash = get_key_hash_value(key, type, @word_size)
+                    p = resolver.resolve(hash, partition_size)
+                    count += 1
+                    if p != partition then
+                      cout.puts "dump failed: the partition for #{key}(#{hash}) is #{p} but it was dumpped from #{partition}." if @debug
+                      status = S_NG
+                      msg = "NG"
+                    end
+                    false
+                  end
+                  cout.write "#{count} entries. "
+                end # interruptible
+                cout.write "#{msg}\n"
+              end # Node.open
+            end # nodes.each
+
+            if @use_test_data && keys.size > 0
+              # check total result
+              remain = 0
+              keys.each do |k,state|
+                if state != :found
+                  error "failed: not found '#{k}'" if @verbose
+                  remain += 1
+                end
+              end
+              error "failed: not found #{remain} keys" if remain > 0
+
+              # delete
+              Flare::Tools::Node.open(hostname0, port0.to_i, config[:timeout]) do |n|
+                keys.each do |k,v|
+                  n.delete(k)
+                end
+              end
+            end
+
+            # end of connection
+          end
+          if status == S_OK
+            cout.puts "OK"
+          else
+            cout.puts "NG"
+          end
+          status
+        end
+        
+      end
+    end
+  end
+end
+