flare-tools 0.4.5.1 → 0.5.0

data/lib/flare/tools/cli/deploy.rb

@@ -6,68 +6,70 @@
 require 'flare/util'
 require 'flare/tools/index_server'
 require 'flare/tools/cli/sub_command'
+require 'flare/tools/cli/index_server_config'
 
-# 
 module Flare
   module Tools
     module Cli
 
-      # == Description
-      # 
       class Deploy < SubCommand
         include Flare::Util::Conversion
         include Flare::Util::FileSystem
+        include Flare::Tools::Cli::IndexServerConfig
 
         myname :deploy
         desc "deploy for Debian-based system."
         usage "deploy [hostname:port:balance:partition] ..."
-  
-        def setup(opt)
-          opt.on('--deploy-index',                        "deploy the index server") {@deploy_index = true}
-          opt.on('--delete',                              "delete existing contents before deploying") {@delete = true}
-          opt.on('--flarei=PATH',                       "a path for flarei executable") {|v| @flarei = v}
-          opt.on('--flared=PATH',                       "a path for flared executable") {|v| @flared = v}
-          opt.separator('flarei/flared options:')
-          opt.on('--back-log=BACKLOG',                  "back log parameter for listen()"    ) {|v| @iconf["back-log"] = v}
-          opt.on('--thread-pool-size=SIZE',             "thread pool size"                   ) {|v| @iconf["thread_pool_size"] = @dconf["thread_pool_size"] = v}
-          opt.on('--daemonize=BOOL',                    "daemonize"                          ) {|v| @iconf["daemonize"] = @dconf["daemonize"] = v}
-          opt.on('--data-dir=PATH',                     "data directory"                     ) {|v| @iconf["data-dir"] = @dconf["data-dir"] = v}
-          opt.on('--log-facility=NAME',                 "log facility"                       ) {|v| @iconf["log-facility"] = @dconf["log-facility"] = v}
-          opt.on('--max-connection=SIZE',               "max connection"                     ) {|v| @iconf["max-connection"] = @dconf["max-connection"] = v}
-          opt.on('--net-read-timeout=MILLISECOND',      "read timeout for server connections") {|v| @iconf["net-read-timeout"] = @dconf["net-read-timeout"] = v}
-          opt.on('--stack-size=KB',                     "stack size"                         ) {|v| @iconf["stack-size"] = @dconf["stack-size"] = v}
-          opt.separator('flarei options:')
-          opt.on('--monitor-threshold=COUNT',           "monitor threshold"                  ) {|v| @iconf["monitor-threshold"] = v}
-          opt.on('--monitor-interval=SECOND',           "monitor interval"                   ) {|v| @iconf["monitor-interval"] = v}
-          opt.on('--monitor-read-timeout=MILLISECOND',  "monitor read timeout in millisecond") {|v| @iconf["monitor-read-timeout"] = v}
-          opt.on('--partition-type=NAME',               "partition type(modular)"            ) {|v| @iconf["partition-type"] = v}
-          opt.on('--key-hash-algorithm=NAME'            "hash algorithm for key distribution") {|v| @iconf["key-hash-algorithm"] = v}
-          opt.separator('flared options:')
-          opt.on('--proxy-concurrency=SIZE',            "proxy concurrency"                  ) {|v| @dconf["proxy-concurrency"] = v}
-          opt.on('--mutex-slot=SIZE',                   "mutex slot size"                    ) {|v| @dconf["mutex-slot"]= v}
-          opt.on('--reconstruction-interval=MICROSEC',  "reconstruction interval"            ) {|v| @dconf["reconstruction-interval"] = v}
-          opt.on('--reconstruction-bwlimit=BYTES',      "reconstruction bandwitdh limit"     ) {|v| @dconf["reconstruction-bwlimit"] = v}
-          opt.on('--storage-ap=SIZE',                   "alignment power"                    ) {|v| @dconf["storage-ap"] = v}
-          opt.on('--storage-bucket-size=SIZE',          "storage bucket size"                ) {|v| @dconf["storage-bucket-size"] = v}
-          opt.on('--storage-cache-size=SIZE',           "storage cache size"                 ) {|v| @dconf["storage-cache-size"] = v}
-          opt.on('--storage-compress=NAME',             "storage compress type(deflate|bz2|tcbs)") {|v| @dconf["storage-compress"] = v}
-          opt.on('--storage-large',                       "strage large"                       ) {@dconf["storage-large"] = true}
-          opt.on('--storage-type=NAME',                 "storage type"                       ) {|v| @dconf["storage-type"] = v}
-          opt.on('--proxy-prior-netmask=SIZE',          "proxy priority mask (ex. 24 for 255.255.255.0)") {|v| @dconf["proxy-prior-netmask"] = v}
-          opt.on('--max-total-thread-queue=SIZE',       "max total thread queue"             ) {|v| @dconf["max-total-thread-queue"] = v}
-          opt.on('--index-servers=NAME',                "index servers"                      ) {|v| @dconf["index-servers"] = v}
-          opt.on('--noreply-window-limit=SIZE',         "noreply window limit (experimental)") {|v| @dconf["noreply-window-limit"] = v}
-          opt.on('--mysql-replication',                   "MySQL replication (experimental)"   ) {@dconf["mysql-replication"] = true}
-          opt.on('--mysql-replication-port=PORT',       "MySQL replication port (experimental)") {|v| @dconf["mysql-replication-port"] = v}
-          opt.on('--mysql-replication-id=ID',           "MySQL replication ID (experimental)") {|v| @dconf["mysql-replication-id"] = v}
-          opt.on('--mysql-replication-db=NAME',         "MySQL replication DB (experimental)") {|v| @dconf["mysql-replication-db"] = v}
-          opt.on('--mysql-replication-table=NAME',      "MySQL replication table (experimental)") {|v| @dconf["mysql-replication-table"] = v}
-          opt.on('--proxy-cache-size=SIZE',             "proxy cache entry size (experimental)") {|v| @dconf["proxy-cache-size"] = v}
-          opt.on('--proxy-cache-expire=SECOND',         "cache life-time in second (experimental)") {|v| @dconf["proxy-cache-expire"] = v}
+
+        def setup
+          super
+          set_option_index_server
+          @optp.on('--deploy-index',                        "deploy the index server") {@deploy_index = true}
+          @optp.on('--delete',                              "delete existing contents before deploying") {@delete = true}
+          @optp.on('--flarei=PATH',                       "a path for flarei executable") {|v| @flarei = v}
+          @optp.on('--flared=PATH',                       "a path for flared executable") {|v| @flared = v}
+          @optp.separator('flarei/flared options:')
+          @optp.on('--back-log=BACKLOG',                  "back log parameter for listen()"    ) {|v| @iconf["back-log"] = v}
+          @optp.on('--thread-pool-size=SIZE',             "thread pool size"                   ) {|v| @iconf["thread_pool_size"] = @dconf["thread_pool_size"] = v}
+          @optp.on('--daemonize=BOOL',                    "daemonize"                          ) {|v| @iconf["daemonize"] = @dconf["daemonize"] = v}
+          @optp.on('--data-dir=PATH',                     "data directory"                     ) {|v| @iconf["data-dir"] = @dconf["data-dir"] = v}
+          @optp.on('--log-facility=NAME',                 "log facility"                       ) {|v| @iconf["log-facility"] = @dconf["log-facility"] = v}
+          @optp.on('--max-connection=SIZE',               "max connection"                     ) {|v| @iconf["max-connection"] = @dconf["max-connection"] = v}
+          @optp.on('--net-read-timeout=MILLISECOND',      "read timeout for server connections") {|v| @iconf["net-read-timeout"] = @dconf["net-read-timeout"] = v}
+          @optp.on('--stack-size=KB',                     "stack size"                         ) {|v| @iconf["stack-size"] = @dconf["stack-size"] = v}
+          @optp.separator('flarei options:')
+          @optp.on('--monitor-threshold=COUNT',           "monitor threshold"                  ) {|v| @iconf["monitor-threshold"] = v}
+          @optp.on('--monitor-interval=SECOND',           "monitor interval"                   ) {|v| @iconf["monitor-interval"] = v}
+          @optp.on('--monitor-read-timeout=MILLISECOND',  "monitor read timeout in millisecond") {|v| @iconf["monitor-read-timeout"] = v}
+          @optp.on('--partition-type=NAME',               "partition type(modular)"            ) {|v| @iconf["partition-type"] = v}
+          @optp.on('--key-hash-algorithm=NAME'            "hash algorithm for key distribution") {|v| @iconf["key-hash-algorithm"] = v}
+          @optp.separator('flared options:')
+          @optp.on('--proxy-concurrency=SIZE',            "proxy concurrency"                  ) {|v| @dconf["proxy-concurrency"] = v}
+          @optp.on('--mutex-slot=SIZE',                   "mutex slot size"                    ) {|v| @dconf["mutex-slot"]= v}
+          @optp.on('--reconstruction-interval=MICROSEC',  "reconstruction interval"            ) {|v| @dconf["reconstruction-interval"] = v}
+          @optp.on('--reconstruction-bwlimit=BYTES',      "reconstruction bandwitdh limit"     ) {|v| @dconf["reconstruction-bwlimit"] = v}
+          @optp.on('--storage-ap=SIZE',                   "alignment power"                    ) {|v| @dconf["storage-ap"] = v}
+          @optp.on('--storage-bucket-size=SIZE',          "storage bucket size"                ) {|v| @dconf["storage-bucket-size"] = v}
+          @optp.on('--storage-cache-size=SIZE',           "storage cache size"                 ) {|v| @dconf["storage-cache-size"] = v}
+          @optp.on('--storage-compress=NAME',             "storage compress type(deflate|bz2|tcbs)") {|v| @dconf["storage-compress"] = v}
+          @optp.on('--storage-large',                       "strage large"                       ) {@dconf["storage-large"] = true}
+          @optp.on('--storage-type=NAME',                 "storage type"                       ) {|v| @dconf["storage-type"] = v}
+          @optp.on('--proxy-prior-netmask=SIZE',          "proxy priority mask (ex. 24 for 255.255.255.0)") {|v| @dconf["proxy-prior-netmask"] = v}
+          @optp.on('--max-total-thread-queue=SIZE',       "max total thread queue"             ) {|v| @dconf["max-total-thread-queue"] = v}
+          @optp.on('--index-servers=NAME',                "index servers"                      ) {|v| @dconf["index-servers"] = v}
+          @optp.on('--noreply-window-limit=SIZE',         "noreply window limit (experimental)") {|v| @dconf["noreply-window-limit"] = v}
+          @optp.on('--mysql-replication',                   "MySQL replication (experimental)"   ) {@dconf["mysql-replication"] = true}
+          @optp.on('--mysql-replication-port=PORT',       "MySQL replication port (experimental)") {|v| @dconf["mysql-replication-port"] = v}
+          @optp.on('--mysql-replication-id=ID',           "MySQL replication ID (experimental)") {|v| @dconf["mysql-replication-id"] = v}
+          @optp.on('--mysql-replication-db=NAME',         "MySQL replication DB (experimental)") {|v| @dconf["mysql-replication-db"] = v}
+          @optp.on('--mysql-replication-table=NAME',      "MySQL replication table (experimental)") {|v| @dconf["mysql-replication-table"] = v}
+          @optp.on('--proxy-cache-size=SIZE',             "proxy cache entry size (experimental)") {|v| @dconf["proxy-cache-size"] = v}
+          @optp.on('--proxy-cache-expire=SECOND',         "cache life-time in second (experimental)") {|v| @dconf["proxy-cache-expire"] = v}
 
         end
 
         def initialize
+          super
           @deploy_index = false
           @delete = false
           @flarei = "/usr/local/bin/flarei"
@@ -87,14 +89,14 @@ module Flare
             f.puts starting
           end
           File.chmod(0744, start)
-          
+
           stop = basedir+"/stop.sh"
           open(stop, "w") do |f|
             f.puts "#!/bin/sh"
             f.puts stopping
           end
           File.chmod(0744, stop)
-          
+
           restart = basedir+"/restart.sh"
           open(restart, "w") do |f|
             f.puts "#!/bin/sh"
@@ -105,7 +107,9 @@ module Flare
           File.chmod(0744, restart)
         end
 
-        def execute(config, *args)
+        def execute(config, args)
+          parse_index_server(config, args)
+
           if @deploy_index
             hostname = config[:index_server_hostname]
             port = config[:index_server_port]

data/lib/flare/tools/cli/dispatch.rb

@@ -0,0 +1,101 @@
+require 'optparse'
+require 'resolv'
+require 'flare/tools'
+require 'flare/util/logging'
+require 'flare/util/conversion.rb'
+require 'flare/tools/cli/option'
+require 'flare/util/constant'
+
+module Flare; end
+module Flare::Tools; end
+module Flare::Tools::Cli; end
+class Flare::Tools::Cli::Dispatch
+  include Flare::Util::Logging
+  include Flare::Tools::Cli::Option
+  include Flare::Util::Constant
+  Cli = Flare::Tools::Cli
+
+  attr_accessor :subcommands
+
+  def initialize
+    @subcommands = {
+      'list'        => Cli::List,
+      'balance'     => Cli::Balance,
+      'down'        => Cli::Down,
+      'slave'       => Cli::Slave,
+      'reconstruct' => Cli::Reconstruct,
+      'master'      => Cli::Master,
+      'threads'     => Cli::Threads,
+      'ping'        => Cli::Ping,
+      'remove'      => Cli::Remove,
+      'index'       => Cli::Index,
+      'activate'    => Cli::Activate,
+      'dump'        => Cli::Dump,
+      'dumpkey'     => Cli::Dumpkey,
+      'verify'      => Cli::Verify,
+      'stats'       => Cli::Stats,
+      'restore'     => Cli::Restore,
+      'summary'     => Cli::Summary,
+      'part'        => Cli::Part,
+    }
+  end
+
+  def main(subcommand_name, argv, as_subcommand)
+    _main(subcommand_name, argv, as_subcommand)
+  rescue => e
+    level = 1
+    error(e.to_s)
+    e.backtrace.each do |line|
+      error("  %3s: %s" % [level, line])
+      level += 1
+    end
+    raise e if $DEBUG
+    STATUS_NG
+  end
+
+private
+
+  def _main(subcommand_name, argv, as_subcommand)
+    @subcommand_name = subcommand_name
+
+    subc = dispatch_subcommmand(@subcommand_name)
+    unless subc
+      show_undefined_command_error(as_subcommand)
+      exit STATUS_NG
+    end
+    set_banner subc.optp, as_subcommand
+    config = {
+      :command => File.basename($PROGRAM_NAME),
+    }
+    subc.execute_subcommand(config, argv)
+  end
+
+  def dispatch_subcommmand(name)
+    return nil unless @subcommands.include?(name)
+    @subcommands[name].new
+  end
+
+  def set_banner(optp, as_subcommand)
+    if as_subcommand
+      optp.banner = "#{Flare::Tools::TITLE}\nUsage: flare-#{@subcommand_name} [options]"
+    else
+      optp.banner = "#{Flare::Tools::TITLE}\nUsage: flare-admin #{@subcommand_name} [options]"
+    end
+  end
+
+  def show_undefined_command_error(as_subcommand)
+    if as_subcommand
+      error "unknown subcommand '#{@subcommand_name}'"
+      puts "subcommands:"
+      @subcommands.each do |name, klass|
+        sc = klass.new
+        sc.setup
+        sc.optp.banner = "[#{klass.to_s}] " + klass.desc
+        sc.optp.separator("  Usage: flare-admin " + klass.usage)
+        puts sc.optp.help
+      end
+    else
+      puts "unknown command"
+    end
+  end
+end

data/lib/flare/tools/cli/down.rb

@@ -11,30 +11,35 @@ require 'flare/tools/cluster'
 require 'flare/util/conversion'
 require 'flare/util/constant'
 require 'flare/tools/cli/sub_command'
+require 'flare/tools/cli/index_server_config'
 
 module Flare
   module Tools
     module Cli
-      
+
       class Down < SubCommand
         include Flare::Util::Conversion
         include Flare::Util::Constant
         include Flare::Tools::Common
-        
+        include Flare::Tools::Cli::IndexServerConfig
+
         myname :down
         desc   "turn down nodes and move them to proxy state."
         usage  "down [hostname:port] ..."
-        
-        def setup(opt)
-          opt.on('--force',            "commit changes without confirmation") {@force = true}
+
+        def setup
+          super
+          set_option_index_server
+          set_option_dry_run
+          set_option_force
         end
 
         def initialize
           super
-          @force = false
         end
 
-        def execute(config, *args)
+        def execute(config, args)
+          parse_index_server(config, args)
           return S_NG if args.size < 1
 
           hosts = args.map {|x| x.split(':')}
@@ -44,15 +49,15 @@ module Flare
               return S_NG
             end
           end
-          
-          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
             cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
-          
+
             hosts.each do |hostname,port|
               down = 'down'
               nodekey = nodekey_of hostname, port
               ipaddr = address_of_hostname(hostname)
-          
+
               unless cluster.has_nodekey? nodekey
                 error "invalid 'hostname:port' pair: #{nodekey}"
                 return S_NG
@@ -69,16 +74,16 @@ module Flare
                 exec = interruptible {(gets.chomp.upcase == "Y")}
               end
               if exec
-                s.set_state(hostname, port, down) unless config[:dry_run]
+                s.set_state(hostname, port, down) unless @dry_run
               end
             end
 
             puts string_of_nodelist(s.stats_nodes, hosts.map {|x| "#{x[0]}:#{x[1]}"})
           end
-          
+
           S_OK
         end # execute()
-        
+
       end
     end
   end

data/lib/flare/tools/cli/dump.rb

@@ -10,6 +10,7 @@ require 'flare/util/conversion'
 require 'flare/util/constant'
 require 'flare/util/bwlimit'
 require 'flare/tools/cli/sub_command'
+require 'flare/tools/cli/index_server_config'
 require 'csv'
 
 begin
@@ -20,7 +21,7 @@ end
 module Flare
   module Tools
     module Cli
-      
+
       class Dump < SubCommand
 
         class Dumper
@@ -105,19 +106,22 @@ module Flare
         include Flare::Util::Conversion
         include Flare::Util::Constant
         include Flare::Tools::Common
-        
+        include Flare::Tools::Cli::IndexServerConfig
+
         myname :dump
         desc   "dump data from nodes. (experimental)"
         usage  "dump [hostname:port] ..."
-        
-        def setup(opt)
-          opt.on('-o', '--output=FILE',            "output to file") {|v| @output = v}
-          opt.on('-f', '--format=FORMAT',          "specify output format [#{Formats.join(',')}]") {|v| @format = v}
-          opt.on(      '--bwlimit=BANDWIDTH',      "specify bandwidth limit (bps)") {|v|
+
+        def setup
+          super
+          set_option_index_server
+          @optp.on('-o', '--output=FILE',            "output to file") {|v| @output = v}
+          @optp.on('-f', '--format=FORMAT',          "specify output format [#{Formats.join(',')}]") {|v| @format = v}
+          @optp.on(      '--bwlimit=BANDWIDTH',      "specify bandwidth limit (bps)") {|v|
             @bwlimit = Flare::Util::Bwlimit.bps(v)
           }
-          opt.on('--all',                            "dump from all master nodes") {|v| @all = true}
-          opt.on('--raw',                            "raw dump mode (for debugging)") {|v| @raw = true}
+          @optp.on('--all',                            "dump from all master nodes") {|v| @all = true}
+          @optp.on('--raw',                            "raw dump mode (for debugging)") {|v| @raw = true}
         end
 
         def initialize
@@ -130,11 +134,12 @@ module Flare
           @partition_size = 1
         end
 
-        def execute(config, *args)
+        def execute(config, args)
+          parse_index_server(config, args)
           STDERR.puts "please install tokyocabinet via gem command." unless defined? TokyoCabinet
 
           cluster = nil
-          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
             cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
           end
           return S_NG if cluster.nil?
@@ -177,7 +182,7 @@ module Flare
               return S_NG
             end
           end
-          
+
           dumper = case @format
                    when CsvDumper.myname
                      CsvDumper.new(@output || STDOUT)
@@ -188,7 +193,7 @@ module Flare
                    end
 
           hosts.each do |hostname,port,partition|
-            Flare::Tools::Node.open(hostname, port.to_i, config[:timeout], 0, @bwlimit) do |n|
+            Flare::Tools::Node.open(hostname, port.to_i, @timeout, 0, @bwlimit) do |n|
               interval = 0
               part, partsize = if @raw
                                  [0, 1]
@@ -208,7 +213,7 @@ module Flare
           end
 
           dumper.close
-          
+
           S_OK
         end # execute()
 

data/lib/flare/tools/cli/dumpkey.rb

@@ -10,29 +10,33 @@ require 'flare/util/conversion'
 require 'flare/util/constant'
 require 'flare/util/bwlimit'
 require 'flare/tools/cli/sub_command'
+require 'flare/tools/cli/index_server_config'
 
 require 'csv'
 
 module Flare
   module Tools
     module Cli
-      
+
       class Dumpkey < SubCommand
         include Flare::Util::Conversion
         include Flare::Util::Constant
         include Flare::Tools::Common
-        
+        include Flare::Tools::Cli::IndexServerConfig
+
         myname :dumpkey
         desc   "dump key from nodes."
         usage  "dumpkey [hostname:port] ..."
-        
-        def setup(opt)
-          opt.on('-o', '--output=FILE',            "output to file"         ) {|v| @output = v}
-          opt.on('-f', '--format=FORMAT',          "output format [csv]"     ) {|v| @format = v}
-          opt.on('-p', '--partition=NUMBER',       "partition number"        ) {|v| @part = v.to_i if v.to_i >= 0}
-          opt.on('-s', '--partition-size=SIZE',    "partition size"          ) {|v| @partsize = v.to_i if v.to_i > 0}
-          opt.on(      '--bwlimit=BANDWIDTH',      "bandwidth limit (bps)"   ) {|v| @bwlimit = v if v.to_i > 0}
-          opt.on(      '--all',                      "dump form all partitions") {@all = true}
+
+        def setup
+          super
+          set_option_index_server
+          @optp.on('-o', '--output=FILE',            "output to file"         ) {|v| @output = v}
+          @optp.on('-f', '--format=FORMAT',          "output format [csv]"     ) {|v| @format = v}
+          @optp.on('-p', '--partition=NUMBER',       "partition number"        ) {|v| @part = v.to_i if v.to_i >= 0}
+          @optp.on('-s', '--partition-size=SIZE',    "partition size"          ) {|v| @partsize = v.to_i if v.to_i > 0}
+          @optp.on(      '--bwlimit=BANDWIDTH',      "bandwidth limit (bps)"   ) {|v| @bwlimit = v if v.to_i > 0}
+          @optp.on(      '--all',                      "dump form all partitions") {@all = true}
         end
 
         def initialize
@@ -45,9 +49,10 @@ module Flare
           @all = false
         end
 
-        def execute(config, *args)
+        def execute(config, args)
+          parse_index_server(config, args)
           cluster = nil
-          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
             cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
           end
           return S_NG if cluster.nil?
@@ -80,9 +85,9 @@ module Flare
               return S_NG
             end
           end
-          
+
           hosts.each do |hostname,port|
-            Flare::Tools::Node.open(hostname, port.to_i, config[:timeout], @bwlimit, @bwlimit) do |n|
+            Flare::Tools::Node.open(hostname, port.to_i, @timeout, @bwlimit, @bwlimit) do |n|
               output = STDOUT
               unless @output.nil?
                 output = File.open(@output, "w")
@@ -106,10 +111,10 @@ module Flare
               output.close if output != STDOUT
             end
           end
-          
+
           S_OK
         end # execute()
-        
+
       end
     end
   end