flare-tools 0.1.4 → 0.4.5.1

data/lib/flare/tools/cli/down.rb

@@ -0,0 +1,85 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/stats'
+require 'flare/tools/node'
+require 'flare/tools/index_server'
+require 'flare/tools/common'
+require 'flare/tools/cluster'
+require 'flare/util/conversion'
+require 'flare/util/constant'
+require 'flare/tools/cli/sub_command'
+
+module Flare
+  module Tools
+    module Cli
+      
+      class Down < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::Constant
+        include Flare::Tools::Common
+        
+        myname :down
+        desc   "turn down nodes and move them to proxy state."
+        usage  "down [hostname:port] ..."
+        
+        def setup(opt)
+          opt.on('--force',            "commit changes without confirmation") {@force = true}
+        end
+
+        def initialize
+          super
+          @force = false
+        end
+
+        def execute(config, *args)
+          return S_NG if args.size < 1
+
+          hosts = args.map {|x| x.split(':')}
+          hosts.each do |x|
+            if x.size != 2
+              puts "invalid argument '#{x.join(':')}'."
+              return S_NG
+            end
+          end
+          
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
+          
+            hosts.each do |hostname,port|
+              down = 'down'
+              nodekey = nodekey_of hostname, port
+              ipaddr = address_of_hostname(hostname)
+          
+              unless cluster.has_nodekey? nodekey
+                error "invalid 'hostname:port' pair: #{nodekey}"
+                return S_NG
+              end
+
+              node = cluster.node_stat(nodekey)
+
+              exec = @force
+              if exec
+              elsif node['state'] == down
+                puts "#{ipaddr}:#{port} is already down."
+              else
+                STDERR.print "turning node down (node=#{ipaddr}:#{port}, state=#{node['state']} -> #{down}) (y/n): "
+                exec = interruptible {(gets.chomp.upcase == "Y")}
+              end
+              if exec
+                s.set_state(hostname, port, down) unless config[:dry_run]
+              end
+            end
+
+            puts string_of_nodelist(s.stats_nodes, hosts.map {|x| "#{x[0]}:#{x[1]}"})
+          end
+          
+          S_OK
+        end # execute()
+        
+      end
+    end
+  end
+end

data/lib/flare/tools/cli/dump.rb

@@ -0,0 +1,219 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/node'
+require 'flare/tools/index_server'
+require 'flare/tools/common'
+require 'flare/util/conversion'
+require 'flare/util/constant'
+require 'flare/util/bwlimit'
+require 'flare/tools/cli/sub_command'
+require 'csv'
+
+begin
+  require 'tokyocabinet'
+rescue LoadError => e
+end
+
+module Flare
+  module Tools
+    module Cli
+      
+      class Dump < SubCommand
+
+        class Dumper
+          attr_reader :name
+          def write data, key, flag, len, version, expire
+            raise "internal error"
+          end
+          def close
+            raise "internal error"
+          end
+        end
+
+        class DefaultDumper < Dumper
+          def self.myname
+            "default"
+          end
+          def initialize filepath_or_writable
+            @output = if filepath_or_writable.kind_of?(String)
+                        open(filepath_or_writable, 'w') 
+                      else
+                        filepath_or_writable
+                      end
+          end
+          def write data, key, flag, len, version, expire
+            @output.puts "#{key} #{flag} #{len} #{version} #{expire} '#{data}'"
+          end
+          def close
+            @output.close unless @output == STDOUT || @output == STDERR
+          end
+        end
+
+        class CsvDumper < Dumper
+          def self.myname
+            "csv"
+          end
+          def initialize filepath_or_writable
+            @output = if filepath_or_writable.kind_of?(String)
+                        open(filepath_or_writable, 'w')
+                      else
+                        filepath_or_writable
+                      end
+            @output.puts "# key, flag, len, version, expire, data"
+            @writer = CSV::Writer.generate(@output, ',')
+          end
+          def write data, key, flag, len, version, expire
+            @writer << [key, flag, len, version, expire, data]
+          end
+          def close
+            @output.close unless @output == STDOUT || @output == STDERR
+          end
+        end
+
+        class TchDumper < Dumper
+          def self.myname
+            "tch"
+          end
+          def initialize filepath
+            raise "output file not specified." if filepath.nil?
+            raise "#{filepath} isn't a path." unless filepath.kind_of?(String)
+            @hdb = TokyoCabinet::HDB.new
+            @hdb.open(filepath, TokyoCabinet::HDB::OCREAT|TokyoCabinet::HDB::OWRITER)
+          end
+          def write data, key, flag, size, version, expire
+            # uint32_t flag -> L    // uint32_t
+            # time_t   expire -> Q  // unsigned long
+            # uint64_t size -> Q    // uint64_t
+            # uint64_t version -> Q // uint64_t
+            # uint32_t option -> L  // uint32_t
+            value = [flag, expire, size, version].pack("LQQQ")+data
+            @hdb.put(key, value)
+          end
+          def close
+            @hdb.close
+          end
+        end
+
+        Iterators = [DefaultDumper, CsvDumper]
+        Iterators << TchDumper if defined? TokyoCabinet
+        Formats = Iterators.map {|n| n.myname}
+        SizeOfByte = 8
+
+        include Flare::Util::Conversion
+        include Flare::Util::Constant
+        include Flare::Tools::Common
+        
+        myname :dump
+        desc   "dump data from nodes. (experimental)"
+        usage  "dump [hostname:port] ..."
+        
+        def setup(opt)
+          opt.on('-o', '--output=FILE',            "output to file") {|v| @output = v}
+          opt.on('-f', '--format=FORMAT',          "specify output format [#{Formats.join(',')}]") {|v| @format = v}
+          opt.on(      '--bwlimit=BANDWIDTH',      "specify bandwidth limit (bps)") {|v|
+            @bwlimit = Flare::Util::Bwlimit.bps(v)
+          }
+          opt.on('--all',                            "dump from all master nodes") {|v| @all = true}
+          opt.on('--raw',                            "raw dump mode (for debugging)") {|v| @raw = true}
+        end
+
+        def initialize
+          super
+          @output = nil
+          @format = nil
+          @bwlimit = 0
+          @all = false
+          @raw = false
+          @partition_size = 1
+        end
+
+        def execute(config, *args)
+          STDERR.puts "please install tokyocabinet via gem command." unless defined? TokyoCabinet
+
+          cluster = nil
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
+          end
+          return S_NG if cluster.nil?
+
+          partition_size = cluster.partition_size
+
+          if @all
+            if args.size > 0
+              STDERR.puts "don't specify any nodes with --all option."
+              return S_NG
+            else
+              args = cluster.master_nodekeys
+            end
+          else
+            if args.size == 0
+              STDERR.puts "please specify --all option to get complete dump."
+              return S_NG
+            end
+          end
+
+          unless Formats.include?(@format)
+            STDERR.puts "unknown format: #{@format}"
+            return S_NG
+          end
+
+          hosts = args.map {|x| x.split(':')}
+          hosts.each do |x|
+            if x.size == 2
+              x << cluster.partition_of_nodename("#{x[0]}:#{x[1]}")
+            elsif x.size == 4
+              if x[3] =~ /^\d+$/
+                STDERR.puts "invalid partition number '#{x.join(':')}'."
+                x[3] = x[3].to_i
+              else
+                STDERR.puts "invalid partition number '#{x.join(':')}'."
+                return S_NG
+              end
+            else
+              STDERR.puts "invalid argument '#{x.join(':')}'."
+              return S_NG
+            end
+          end
+          
+          dumper = case @format
+                   when CsvDumper.myname
+                     CsvDumper.new(@output || STDOUT)
+                   when TchDumper.myname
+                     TchDumper.new @output
+                   else
+                     DefaultDumper.new(@output || STDOUT)
+                   end
+
+          hosts.each do |hostname,port,partition|
+            Flare::Tools::Node.open(hostname, port.to_i, config[:timeout], 0, @bwlimit) do |n|
+              interval = 0
+              part, partsize = if @raw
+                                 [0, 1]
+                               else
+                                 [partition, partition_size]
+                               end
+              bwlimit = @bwlimit/1024/SizeOfByte
+              count = 0
+              STDERR.print "dumping from #{hostname}:#{port}::#{part} of #{partsize} partitions ... "
+              n.dump(interval, part, partsize, bwlimit) do |data, key, flag, len, version, expire|
+                dumper.write data, key, flag, len, version, expire
+                count += 1
+                false
+              end
+              STDERR.puts "#{count}"
+            end
+          end
+
+          dumper.close
+          
+          S_OK
+        end # execute()
+
+      end
+    end
+  end
+end
+

data/lib/flare/tools/cli/dumpkey.rb

@@ -0,0 +1,117 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/node'
+require 'flare/tools/index_server'
+require 'flare/tools/common'
+require 'flare/util/conversion'
+require 'flare/util/constant'
+require 'flare/util/bwlimit'
+require 'flare/tools/cli/sub_command'
+
+require 'csv'
+
+module Flare
+  module Tools
+    module Cli
+      
+      class Dumpkey < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::Constant
+        include Flare::Tools::Common
+        
+        myname :dumpkey
+        desc   "dump key from nodes."
+        usage  "dumpkey [hostname:port] ..."
+        
+        def setup(opt)
+          opt.on('-o', '--output=FILE',            "output to file"         ) {|v| @output = v}
+          opt.on('-f', '--format=FORMAT',          "output format [csv]"     ) {|v| @format = v}
+          opt.on('-p', '--partition=NUMBER',       "partition number"        ) {|v| @part = v.to_i if v.to_i >= 0}
+          opt.on('-s', '--partition-size=SIZE',    "partition size"          ) {|v| @partsize = v.to_i if v.to_i > 0}
+          opt.on(      '--bwlimit=BANDWIDTH',      "bandwidth limit (bps)"   ) {|v| @bwlimit = v if v.to_i > 0}
+          opt.on(      '--all',                      "dump form all partitions") {@all = true}
+        end
+
+        def initialize
+          super
+          @output = nil
+          @format = nil
+          @part = nil
+          @partsize = nil
+          @bwlimit = nil
+          @all = false
+        end
+
+        def execute(config, *args)
+          cluster = nil
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
+          end
+          return S_NG if cluster.nil?
+
+          if @all
+            unless args.empty?
+              STDERR.puts "don't specify any nodes with --all option."
+              return S_NG
+            else
+              args = cluster.master_nodekeys
+            end
+          else
+            if args.empty?
+              STDERR.puts "please specify --all option to get complete dump."
+              return S_NG
+            end
+          end
+
+          unless @format.nil?
+            unless ["csv"].include? @format
+              puts "unknown format: #{@format}"
+              return S_NG
+            end
+          end
+
+          hosts = args.map {|x| x.split(':')}
+          hosts.each do |x|
+            if x.size != 2
+              puts "invalid argument '#{x.join(':')}'."
+              return S_NG
+            end
+          end
+          
+          hosts.each do |hostname,port|
+            Flare::Tools::Node.open(hostname, port.to_i, config[:timeout], @bwlimit, @bwlimit) do |n|
+              output = STDOUT
+              unless @output.nil?
+                output = File.open(@output, "w")
+              end
+              case @format
+              when "csv"
+                writer = CSV::Writer.generate(output)
+                output.puts "# key"
+              end
+              interruptible {
+                n.dumpkey(@part, @partsize) do |key|
+                  case @format
+                  when "csv"
+                    writer << [key]
+                  else
+                    output.puts "#{key}"
+                  end
+                  false
+                end
+              }
+              output.close if output != STDOUT
+            end
+          end
+          
+          S_OK
+        end # execute()
+        
+      end
+    end
+  end
+end
+

data/lib/flare/tools/cli/flare_admin.rb

@@ -0,0 +1,81 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'optparse'
+require 'flare/util/logging'
+require 'flare/util/constant'
+require 'flare/tools'
+require 'flare/tools/cli'
+require 'flare/tools/cli/cli_util'
+
+require 'flare/util/command_line'
+
+Version = Flare::Tools::VERSION
+include Flare::Util::Logging
+include Flare::Util::Constant
+include Flare::Tools::Cli::CliUtil
+Cli = Flare::Tools::Cli
+
+index_server_hostname = nil
+index_server_port = nil
+timeout = DefaultTimeout
+dry_run = false
+cluster = nil
+scname = ''
+subc = nil
+
+scclasses = [Cli::List, Cli::Balance, Cli::Down, Cli::Slave, Cli::Reconstruct, Cli::Master, Cli::Threads, Cli::Ping, Cli::Remove, Cli::Index, Cli::Activate, Cli::Dump, Cli::Dumpkey, Cli::Verify, Cli::Stats, Cli::Restore, Cli::Summary]
+unsupported = [Cli::Deploy]
+scclasses.concat unsupported
+
+subcommands = Hash[*scclasses.map {|x| [x.to_sym, x]}.flatten]
+
+setup do |opt|
+  opt.banner = "#{Flare::Tools::TITLE}\nUsage: flare-admin [subcommand] [options] [arguments]"
+  opt.on("-n",           '--dry-run',                "dry run")                                                  {dry_run = true}
+  opt.on("-i HOSTNAME",  '--index-server=HOSTNAME',  "index server hostname(default:#{DefaultIndexServerName})") {|v| index_server_hostname = v}
+  opt.on("-p PORT",      '--index-server-port=PORT', "index server port(default:#{DefaultIndexServerPort})")     {|v| index_server_port = v.to_i}
+  opt.on(                '--log-file=LOGFILE',       "output log to LOGFILE")                                    {|v| Flare::Util::Logging.set_logger(v)}
+  opt.on(                '--cluster=NAME',           "specify a cluster name")                                   {|v| cluster = v}
+  opt.on(                '--timeout=SECOND',         "specify timeout")                                          {|v| timeout = v.to_i}
+  
+  preparsed = opt.order(ARGV)
+  scname = preparsed.shift.to_sym if preparsed.size > 0
+  
+  if subcommands.include?(scname)
+    subc = subcommands[scname].new 
+    opt.separator("#{scname} subcommand:")
+    subc.setup(opt)
+  else
+    error "unknown subcommand '#{scname}'" unless scname == ''
+    opt.separator("subcommands:")
+    puts opt.help
+    subcommands.each do |k,v|
+      next if unsupported.include?(v)
+      o = OptionParser.new
+      o.banner = "[#{k.to_s}] "+v.desc
+      o.separator("  Usage: flare-admin "+v.usage)
+      v.new.setup(o)
+      puts o.help
+    end
+    exit 1
+  end
+end
+
+status = execute do |args|
+  command = args.shift
+  ihostname, iport = get_index_server_from_nodekeys(args) ||
+    get_index_server_name_and_port(index_server_hostname, index_server_port)
+  ihostname, iport = get_index_server_from_cluster(cluster) unless cluster.nil?
+  subc.execute({ :command => command,
+                 :index_server_hostname => ihostname,
+                 :index_server_port => iport,
+                 :dry_run => dry_run,
+                 :timeout => timeout,
+                 :cluster => cluster },
+               *args) if subc
+end
+
+exit status