flare-tools 0.1.4 → 0.4.5.1

data/lib/flare/test/node.rb

@@ -0,0 +1,62 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'rubygems'
+require 'flare/tools/node'
+
+# 
+module Flare
+  module Test
+
+    # == Description
+    #
+    class Node
+      def initialize(hostname_port, pid)
+        @hostname_port = hostname_port
+        @hostname, @port = hostname_port.split(':')
+        @pid = pid
+        @alive = true
+      end
+
+      def open(&block)
+        return nil unless @alive
+        node = Flare::Tools::Node.open(@hostname, @port)
+        return node if block.nil?
+        ret = nil
+        begin
+          ret = block.call(node)
+        rescue => e
+          node.close
+          raise e
+        end
+        ret
+      end
+
+      def stop
+        Process.kill :STOP, @pid
+      end
+
+      def cont
+        Process.kill :CONT, @pid
+      end
+
+      def terminate
+        puts "killing... #{@pid}"
+        begin
+          timeout(10) do
+            Process.kill :TERM, @pid
+            Process.waitpid @pid
+          end
+        rescue TimeoutError => e
+          Process.kill :KILL, @pid
+          Process.waitpid @pid
+        end
+        @alive = false
+      end
+
+      attr_reader :hostname, :port, :hostname_port
+    end
+  end
+end

data/lib/flare/tools.rb

@@ -1,20 +1,22 @@
 # -*- coding: utf-8; -*-
-# Author::    kgws  (http://d.hatena.ne.jp/kgws/)
-# Copyright:: Copyright (c) 2010- kgws.
-# License::   This program is licenced under the same licence as kgws.
-#
-# $--- flare-tools - [ by Ruby ] $
-# vim: foldmethod=marker tabstop=2 shiftwidth=2
-$:.unshift(File.dirname(__FILE__)) unless
-  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
 
-require 'optparse'
-require 'socket'
-require 'timeout'
-require 'resolv'
-require 'tools/core'
-require 'tools/logger'
+# 
+module Flare
 
-module FlareTools
-  VERSION = '0.1.4'
+  # flare-tools module.
+  module Tools
+    # the version number of flare-tools
+    VERSION = '0.4.5.1'
+    TITLE = "Flare-tools version #{VERSION} Copyright (C) GREE, Inc. 2011"
+    autoload :Common,      'flare/tools/common'
+    autoload :Cluster,     'flare/tools/cluster'
+    autoload :Stats,       'flare/tools/stats'
+    autoload :Node,        'flare/tools/node'
+    autoload :IndexServer, 'flare/tools/index_server'
+    autoload :Cli,         'flare/tools/cli'
+    autoload :ZkUtil,      'flare/tools/zk_util'
+  end
 end

data/lib/flare/tools/cli.rb

@@ -0,0 +1,32 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+# 
+module Flare
+  module Tools
+    module Cli
+      autoload :List,        'flare/tools/cli/list'
+      autoload :Stats,       'flare/tools/cli/stats'
+      autoload :Index,       'flare/tools/cli/index'
+      autoload :Balance,     'flare/tools/cli/balance'
+      autoload :Down,        'flare/tools/cli/down'
+      autoload :Slave,       'flare/tools/cli/slave'
+      autoload :Reconstruct, 'flare/tools/cli/reconstruct'
+      autoload :Master,      'flare/tools/cli/master'
+      autoload :Deploy,      'flare/tools/cli/deploy'
+      autoload :Threads,     'flare/tools/cli/threads'
+      autoload :Ping,        'flare/tools/cli/ping'
+      autoload :Remove,      'flare/tools/cli/remove'
+      autoload :Activate,    'flare/tools/cli/activate'
+      autoload :Dump,        'flare/tools/cli/dump'
+      autoload :Dumpkey,     'flare/tools/cli/dumpkey'
+      autoload :Verify,      'flare/tools/cli/verify'
+      autoload :Restore,     'flare/tools/cli/restore'
+      autoload :Summary,     'flare/tools/cli/summary'
+    end
+  end
+end
+
+

data/lib/flare/tools/cli/activate.rb

@@ -0,0 +1,106 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/stats'
+require 'flare/tools/node'
+require 'flare/tools/index_server'
+require 'flare/tools/common'
+require 'flare/tools/cluster'
+require 'flare/util/conversion'
+require 'flare/util/constant'
+require 'flare/tools/cli/sub_command'
+
+module Flare
+  module Tools
+    module Cli
+      
+      class Activate < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::Constant
+        include Flare::Util::Logging
+        include Flare::Tools::Common
+        
+        myname :activate
+        desc   "activate "
+        usage  "activate [hostname:port] ..."
+        
+        def setup(opt)
+          opt.on('--force',            "commit changes without confirmation") {@force = true}
+        end
+
+        def initialize
+          super
+          @force = false
+        end
+
+        def execute(config, *args)
+          return S_NG if args.size < 1
+
+          hosts = args.map {|x| x.split(':')}
+          hosts.each do |x|
+            if x.size != 2
+              puts "invalid argument '#{x.join(':')}'."
+              return S_NG
+            end
+          end
+          
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
+            nodes = s.stats_nodes.sort_by{|key, val| [val['partition'], val['role'], key]}
+          
+            hosts.each do |hostname,port|
+              nodekey = nodekey_of hostname, port
+              ipaddr = address_of_hostname(hostname)
+          
+              unless cluster.has_nodekey? nodekey
+                error "invalid 'hostname:port' pair: #{nodekey}"
+                return S_NG
+              end
+
+              node = cluster.node_stat(nodekey)
+
+              exec = @force
+              if exec
+              elsif node['state'] == 'active'
+                warn "#{ipaddr}:#{port} is already active."
+              else
+                STDERR.print "turning node up (node=#{ipaddr}:#{port}, state=#{node['state']} -> activate) (y/n): "
+                exec = interruptible {
+                  (gets.chomp.upcase == "Y")
+                }
+              end
+              if exec && !config[:dry_run]
+                if @force
+                  begin
+                    s.set_state(hostname, port, 'active')
+                  rescue Timeout::Error => e
+                    error "failed to activate #{nodekey} (timeout)"
+                    raise e
+                  end
+                else
+                  resp = false
+                  until resp
+                    resp = s.set_state(hostname, port, 'active')
+                    unless resp
+                      STDERR.print "turning node up (node=#{ipaddr}:#{port}, state=#{node['state']} -> activate) (y/n): "
+                      exec = interruptible {
+                        (gets.chomp.upcase == "Y")
+                      }
+                    end
+                  end
+                end
+              end
+            end
+
+            STDOUT.puts string_of_nodelist(s.stats_nodes, hosts.map {|x| "#{x[0]}:#{x[1]}"})
+          end
+          
+          S_OK
+        end # execute()
+        
+      end
+    end
+  end
+end

data/lib/flare/tools/cli/balance.rb

@@ -0,0 +1,83 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/stats'
+require 'flare/tools/index_server'
+require 'flare/tools/common'
+require 'flare/util/conversion'
+require 'flare/util/constant'
+require 'flare/tools/cli/sub_command'
+
+module Flare
+  module Tools
+    module Cli
+      class Balance < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::Constant
+        include Flare::Tools::Common
+
+        myname :balance
+        desc   "set the balance values of nodes."
+        usage  "balance [hostname:port:balance] ..."
+
+        def setup(opt)
+          opt.on('--force',            "commit changes without confirmation") {@force = true}
+        end
+
+        def initialize
+          @force = false
+        end
+  
+        def execute(config, *args)
+          return S_NG if args.empty?
+
+          hosts = args.map {|x| x.to_s.split(':')}
+          hosts.each do |x|
+            if x.size != 3
+              puts "invalid argument '#{x.join(':')}'."
+              return S_NG
+            end
+          end
+          
+          Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            cluster = Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
+
+            hosts.each do |hostname,port,balance|
+              balance = balance.to_i
+              nodekey = nodekey_of hostname, port
+              ipaddr = address_of_hostname(hostname)
+          
+              unless cluster.has_nodekey? nodekey
+                error "unknown host: #{nodekey}"
+                return S_NG
+              end
+
+              node = cluster.node_stat(nodekey)
+
+              exec = false
+              if @force
+                exec = true
+              elsif node['balance'].to_i == balance
+                STDERR.puts "no need to change the balance of #{ipaddr}:#{port}."
+              else
+                interruptible do
+                  STDERR.print "updating node balance (node=#{ipaddr}:#{port}, balance=#{node['balance']} -> #{balance}) (y/n): "
+                  exec = true if gets.chomp.upcase == "Y"
+                end
+              end
+              if exec
+                s.set_role(hostname, port.to_i, node['role'], balance, node['partition']) unless config[:dry_run]
+              end
+            end
+            STDOUT.puts string_of_nodelist(s.stats_nodes, hosts.map {|x| "#{x[0]}:#{x[1]}"})
+          end
+
+          return S_OK
+        end # execute()
+
+      end
+    end
+  end
+end

data/lib/flare/tools/cli/cli_util.rb

@@ -0,0 +1,77 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/index_server'
+
+module Flare
+  module Tools
+    module Cli
+      module CliUtil
+        FLARE_INDEX_SERVERS = "FLARE_INDEX_SERVERS"
+        FLARE_INDEX_SERVER = "FLARE_INDEX_SERVER"
+
+        def get_index_server_from_cluster(name, envname = FLARE_INDEX_SERVERS)
+          return nil if name.nil?
+          if ENV.has_key? envname
+            clusters = ENV[envname].split(';').map{|s| s.split(':')}
+            clusters.each do |cluster_name,index_name,index_port|
+              return [index_name, index_port] if cluster_name == name
+            end
+          end
+          return nil
+        end
+
+        def get_index_server_from_nodekeys(dnodekeys, envname = FLARE_INDEX_SERVERS)
+          nodekeys = []
+          return nil if dnodekeys.empty?
+          dnodekeys.each do |n|
+            l = n.split(':')
+            return nil if l.size < 2
+            nodekeys << "#{l[0]}:#{l[1]}"
+          end
+          if ENV.has_key? envname
+            clusters = ENV[envname].split(';').map{|s| s.split(':')}
+            clusters.each do |cluster_name,index_name,index_port|
+              ret = Flare::Tools::IndexServer.open(index_name, index_port.to_i) do |s|
+                cluster_nodekeys = s.stats_nodes.map {|x| x[0]}
+                included = true
+                nodekeys.each do |nodekey|
+                  included = false unless cluster_nodekeys.include? nodekey
+                end
+                if included
+                  [index_name, index_port]
+                else
+                  nil
+                end
+              end
+              return ret unless ret.nil?
+            end
+          end
+          nil
+        rescue => e
+          nil
+        end
+
+        def get_index_server_name_and_port(index_server_hostname, index_server_port, envname = FLARE_INDEX_SERVER)
+          env_ihostname = nil
+          env_iport = nil
+          if ENV.has_key? envname
+            env_ihostname, env_iport = ENV[envname].split(':')
+          end
+          ihostname, iport = index_server_hostname.split(':') unless index_server_hostname.nil?
+          ihostname = ihostname || env_ihostname || DefaultIndexServerName
+          if iport && index_server_port
+            raise "--index-server-port option isn't allowed."
+          else
+            iport = index_server_port || env_iport || DefaultIndexServerPort if iport.nil?
+          end
+          [ihostname, iport]
+        end
+
+      end
+    end
+  end
+end
+

data/lib/flare/tools/cli/deploy.rb

@@ -0,0 +1,170 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/util'
+require 'flare/tools/index_server'
+require 'flare/tools/cli/sub_command'
+
+# 
+module Flare
+  module Tools
+    module Cli
+
+      # == Description
+      # 
+      class Deploy < SubCommand
+        include Flare::Util::Conversion
+        include Flare::Util::FileSystem
+
+        myname :deploy
+        desc "deploy for Debian-based system."
+        usage "deploy [hostname:port:balance:partition] ..."
+  
+        def setup(opt)
+          opt.on('--deploy-index',                        "deploy the index server") {@deploy_index = true}
+          opt.on('--delete',                              "delete existing contents before deploying") {@delete = true}
+          opt.on('--flarei=PATH',                       "a path for flarei executable") {|v| @flarei = v}
+          opt.on('--flared=PATH',                       "a path for flared executable") {|v| @flared = v}
+          opt.separator('flarei/flared options:')
+          opt.on('--back-log=BACKLOG',                  "back log parameter for listen()"    ) {|v| @iconf["back-log"] = v}
+          opt.on('--thread-pool-size=SIZE',             "thread pool size"                   ) {|v| @iconf["thread_pool_size"] = @dconf["thread_pool_size"] = v}
+          opt.on('--daemonize=BOOL',                    "daemonize"                          ) {|v| @iconf["daemonize"] = @dconf["daemonize"] = v}
+          opt.on('--data-dir=PATH',                     "data directory"                     ) {|v| @iconf["data-dir"] = @dconf["data-dir"] = v}
+          opt.on('--log-facility=NAME',                 "log facility"                       ) {|v| @iconf["log-facility"] = @dconf["log-facility"] = v}
+          opt.on('--max-connection=SIZE',               "max connection"                     ) {|v| @iconf["max-connection"] = @dconf["max-connection"] = v}
+          opt.on('--net-read-timeout=MILLISECOND',      "read timeout for server connections") {|v| @iconf["net-read-timeout"] = @dconf["net-read-timeout"] = v}
+          opt.on('--stack-size=KB',                     "stack size"                         ) {|v| @iconf["stack-size"] = @dconf["stack-size"] = v}
+          opt.separator('flarei options:')
+          opt.on('--monitor-threshold=COUNT',           "monitor threshold"                  ) {|v| @iconf["monitor-threshold"] = v}
+          opt.on('--monitor-interval=SECOND',           "monitor interval"                   ) {|v| @iconf["monitor-interval"] = v}
+          opt.on('--monitor-read-timeout=MILLISECOND',  "monitor read timeout in millisecond") {|v| @iconf["monitor-read-timeout"] = v}
+          opt.on('--partition-type=NAME',               "partition type(modular)"            ) {|v| @iconf["partition-type"] = v}
+          opt.on('--key-hash-algorithm=NAME'            "hash algorithm for key distribution") {|v| @iconf["key-hash-algorithm"] = v}
+          opt.separator('flared options:')
+          opt.on('--proxy-concurrency=SIZE',            "proxy concurrency"                  ) {|v| @dconf["proxy-concurrency"] = v}
+          opt.on('--mutex-slot=SIZE',                   "mutex slot size"                    ) {|v| @dconf["mutex-slot"]= v}
+          opt.on('--reconstruction-interval=MICROSEC',  "reconstruction interval"            ) {|v| @dconf["reconstruction-interval"] = v}
+          opt.on('--reconstruction-bwlimit=BYTES',      "reconstruction bandwitdh limit"     ) {|v| @dconf["reconstruction-bwlimit"] = v}
+          opt.on('--storage-ap=SIZE',                   "alignment power"                    ) {|v| @dconf["storage-ap"] = v}
+          opt.on('--storage-bucket-size=SIZE',          "storage bucket size"                ) {|v| @dconf["storage-bucket-size"] = v}
+          opt.on('--storage-cache-size=SIZE',           "storage cache size"                 ) {|v| @dconf["storage-cache-size"] = v}
+          opt.on('--storage-compress=NAME',             "storage compress type(deflate|bz2|tcbs)") {|v| @dconf["storage-compress"] = v}
+          opt.on('--storage-large',                       "strage large"                       ) {@dconf["storage-large"] = true}
+          opt.on('--storage-type=NAME',                 "storage type"                       ) {|v| @dconf["storage-type"] = v}
+          opt.on('--proxy-prior-netmask=SIZE',          "proxy priority mask (ex. 24 for 255.255.255.0)") {|v| @dconf["proxy-prior-netmask"] = v}
+          opt.on('--max-total-thread-queue=SIZE',       "max total thread queue"             ) {|v| @dconf["max-total-thread-queue"] = v}
+          opt.on('--index-servers=NAME',                "index servers"                      ) {|v| @dconf["index-servers"] = v}
+          opt.on('--noreply-window-limit=SIZE',         "noreply window limit (experimental)") {|v| @dconf["noreply-window-limit"] = v}
+          opt.on('--mysql-replication',                   "MySQL replication (experimental)"   ) {@dconf["mysql-replication"] = true}
+          opt.on('--mysql-replication-port=PORT',       "MySQL replication port (experimental)") {|v| @dconf["mysql-replication-port"] = v}
+          opt.on('--mysql-replication-id=ID',           "MySQL replication ID (experimental)") {|v| @dconf["mysql-replication-id"] = v}
+          opt.on('--mysql-replication-db=NAME',         "MySQL replication DB (experimental)") {|v| @dconf["mysql-replication-db"] = v}
+          opt.on('--mysql-replication-table=NAME',      "MySQL replication table (experimental)") {|v| @dconf["mysql-replication-table"] = v}
+          opt.on('--proxy-cache-size=SIZE',             "proxy cache entry size (experimental)") {|v| @dconf["proxy-cache-size"] = v}
+          opt.on('--proxy-cache-expire=SECOND',         "cache life-time in second (experimental)") {|v| @dconf["proxy-cache-expire"] = v}
+
+        end
+
+        def initialize
+          @deploy_index = false
+          @delete = false
+          @flarei = "/usr/local/bin/flarei"
+          @flared = "/usr/local/bin/flared"
+          @iconf = {}
+          @dconf = {}
+        end
+
+        def output_scripts(basedir, datadir, name, exec)
+          starting = "/sbin/start-stop-daemon --start --quiet --pidfile #{datadir}/#{name}.pid --exec #{exec}"
+          starting += " -- -f #{basedir}/#{name}.conf --daemonize"
+          stopping = "/sbin/start-stop-daemon --stop --retry=TERM/30/KILL/5 --quiet --pidfile #{datadir}/#{name}.pid --exec #{exec}"
+
+          start = basedir+"/start.sh"
+          open(start, "w") do |f|
+            f.puts "#!/bin/sh"
+            f.puts starting
+          end
+          File.chmod(0744, start)
+          
+          stop = basedir+"/stop.sh"
+          open(stop, "w") do |f|
+            f.puts "#!/bin/sh"
+            f.puts stopping
+          end
+          File.chmod(0744, stop)
+          
+          restart = basedir+"/restart.sh"
+          open(restart, "w") do |f|
+            f.puts "#!/bin/sh"
+            f.puts stopping
+            f.puts "sleep 5"
+            f.puts starting
+          end
+          File.chmod(0744, restart)
+        end
+
+        def execute(config, *args)
+          if @deploy_index
+            hostname = config[:index_server_hostname]
+            port = config[:index_server_port]
+            hostname_port = "#{hostname}:#{port}"
+            basedir = Dir.pwd+"/"+hostname_port
+            datadir = basedir+"/data"
+
+            delete_all(basedir) if @delete && FileTest.exist?(basedir)
+            if FileTest.exist?(basedir)
+              warn "directory already exists: #{basedir}"
+            else
+              Dir.mkdir(basedir)
+            end
+
+            conf = Flare::Util::FlareiConf.new({
+                                                 'server-name' => hostname,
+                                                 'server-port' => port,
+                                                 'data-dir' => datadir,
+                                               }.merge(@iconf))
+            open(basedir+"/flarei.conf", "w") {|f| f.puts conf}
+            Dir.mkdir(datadir) unless FileTest.exist?(datadir)
+            output_scripts(basedir, datadir, "flarei", @flarei)
+          end
+
+          args.each do |host|
+            hostname, port, balance, partition = host.split(':', 4)
+            hostname_port = "#{hostname}:#{port}"
+            basedir = Dir.pwd+"/"+hostname_port
+            datadir = basedir+"/data"
+
+            info "generateing ... #{hostname_port}"
+            delete_all(basedir) if @delete && FileTest.exist?(basedir)
+            if FileTest.exist?(basedir)
+              warn "directory already exists: #{basedir}"
+            else
+              Dir.mkdir(basedir)
+            end
+
+            if @dconf.has_key?("index-servers")
+              @dconf['index-server-name'] = nil
+              @dconf['index-server-port'] = nil
+            end
+
+            conf = Flare::Util::FlaredConf.new({
+                                                 'index-server-name' => config[:index_server_hostname],
+                                                 'index-server-port' => config[:index_server_port],
+                                                 'server-name' => hostname,
+                                                 'server-port' => port,
+                                                 'data-dir' => datadir,
+                                               }.merge(@dconf))
+            open(basedir+"/flared.conf", "w") {|f| f.puts conf}
+            Dir.mkdir(datadir) unless FileTest.exist?(datadir)
+            output_scripts(basedir, datadir, "flared", @flared)
+          end
+
+          return 0
+        end
+        
+      end
+    end
+  end
+end