flare-tools 0.1.4 → 0.4.5.1

data/lib/flare/tools/cli/flare_argv0.rb

@@ -0,0 +1,60 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+cliname = File.basename($PROGRAM_NAME)
+cliname[/flare-/] = ""
+
+require 'resolv'
+require 'flare/tools'
+require 'flare/tools/cli/cli_util'
+require 'flare/util/logging'
+begin
+  require "flare/tools/cli/#{cliname}"
+rescue LoadError
+  exit 1
+end
+require 'flare/util/command_line.rb'
+require 'flare/util/conversion.rb'
+
+include Flare::Util::Logging
+include Flare::Util::Constant
+include Flare::Tools::Cli::CliUtil
+
+index_server_hostname = nil
+index_server_port = nil
+timeout = DefaultTimeout
+dry_run = false
+cluster = nil
+
+if ENV.has_key? "FLARE_INDEX_SERVER"
+  h, p = ENV["FLARE_INDEX_SERVER"].split(':')
+  index_server_hostname = h unless h.nil?
+  index_server_port = p unless p.nil?
+end
+
+subc = eval "Flare::Tools::Cli::#{cliname.capitalize}.new"
+
+setup do |opt|
+  opt.banner = "#{Flare::Tools::TITLE}\nUsage: flare-#{cliname} [options]"
+  opt.on('-n',          '--dry-run',                "dry run") {dry_run = true}
+  opt.on('-i HOSTNAME', '--index-server=HOSTNAME',  "index server hostname(default:#{DefaultIndexServerName})") {|v| index_server_hostname = v}
+  opt.on('-p PORT',     '--index-server-port=PORT', "index server port(default:#{DefaultIndexServerPort})") {|v| index_server_port = v.to_i}
+  opt.on(               '--log-file=LOGFILE',       "output log to LOGFILE") {|v| Flare::Util::Logging.set_logger(v)}
+  opt.on(               '--cluster=NAME',           "specify a cluster name") {|v| cluster = v}
+
+  subc.setup(opt)
+end
+
+execute do |args|
+  ihostname, iport = get_index_server_from_nodekeys(args) ||
+    get_index_server_name_and_port(index_server_hostname, index_server_port)
+  ihostname, iport = get_index_server_from_cluster(cluster) unless cluster.nil?
+  subc.execute({ :command => File.basename($PROGRAM_NAME),
+                 :index_server_hostname => ihostname,
+                 :index_server_port => iport,
+                 :dry_run => dry_run,
+                 :timeout => timeout },
+               *args)
+end

data/lib/flare/tools/cli/flare_keychecker.rb

@@ -0,0 +1,106 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'optparse'
+require "zlib"
+
+# Hash
+hashfunc = {
+  "simple" => lambda {|k| r = 0; k.each_byte {|c| r += c }; r },
+  "bitshift" => lambda {|k| r = 19790217; k.each_byte {|c| r = (r << 5) + (r << 2) + r + c }; r },
+  "crc32" => lambda {|k| Zlib.crc32(k, 0) },
+}
+
+# Variables
+delimiter = "::"
+use_prefix = false
+use_bucket = false
+prefixes = {}
+virtual = 4096
+size = 0
+buckets = {}
+bucket_file = nil
+partition_size = 100
+hint = 1
+hash = hashfunc["simple"]
+output_file = nil
+input_file = nil
+default_bucket_file = "bucket.csv"
+
+# Options
+opt = OptionParser.new
+opt.on('-d', '--delimiter=STRING',        "delimiter (defalut #{delimiter})") {|v| delimiter = v; use_prefix = true }
+opt.on('-b', '--bucket-file=CSVFILE',     "bucket file (default #{default_bucket_file})") {|v| bucket_file = v; }
+opt.on('-i', '--input=CSVFILE',           "input file") {|v| input_file = v; }
+opt.on('-o', '--output=CSVFILE',          "output file") {|v| output_file = v; }
+opt.on('-s', '--partition-size=SIZE',     "max partition size (default #{partition_size})") {|v| partition_size = v.to_i }
+opt.on('-h', '--hash=TYPE',               "hash function (simple,bitshift,crc32)") {|v|
+  default_bucket_file = "bucket.#{v}.csv"
+  hash = if hashfunc.has_key? v then hashfunc[v] else abort "invalid hash function." end
+}
+opt.on('-n', '--hint=SIZE',               "hint (default #{hint})") {|v| hint = v; }
+opt.parse!(ARGV)
+
+# Bucket
+if bucket_file.nil? || (!input_file.nil? && File.exist?(input_file))
+  bucket_file = default_bucket_file
+  input = if input_file.nil? then STDIN else open(input_file) end
+  while line = input.gets
+    key = line.chomp
+    parts = split(delimiter)
+    suffix = parts.pop
+    prefix = parts.join(delimiter)
+    unless prefixes.has_key? prefix
+      prefixes[prefix] = 0
+      buckets[prefix] = Array.new(virtual, 0)
+    end
+    prefixes[prefix] += 1
+    key_hash_value = [hash.call(key)].pack("l").unpack("L")[0]
+    key_hash_value = -key_hash_value if key_hash_value < 0
+    buckets[prefix][key_hash_value%virtual] += 1
+    size += 1
+  end
+  if File.exist? bucket_file
+    print "#{bucket_file} exists. overwrite? (y/n): "
+    exit unless gets.chomp.upcase == "Y"
+  end
+  open(bucket_file, 'w') do |f|
+    f.puts "# key, total, ..."
+    prefixes.each do |k,total|
+      f.puts k+", "+total.to_s+", "+buckets[k].join(" ,")
+    end
+  end
+else
+  open(bucket_file) do |f|
+    while line = f.gets
+      prefix, total, data = line.split(',', 3)
+      prefixes[prefix] = total.to_i
+      buckets[prefix] = data.split(',').map {|x| x.to_i}
+    end
+  end
+end
+
+# Assign
+map = Array.new(partition_size+1).map!{Array.new(virtual, 0)}
+(1..partition_size).each do |i|
+  counter = Array.new(partition_size, 0)
+  (0..hint).each {|j| map[i][j] = j % i }
+  ((hint+1)...virtual).each do |j|
+    k = map[i-1][j]
+    counter[k] += 1
+    map[i][j] = if (counter[k] % i) == (i - 1) then i - 1 else map[i-1][j] end
+  end
+end
+
+# Result
+output = if output_file.nil? || !File.exist?(output_file) then STDOUT else open(output_file, "w") end
+output.puts "partition size, prefix, "+(0...partition_size).map{|x| "#{x}"}.join(' ,')
+for i in 1..partition_size
+  prefixes.each do |prefix, total|
+    nitem = Array.new(i, 0)
+    (0...virtual).each {|j| nitem[map[i][j]] += buckets[prefix][j] }
+    output.puts "#{i}, #{prefix},"+nitem.join(' ,')
+  end
+end

data/lib/flare/tools/cli/flare_zkadmin.rb

@@ -0,0 +1,226 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'rubygems'
+require 'optparse'
+require 'uri'
+require 'zookeeper'
+require 'flare/tools/cluster'
+
+uri = "zookeeper://localhost:2181/ckvs/clusters/mycluster"
+path = nil
+indexdb_elements = []
+
+ZOK = Zookeeper::ZOK
+
+def init z, path
+  puts "initializing #{path}"
+  path_cluster = ""
+  path.split('/').each do |e|
+    unless e.empty?
+      path_cluster += "/#{e}"
+      z.create(:path => path_cluster)
+    end
+  end
+  
+  path_index = "#{path_cluster}/index"
+  r = z.create(:path => "#{path_index}")
+  raise "already initialized." unless r[:rc] == ZOK
+
+  z.create(:path => "#{path_index}/lock")
+  z.create(:path => "#{path_index}/primary")
+  z.create(:path => "#{path_index}/servers")
+  z.create(:path => "#{path_index}/nodemap")
+
+  path_meta = "#{path_index}/meta"
+  z.create(:path => path_meta)
+  entries = [['partition-size', '1024'],
+             ['key-hash-algorithm', 'crc32'],
+             ['partition-type', 'modular'],
+             ['partition-modular-hint', '1'],
+             ['partition-modular-virtual', '65536']]
+  entries.each do |k,v|
+    r = z.create(:path => "#{path_meta}/#{k}", :data => v)
+  end
+
+  clear_nodemap z, path
+end
+
+def destroy z, path
+  result = z.get_children(:path => path)
+  raise "failed to fetch child nodes." if result[:rc] != ZOK
+  result[:children].each do |entry|
+    destroy z, "#{path}/#{entry}"
+  end
+  z.delete(:path => path)
+end
+
+def nodemap z, path
+  result = z.get(:path => "#{path}/index/nodemap")
+  if result[:rc] == ZOK
+    xml = result[:data]
+    unless xml.nil?
+      cluster = Flare::Tools::Cluster.build xml
+      print cluster.serialize
+    end
+  end
+end
+
+def set_nodemap z, path
+  path_nodemap = "#{path}/index/nodemap"
+  xml = ""
+  while line = STDIN.gets
+    xml += line
+  end
+  result = z.set(:path => path_nodemap, :data => xml)
+  rc = result[:rc]
+  raise "failed to set nodemap (#{rc})" if rc != ZOK
+end
+
+def clear_nodemap z, path
+  path_nodemap = "#{path}/index/nodemap"
+  xml = <<EOS
+<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+<!DOCTYPE boost_serialization>
+<boost_serialization signature="serialization::archive" version="4">
+<node_map class_id="0" tracking_level="0" version="0">
+	<count>0</count>
+	<item_version>0</item_version>
+</node_map>
+<thread_type>16</thread_type>
+</boost_serialization>
+EOS
+  STDOUT.print "do you really want to clear nodemap? (y/n):"
+  STDOUT.flush
+  if gets.chomp.upcase == 'Y'
+    result = z.set(:path => path_nodemap, :data => xml)
+    rc = result[:rc]
+    raise "failed to clear nodemap (#{rc})" if rc != ZOK
+  end
+end
+
+def servers z, path
+  result = z.get(:path => "#{path}/index/servers")
+  if result[:rc] == ZOK
+    puts result[:data].split(',').join(' ')
+  end
+end
+
+def set_servers z, path, *args
+  path_servers = "#{path}/index/servers"
+  result = z.set(:path => path_servers, :data => args.join(','))
+  rc = result[:rc]
+  raise "failed to set nodemap (#{rc})" if rc != ZOK
+end
+
+def show z, path
+  path_index = "#{path}/index"
+  path_lock = "#{path_index}/lock"
+  path_servers = "#{path_index}/servers"
+  path_nodemap = "#{path_index}/nodemap"
+  path_primary = "#{path_index}/primary"
+  path_meta = "#{path_index}/meta"
+  result = z.get_children(:path => path_index)
+  raise "failed to fetch child nodes." if result[:rc] != ZOK
+  result[:children].each do |entry|
+    puts "#{entry}:"
+    case entry
+    when "lock"
+      r = z.get_children(:path => path_lock)
+      if r[:rc] == ZOK
+        r[:children].sort_by {|n| n.split('-').last}.each do |m|
+          r2 = z.get(:path => "#{path_lock}/#{m}")
+          if r2[:rc] == ZOK
+            puts "\t#{m} #{r2[:data]}"
+          else
+            puts "\t#{m}"
+          end
+        end
+      end
+    when "primary"
+      r = z.get(:path => path_primary)
+      if r[:rc] == ZOK && !r[:data].nil?
+        puts "\t#{r[:data]}"
+      end
+    when "servers"
+      r = z.get(:path => path_servers)
+      puts "\t#{r[:data]}" if r[:rc] == ZOK && !r[:data].nil?
+    when "nodemap"
+      role_to_s = ["master", "slave", "proxy"]
+      state_to_s = ["active", "prepare", "down", "ready"]
+      r = z.get(:path => path_nodemap)
+      if r[:rc] == ZOK
+        xml = r[:data]
+        unless xml.nil?
+          cluster = Flare::Tools::Cluster.build xml
+          cluster.nodekeys.each do |nodekey|
+            n = cluster.node_stat(nodekey)
+            p = if n.partition == -1 then "-" else n.partition end
+            m = "#{n.server_name}:#{n.server_port}:#{n.balance}:#{p}"
+            puts "\t#{m} #{role_to_s[n.role.to_i]} #{state_to_s[n.state.to_i]} #{n.thread_type}"
+          end
+        end
+      end
+    when "meta"
+      r = z.get_children(:path => path_meta)
+      if r[:rc] == ZOK
+        r[:children].each do |m|
+          r2 = z.get(:path => "#{path_meta}/#{m}")
+          puts "\t#{m} #{r2[:data]}" if r2[:rc] == ZOK
+        end
+      end
+    else
+      puts "\tunknown entry"
+    end
+  end
+end
+
+def execute(subc, args, options)
+  scheme, userinfo, host, port, registry, path, opaque, query, flagment = URI.split(options[:indexdb])
+  # p scheme, userinfo, host, port, registry, path, opaque, query, flagment
+
+  z = case scheme
+      when "zookeeper"
+        Zookeeper.new("#{host}:#{port}")
+      else
+        raise "invalid scheme: #{scheme}"
+      end
+  
+  case subc
+  when "set-servers"
+    set_servers z, path, *args
+  when "servers"
+    servers z, path
+  when "set-nodemap"
+    set_nodemap z, path
+  when "clear-nodemap"
+    clear_nodemap z, path
+  when "nodemap"
+    nodemap z, path
+  when "init"
+    init z, path
+  when "destroy"
+    destroy z, path
+  when "show"
+    show z, path
+  end
+  z.close
+end
+
+options = {
+  :indexdb => 'zookeeper://localhost:2181/ckvs/clusters/mycluster'
+}
+
+opt = OptionParser.new
+opt.on('-i URI', '--index-db=URI') {|v|
+  options[:indexdb] = v
+}
+
+opt.parse!(ARGV)
+
+subc = ARGV.shift
+args = ARGV.dup
+
+execute(subc, args, options)

data/lib/flare/tools/cli/index.rb

@@ -0,0 +1,54 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'flare/tools/stats'
+require 'flare/tools/common'
+require 'flare/tools/cluster'
+require 'flare/tools/cli/sub_command'
+require 'flare/util/conversion'
+
+module Flare
+  module Tools
+    module Cli
+      class Index < SubCommand
+        include Flare::Util::Conversion
+        
+        myname :index
+        desc   "print the index XML document from a cluster information."
+        usage  "index"
+
+        def setup(opt)
+          opt.on('--indexdb=URI',            "index database"        ) {|v| @indexdb = v}
+          opt.on('--output=FILE',            "output index to a file") {|v| @output = v}
+        end
+
+        def initialize
+          super
+          @output = nil
+          @indexdb = nil
+        end
+
+        def execute(config, *args)
+          cluster = Flare::Tools::Stats.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
+            nodes = s.stats_nodes.sort_by{|key, val| [val['partition'], val['role'], key]}
+            Flare::Tools::Cluster.new(s.host, s.port, s.stats_nodes)
+          end
+
+          output = cluster.serialize
+          if @output.nil?
+            info output
+          else
+            open(@output, "w") do |f|
+              f.write(output)
+            end
+          end
+
+          S_OK
+        end
+      end
+    end
+  end
+end
+