flare-tools 0.4.5.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. data/.gitignore +24 -0
  2. data/.travis.yml +15 -0
  3. data/Gemfile +9 -0
  4. data/Gemfile.lock +28 -0
  5. data/History.txt +5 -0
  6. data/Makefile +25 -0
  7. data/README.txt +2 -2
  8. data/Rakefile +36 -34
  9. data/flare-tools.gemspec +26 -0
  10. data/lib/flare/entity/server.rb +4 -0
  11. data/lib/flare/tools.rb +3 -3
  12. data/lib/flare/tools/cli.rb +1 -0
  13. data/lib/flare/tools/cli/activate.rb +19 -13
  14. data/lib/flare/tools/cli/balance.rb +15 -8
  15. data/lib/flare/tools/cli/deploy.rb +53 -49
  16. data/lib/flare/tools/cli/dispatch.rb +101 -0
  17. data/lib/flare/tools/cli/down.rb +19 -14
  18. data/lib/flare/tools/cli/dump.rb +19 -14
  19. data/lib/flare/tools/cli/dumpkey.rb +21 -16
  20. data/lib/flare/tools/cli/flare_admin.rb +16 -72
  21. data/lib/flare/tools/cli/flare_argv0.rb +7 -51
  22. data/lib/flare/tools/cli/index.rb +10 -7
  23. data/lib/flare/tools/cli/index_server_config.rb +94 -0
  24. data/lib/flare/tools/cli/list.rb +14 -12
  25. data/lib/flare/tools/cli/master.rb +18 -12
  26. data/lib/flare/tools/cli/option.rb +55 -0
  27. data/lib/flare/tools/cli/part.rb +13 -12
  28. data/lib/flare/tools/cli/ping.rb +11 -13
  29. data/lib/flare/tools/cli/reconstruct.rb +22 -15
  30. data/lib/flare/tools/cli/remove.rb +20 -17
  31. data/lib/flare/tools/cli/restore.rb +20 -20
  32. data/lib/flare/tools/cli/slave.rb +20 -14
  33. data/lib/flare/tools/cli/stats.rb +171 -118
  34. data/lib/flare/tools/cli/sub_command.rb +16 -5
  35. data/lib/flare/tools/cli/summary.rb +12 -10
  36. data/lib/flare/tools/cli/threads.rb +15 -9
  37. data/lib/flare/tools/cli/verify.rb +20 -18
  38. data/lib/flare/tools/cluster.rb +3 -2
  39. data/lib/flare/util/constant.rb +3 -0
  40. data/lib/flare/util/pretty_table.rb +8 -0
  41. data/lib/flare/util/pretty_table/column.rb +41 -0
  42. data/lib/flare/util/pretty_table/row.rb +29 -0
  43. data/lib/flare/util/pretty_table/table.rb +37 -0
  44. data/package/Rakefile +44 -0
  45. data/package/flare-tools/Makefile +23 -0
  46. data/package/flare-tools/debian/changelog +135 -0
  47. data/package/flare-tools/debian/compat +1 -0
  48. data/package/flare-tools/debian/control +20 -0
  49. data/package/flare-tools/debian/copyright +35 -0
  50. data/package/flare-tools/debian/dirs +2 -0
  51. data/{.gemtest → package/flare-tools/debian/docs} +0 -0
  52. data/package/flare-tools/debian/rules +125 -0
  53. data/test/{test/experimental → experimental}/cache_test.rb +0 -0
  54. data/test/{test/experimental → experimental}/key_distribution_test.rb +0 -0
  55. data/test/{test/experimental → experimental}/keychecker_test.rb +0 -0
  56. data/test/{test/experimental → experimental}/list_test.rb +0 -0
  57. data/test/{test/extra → extra}/replication_test.rb +0 -0
  58. data/test/{test/integration → integration}/cli_test.rb +9 -7
  59. data/test/{test/integration → integration}/dump_expired_test.rb +4 -3
  60. data/test/{test/integration → integration}/dump_test.rb +5 -5
  61. data/test/{test/integration → integration}/index_server_test.rb +1 -1
  62. data/test/{test/integration → integration}/node_test.rb +1 -1
  63. data/test/{test/integration → integration}/partition_test.rb +6 -5
  64. data/test/{test/integration → integration}/proxy_test.rb +4 -3
  65. data/test/{test/integration → integration}/stats_test.rb +1 -1
  66. data/test/integration/subcommands.rb +128 -0
  67. data/test/{test/system → system}/flare_admin_test.rb +7 -5
  68. data/test/{test/unit → unit}/bwlimit_test.rb +0 -0
  69. data/test/{test/unit → unit}/cluster_test.rb +1 -1
  70. data/test/{test/unit → unit}/daemon_test.rb +0 -0
  71. data/test/{test/unit → unit}/logger_test.rb +0 -0
  72. data/test/{test/unit → unit}/tools_test.rb +0 -0
  73. data/test/unit/util/pretty_table_test.rb +46 -0
  74. data/test/{test/unit → unit}/util_test.rb +7 -2
  75. metadata +88 -100
  76. data/PostInstall.txt +0 -7
  77. data/lib/flare/tools/cli/cli_util.rb +0 -77
  78. data/lib/flare/util/command_line.rb +0 -79
@@ -5,11 +5,14 @@
5
5
 
6
6
  require 'flare/util/logging'
7
7
  require 'flare/util/interruption'
8
+ require 'flare/tools/cli/option'
8
9
 
9
10
  module Flare
10
11
  module Tools
11
12
  module Cli
12
13
  class SubCommand
14
+ include Flare::Tools::Cli::Option
15
+
13
16
  @@myname = {}
14
17
  @@desc = {}
15
18
  @@usage = {}
@@ -52,14 +55,22 @@ module Flare
52
55
  self.to_sym.to_s
53
56
  end
54
57
 
55
- def initalize
58
+ def initialize
59
+ option_init
56
60
  end
57
61
 
58
- def setup(opt)
59
- raise "setup"
62
+ def setup
63
+ set_option_global
60
64
  end
61
-
62
- def execute(config, *args)
65
+
66
+ def execute_subcommand(config, args)
67
+ setup
68
+ rest_args = parse_options(config, args)
69
+
70
+ execute(config, rest_args)
71
+ end
72
+
73
+ def execute
63
74
  raise "execute"
64
75
  end
65
76
 
@@ -8,19 +8,18 @@ require 'flare/tools/index_server'
8
8
  require 'flare/tools/cli/sub_command'
9
9
  require 'flare/tools/common'
10
10
  require 'flare/util/conversion'
11
+ require 'flare/tools/cli/index_server_config'
11
12
 
12
- #
13
13
  module Flare
14
14
  module Tools
15
15
  module Cli
16
16
 
17
- # == Description
18
- #
19
17
  class Summary < SubCommand
20
18
  include Flare::Util::Conversion
21
19
  include Flare::Util::Logging
22
20
  include Flare::Tools::Common
23
-
21
+ include Flare::Tools::Cli::IndexServerConfig
22
+
24
23
  myname :summary
25
24
  desc "show the summary of a flare cluster."
26
25
  usage "summary [hostname:port] ..."
@@ -33,19 +32,22 @@ module Flare
33
32
  ['%16s', 'size'],
34
33
  ['%13s', 'items'] ]
35
34
 
36
- def setup(opt)
35
+ def setup
36
+ super
37
+ set_option_index_server
37
38
  end
38
39
 
39
40
  def initialize
40
41
  super
41
42
  end
42
-
43
- def execute(config, *args)
43
+
44
+ def execute(config, args)
45
+ parse_index_server(config, args)
44
46
  nodes = {}
45
47
  threads = {}
46
48
  header = HeaderConfig
47
49
 
48
- Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
50
+ Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
49
51
  nodes = s.stats_nodes
50
52
  unless nodes
51
53
  error "Invalid index server."
@@ -69,7 +71,7 @@ module Flare
69
71
 
70
72
  nodes.each do |hostname_port,data|
71
73
  hostname, port = hostname_port.split(":", 2)
72
- Flare::Tools::Stats.open(hostname, data['port'], config[:timeout]) do |s|
74
+ Flare::Tools::Stats.open(hostname, data['port'], @timeout) do |s|
73
75
  stats = s.stats
74
76
  p = data['partition'].to_i
75
77
  total_parts = p+1 if p+1 > total_parts if data['role'] == 'master'
@@ -85,7 +87,7 @@ module Flare
85
87
  label = format % header.map{|x| x[1]}.flatten
86
88
  puts label
87
89
  puts(format % [name, total_parts, total_nodes, total_masters, total_slaves, total_bytes, total_items])
88
-
90
+
89
91
  S_OK
90
92
  end
91
93
 
@@ -7,6 +7,7 @@ require 'flare/tools/index_server'
7
7
  require 'flare/util/conversion'
8
8
  require 'flare/tools/common'
9
9
  require 'flare/tools/cli/sub_command'
10
+ require 'flare/tools/cli/index_server_config'
10
11
 
11
12
  module Flare
12
13
  module Tools
@@ -14,18 +15,23 @@ module Flare
14
15
  class Threads < SubCommand
15
16
  include Flare::Util::Conversion
16
17
  include Flare::Tools::Common
18
+ include Flare::Tools::Cli::IndexServerConfig
17
19
 
18
20
  myname :threads
19
21
  desc "show the list of threads in a flare cluster."
20
22
  usage "threads [hostname:port]"
21
-
22
- def setup(opt)
23
+
24
+ def setup
25
+ super
26
+ set_option_index_server
23
27
  end
24
28
 
25
29
  def initialize
30
+ super
26
31
  end
27
32
 
28
- def execute(config, *args)
33
+ def execute(config, args)
34
+ parse_index_server(config, args)
29
35
  header = [
30
36
  ['%5s', 'id'],
31
37
  ['%-32s', 'peer'],
@@ -49,14 +55,14 @@ module Flare
49
55
  error "invalid arguments: "+args.join(' ')
50
56
  return S_NG
51
57
  end
52
-
58
+
53
59
  threads = []
54
-
55
- Flare::Tools::Stats.open(hostname, port, config[:timeout]) do |s|
60
+
61
+ Flare::Tools::Stats.open(hostname, port, @timeout) do |s|
56
62
  threads = s.stats_threads
57
63
  threads = threads.sort_by{|key,val| [val['peer'], key]}
58
64
  end
59
-
65
+
60
66
  puts format % header.map{|x| x[1]}.flatten
61
67
 
62
68
  threads.each do |thread_id, data|
@@ -68,10 +74,10 @@ module Flare
68
74
  data['queue'],
69
75
  ]
70
76
  end
71
-
77
+
72
78
  S_OK
73
79
  end
74
-
80
+
75
81
  end
76
82
  end
77
83
  end
@@ -9,26 +9,27 @@ require 'flare/util/conversion'
9
9
  require 'flare/util/key_resolver'
10
10
  require 'flare/util/hash_function'
11
11
  require 'flare/tools/cli/sub_command'
12
+ require 'flare/tools/cli/index_server_config'
12
13
  require 'digest/md5'
13
14
 
14
15
 
15
- #
16
16
  module Flare
17
17
  module Tools
18
18
  module Cli
19
19
 
20
- # == Description
21
- #
22
20
  class Verify < SubCommand
23
21
  include Flare::Util::Conversion
24
22
  include Flare::Util::HashFunction
23
+ include Flare::Tools::Cli::IndexServerConfig
25
24
 
26
25
  myname :verify
27
26
  desc "verify the cluster. (experimental)"
28
27
  usage "verify"
29
-
30
- def setup(opt)
31
- opt.on('--key-hash-algorithm=TYPE', "key hash algorithm") do |v|
28
+
29
+ def setup
30
+ super
31
+ set_option_index_server
32
+ @optp.on('--key-hash-algorithm=TYPE', "key hash algorithm") do |v|
32
33
  case @key_hash_algorithm = v.to_sym
33
34
  when :simple, :crc32
34
35
  else
@@ -36,12 +37,12 @@ module Flare
36
37
  exit
37
38
  end
38
39
  end
39
- opt.on('--use-test-data', "store test data") {|v| @use_test_data = true}
40
- opt.on('--debug', "use debug mode") {|v| @debug = true}
41
- opt.on('--64bit', "(experimental) 64bit mode") {|v| @word_size = 64}
42
- opt.on('--verbose', "use verbose mode") {|v| @verbose = true}
43
- opt.on('--meta', "use meta command") {|v| @meta = true}
44
- opt.on('--quiet', "use quiet mode") {|v| @quiet = true}
40
+ @optp.on('--use-test-data', "store test data") {|v| @use_test_data = true}
41
+ @optp.on('--debug', "use debug mode") {|v| @debug = true}
42
+ @optp.on('--64bit', "(experimental) 64bit mode") {|v| @word_size = 64}
43
+ @optp.on('--verbose', "use verbose mode") {|v| @verbose = true}
44
+ @optp.on('--meta', "use meta command") {|v| @meta = true}
45
+ @optp.on('--quiet', "use quiet mode") {|v| @quiet = true}
45
46
  end
46
47
 
47
48
  def initialize
@@ -57,12 +58,13 @@ module Flare
57
58
  @quiet = false
58
59
  end
59
60
 
60
- def execute(config, *args)
61
+ def execute(config, args)
62
+ parse_index_server(config, args)
61
63
  keys = {}
62
64
  cout = STDERR
63
65
  status = S_OK
64
66
  info "connecting to index ..."
65
- Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
67
+ Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
66
68
  nodes = s.stats_nodes.sort_by{|key, val| [val['partition'].to_i, val['role'], key]}
67
69
 
68
70
  # meta
@@ -111,7 +113,7 @@ module Flare
111
113
 
112
114
  if @use_test_data
113
115
  info "storing test data ..."
114
- Flare::Tools::Node.open(hostname0, port0.to_i, config[:timeout]) do |n|
116
+ Flare::Tools::Node.open(hostname0, port0.to_i, @timeout) do |n|
115
117
  (1..10000).each do |i|
116
118
  key = ".test."+Digest::MD5.new.update(i.to_s).to_s
117
119
  n.set(key, i.to_s)
@@ -123,7 +125,7 @@ module Flare
123
125
  nodes.each do |nodekey,val|
124
126
  hostname, port = nodekey.split(":", 2)
125
127
  partition = val['partition'].to_i
126
- Flare::Tools::Node.open(hostname, port.to_i, config[:timeout]) do |n|
128
+ Flare::Tools::Node.open(hostname, port.to_i, @timeout) do |n|
127
129
  cout.write "checking #{nodekey} ... "
128
130
  msg = "OK"
129
131
  interruptible do
@@ -178,7 +180,7 @@ module Flare
178
180
  error "failed: not found #{remain} keys" if remain > 0
179
181
 
180
182
  # delete
181
- Flare::Tools::Node.open(hostname0, port0.to_i, config[:timeout]) do |n|
183
+ Flare::Tools::Node.open(hostname0, port0.to_i, @timeout) do |n|
182
184
  keys.each do |k,v|
183
185
  n.delete(k)
184
186
  end
@@ -194,7 +196,7 @@ module Flare
194
196
  end
195
197
  status
196
198
  end
197
-
199
+
198
200
  end
199
201
  end
200
202
  end
@@ -128,8 +128,8 @@ module Flare
128
128
  true
129
129
  when RoleMaster
130
130
  # if the partition has at least one active slave, one of the slaves will take over the master.
131
- slaves_in_partition(node[StatPartition]).inject(false) do |r,slave_nodekey|
132
- node_stat(slave_nodekey)[State] == StateActive
131
+ slaves_in_partition(node[StatPartition]).inject(false) do |r, slave_nodekey|
132
+ r || node_stat(slave_nodekey)[State] == StateActive
133
133
  end
134
134
  else
135
135
  error "unknown role: #{node[Role]}"
@@ -252,6 +252,7 @@ module Flare
252
252
  <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
253
253
  <!DOCTYPE boost_serialization>
254
254
  <boost_serialization signature="serialization::archive" version="4">
255
+ <version>0</version>
255
256
  <node_map#{serattr_(node_map_id)}>
256
257
  \t<count>#{@nodes.size}</count>
257
258
  \t<item_version>0</item_version>
@@ -20,6 +20,9 @@ module Flare
20
20
  DefaultTimeout = 10
21
21
  # the default bandwidth limit (bps)
22
22
  DefalutBwlimit = 0 # unlimited
23
+
24
+ STATUS_OK = 0
25
+ STATUS_NG = 1
23
26
  end
24
27
  end
25
28
  end
@@ -0,0 +1,8 @@
1
+ # -*- coding: utf-8; -*-
2
+ # Authors:: Yuya YAGUCHI <yuya.yaguchi@gree.net>
3
+ # Copyright:: Copyright (C) GREE, Inc. 2014.
4
+ # License:: MIT-style
5
+
6
+ require 'flare/util/pretty_table/table.rb'
7
+ require 'flare/util/pretty_table/column.rb'
8
+ require 'flare/util/pretty_table/row.rb'
@@ -0,0 +1,41 @@
1
+ # -*- coding: utf-8; -*-
2
+ # Authors:: Yuya YAGUCHI <yuya.yaguchi@gree.net>
3
+ # Copyright:: Copyright (C) GREE, Inc. 2014.
4
+ # License:: MIT-style
5
+
6
+ module Flare
7
+ module Util
8
+ module PrettyTable
9
+ class Column
10
+ def initialize(text, options = {})
11
+ @text = text.to_s
12
+ @align = options[:align] || :left
13
+ end
14
+
15
+ def width
16
+ @text.size
17
+ end
18
+
19
+ def prettify(column_width)
20
+ if self.width >= column_width
21
+ return @text
22
+ end
23
+
24
+ padding = padding(column_width - self.width)
25
+ case @align
26
+ when :left then
27
+ @text + padding
28
+ else # @align == :right
29
+ padding + @text
30
+ end
31
+ end
32
+
33
+ private
34
+
35
+ def padding(length)
36
+ ' ' * length
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,29 @@
1
+ # -*- coding: utf-8; -*-
2
+ # Authors:: Yuya YAGUCHI <yuya.yaguchi@gree.net>
3
+ # Copyright:: Copyright (C) GREE, Inc. 2014.
4
+ # License:: MIT-style
5
+
6
+ module Flare
7
+ module Util
8
+ module PrettyTable
9
+ class Row
10
+ attr_reader :columns
11
+
12
+ def initialize(option = {})
13
+ @columns = []
14
+ @separator = option[:separator] || ' '
15
+ end
16
+
17
+ def add_column(column)
18
+ @columns << column
19
+ end
20
+
21
+ def prettify(column_widths)
22
+ @columns \
23
+ .each_with_index.map {|column, index| column.prettify(column_widths[index]) } \
24
+ .join(@separator)
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,37 @@
1
+ # -*- coding: utf-8; -*-
2
+ # Authors:: Yuya YAGUCHI <yuya.yaguchi@gree.net>
3
+ # Copyright:: Copyright (C) GREE, Inc. 2014.
4
+ # License:: MIT-style
5
+
6
+ module Flare
7
+ module Util
8
+ module PrettyTable
9
+ class Table
10
+ def initialize
11
+ @rows = []
12
+ end
13
+
14
+ def add_row(row)
15
+ @rows << row
16
+ end
17
+
18
+ def prettify
19
+ column_widths = max_column_widths
20
+ @rows \
21
+ .map {|row| row.prettify(column_widths) } \
22
+ .join("\n")
23
+ end
24
+
25
+ def max_column_widths
26
+ widths = []
27
+ @rows.each do |row|
28
+ row.columns.each_with_index do |column, index|
29
+ widths[index] = [column.width, (widths[index] || 0)].max
30
+ end
31
+ end
32
+ widths
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
data/package/Rakefile ADDED
@@ -0,0 +1,44 @@
1
+
2
+ $LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
3
+
4
+ require 'flare/tools'
5
+
6
+ flaretools = "flare-tools"
7
+
8
+ task :default => [:build]
9
+
10
+ task :build do |t|
11
+ sh "(cd #{flaretools} && debuild -us -uc)"
12
+ end
13
+
14
+ task :debclean do
15
+ sh "(cd #{flaretools} && debclean)"
16
+ end
17
+
18
+ task :change do
19
+ puts "================================="
20
+ puts " Flare::Tools::VERSION = #{Flare::Tools::VERSION}"
21
+ puts "================================="
22
+ sh "(cd #{flaretools} && dch -i)"
23
+ end
24
+
25
+ def previous version
26
+ prev = version.split('.').map{|v| v.to_i}
27
+ prev[2] -= 1
28
+ prev.join('.')
29
+ end
30
+
31
+ task :autochange do
32
+ v = Flare::Tools::VERSION
33
+ since = previous(v)
34
+ sh "git-dch --debian-branch='(no branch)' --new-version #{version} --since=#{since}"
35
+ end
36
+
37
+ task :clean => [:debclean] do
38
+ sh "rm -f *.dsc *.tar.gz *.build *.changes"
39
+ end
40
+
41
+ task :cleanall => [:clean] do
42
+ sh "rm -f flare-tools/debian/changelog.dch"
43
+ end
44
+