flare-tools 0.4.5.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. data/.gitignore +24 -0
  2. data/.travis.yml +15 -0
  3. data/Gemfile +9 -0
  4. data/Gemfile.lock +28 -0
  5. data/History.txt +5 -0
  6. data/Makefile +25 -0
  7. data/README.txt +2 -2
  8. data/Rakefile +36 -34
  9. data/flare-tools.gemspec +26 -0
  10. data/lib/flare/entity/server.rb +4 -0
  11. data/lib/flare/tools.rb +3 -3
  12. data/lib/flare/tools/cli.rb +1 -0
  13. data/lib/flare/tools/cli/activate.rb +19 -13
  14. data/lib/flare/tools/cli/balance.rb +15 -8
  15. data/lib/flare/tools/cli/deploy.rb +53 -49
  16. data/lib/flare/tools/cli/dispatch.rb +101 -0
  17. data/lib/flare/tools/cli/down.rb +19 -14
  18. data/lib/flare/tools/cli/dump.rb +19 -14
  19. data/lib/flare/tools/cli/dumpkey.rb +21 -16
  20. data/lib/flare/tools/cli/flare_admin.rb +16 -72
  21. data/lib/flare/tools/cli/flare_argv0.rb +7 -51
  22. data/lib/flare/tools/cli/index.rb +10 -7
  23. data/lib/flare/tools/cli/index_server_config.rb +94 -0
  24. data/lib/flare/tools/cli/list.rb +14 -12
  25. data/lib/flare/tools/cli/master.rb +18 -12
  26. data/lib/flare/tools/cli/option.rb +55 -0
  27. data/lib/flare/tools/cli/part.rb +13 -12
  28. data/lib/flare/tools/cli/ping.rb +11 -13
  29. data/lib/flare/tools/cli/reconstruct.rb +22 -15
  30. data/lib/flare/tools/cli/remove.rb +20 -17
  31. data/lib/flare/tools/cli/restore.rb +20 -20
  32. data/lib/flare/tools/cli/slave.rb +20 -14
  33. data/lib/flare/tools/cli/stats.rb +171 -118
  34. data/lib/flare/tools/cli/sub_command.rb +16 -5
  35. data/lib/flare/tools/cli/summary.rb +12 -10
  36. data/lib/flare/tools/cli/threads.rb +15 -9
  37. data/lib/flare/tools/cli/verify.rb +20 -18
  38. data/lib/flare/tools/cluster.rb +3 -2
  39. data/lib/flare/util/constant.rb +3 -0
  40. data/lib/flare/util/pretty_table.rb +8 -0
  41. data/lib/flare/util/pretty_table/column.rb +41 -0
  42. data/lib/flare/util/pretty_table/row.rb +29 -0
  43. data/lib/flare/util/pretty_table/table.rb +37 -0
  44. data/package/Rakefile +44 -0
  45. data/package/flare-tools/Makefile +23 -0
  46. data/package/flare-tools/debian/changelog +135 -0
  47. data/package/flare-tools/debian/compat +1 -0
  48. data/package/flare-tools/debian/control +20 -0
  49. data/package/flare-tools/debian/copyright +35 -0
  50. data/package/flare-tools/debian/dirs +2 -0
  51. data/{.gemtest → package/flare-tools/debian/docs} +0 -0
  52. data/package/flare-tools/debian/rules +125 -0
  53. data/test/{test/experimental → experimental}/cache_test.rb +0 -0
  54. data/test/{test/experimental → experimental}/key_distribution_test.rb +0 -0
  55. data/test/{test/experimental → experimental}/keychecker_test.rb +0 -0
  56. data/test/{test/experimental → experimental}/list_test.rb +0 -0
  57. data/test/{test/extra → extra}/replication_test.rb +0 -0
  58. data/test/{test/integration → integration}/cli_test.rb +9 -7
  59. data/test/{test/integration → integration}/dump_expired_test.rb +4 -3
  60. data/test/{test/integration → integration}/dump_test.rb +5 -5
  61. data/test/{test/integration → integration}/index_server_test.rb +1 -1
  62. data/test/{test/integration → integration}/node_test.rb +1 -1
  63. data/test/{test/integration → integration}/partition_test.rb +6 -5
  64. data/test/{test/integration → integration}/proxy_test.rb +4 -3
  65. data/test/{test/integration → integration}/stats_test.rb +1 -1
  66. data/test/integration/subcommands.rb +128 -0
  67. data/test/{test/system → system}/flare_admin_test.rb +7 -5
  68. data/test/{test/unit → unit}/bwlimit_test.rb +0 -0
  69. data/test/{test/unit → unit}/cluster_test.rb +1 -1
  70. data/test/{test/unit → unit}/daemon_test.rb +0 -0
  71. data/test/{test/unit → unit}/logger_test.rb +0 -0
  72. data/test/{test/unit → unit}/tools_test.rb +0 -0
  73. data/test/unit/util/pretty_table_test.rb +46 -0
  74. data/test/{test/unit → unit}/util_test.rb +7 -2
  75. metadata +88 -100
  76. data/PostInstall.txt +0 -7
  77. data/lib/flare/tools/cli/cli_util.rb +0 -77
  78. data/lib/flare/util/command_line.rb +0 -79
@@ -11,6 +11,7 @@ require 'flare/tools/common'
11
11
  require 'flare/util/conversion'
12
12
  require 'flare/util/constant'
13
13
  require 'flare/tools/cli/sub_command'
14
+ require 'flare/tools/cli/index_server_config'
14
15
 
15
16
  module Flare
16
17
  module Tools
@@ -19,17 +20,21 @@ module Flare
19
20
  include Flare::Util::Conversion
20
21
  include Flare::Util::Constant
21
22
  include Flare::Tools::Common
23
+ include Flare::Tools::Cli::IndexServerConfig
22
24
 
23
25
  DefaultRetry = 10
24
-
26
+
25
27
  myname :slave
26
28
  desc "construct slaves from proxy nodes."
27
29
  usage "slave [hostname:port:balance:partition] ..."
28
30
 
29
- def setup(opt)
30
- opt.on('--force', "commit changes without confirmation") {@force = true}
31
- opt.on('--retry=COUNT', "specify retry count(default:#{@retry})") {|v| @retry = v.to_i}
32
- opt.on('--clean', "clear datastore before construction") {@clean = true}
31
+ def setup
32
+ super
33
+ set_option_index_server
34
+ set_option_dry_run
35
+ set_option_force
36
+ @optp.on('--retry=COUNT', "specify retry count(default:#{@retry})") {|v| @retry = v.to_i}
37
+ @optp.on('--clean', "clear datastore before construction") {@clean = true}
33
38
  end
34
39
 
35
40
  def initialize
@@ -39,7 +44,8 @@ module Flare
39
44
  @clean = false
40
45
  end
41
46
 
42
- def execute(config, *args)
47
+ def execute(config, args)
48
+ parse_index_server(config, args)
43
49
  return S_NG if args.empty?
44
50
 
45
51
  hosts = args.map do |arg|
@@ -50,10 +56,10 @@ module Flare
50
56
  end
51
57
  [hostname, port, balance.to_i, partition.to_i]
52
58
  end
53
-
54
- Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
59
+
60
+ Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
55
61
  cluster = fetch_cluster(s)
56
-
62
+
57
63
  hosts.each do |hostname,port,balance,partition|
58
64
  nodekey = nodekey_of hostname, port
59
65
 
@@ -68,7 +74,7 @@ module Flare
68
74
  puts "#{nodekey} is not a proxy."
69
75
  next
70
76
  end
71
-
77
+
72
78
  exec = @force
73
79
  unless exec
74
80
  STDERR.print "making node slave (node=#{nodekey}, role=#{node['role']} -> slave) (y/n): "
@@ -76,9 +82,9 @@ module Flare
76
82
  exec = true if gets.chomp.upcase == "Y"
77
83
  end
78
84
  end
79
- if exec && !config[:dry_run]
85
+ if exec && !@dry_run
80
86
  if @clean
81
- Flare::Tools::Node.open(hostname, port, config[:timeout]) do |n|
87
+ Flare::Tools::Node.open(hostname, port, @timeout) do |n|
82
88
  n.flush_all
83
89
  end
84
90
  end
@@ -97,7 +103,7 @@ module Flare
97
103
  end
98
104
  end
99
105
  if resp
100
- wait_for_slave_construction(s, nodekey, config[:timeout])
106
+ wait_for_slave_construction(s, nodekey, @timeout)
101
107
  if balance > 0
102
108
  unless @force
103
109
  STDERR.print "changing node's balance (node=#{nodekey}, balance=0 -> #{balance}) (y/n): "
@@ -115,7 +121,7 @@ module Flare
115
121
  end
116
122
  STDOUT.puts string_of_nodelist(s.stats_nodes, hosts.map {|x| x[0..1].join(':')})
117
123
  end
118
-
124
+
119
125
  return S_OK
120
126
  end # execute()
121
127
 
@@ -8,8 +8,9 @@ require 'flare/tools/index_server'
8
8
  require 'flare/tools/cli/sub_command'
9
9
  require 'flare/tools/common'
10
10
  require 'flare/util/conversion'
11
+ require 'flare/util/pretty_table'
12
+ require 'flare/tools/cli/index_server_config'
11
13
 
12
- #
13
14
  module Flare
14
15
  module Tools
15
16
  module Cli
@@ -20,29 +21,40 @@ module Flare
20
21
  include Flare::Util::Conversion
21
22
  include Flare::Util::Logging
22
23
  include Flare::Tools::Common
23
-
24
+ include Flare::Util::PrettyTable
25
+ include Flare::Tools::Cli::IndexServerConfig
26
+
24
27
  myname :stats
25
28
  desc "show the statistics of a flare cluster."
26
29
  usage "stats [hostname:port] ..."
27
30
 
28
- HeaderConfig = [ ['%-25.25s', 'hostname:port'],
29
- ['%7s', 'state'],
30
- ['%6s', 'role'],
31
- ['%9s', 'partition'],
32
- ['%7s', 'balance'],
33
- ['%8.8s', 'items'],
34
- ['%4s', 'conn'],
35
- ['%6.6s', 'behind'],
36
- ['%3.3s', 'hit'],
37
- ['%4.4s', 'size'],
38
- ['%6.6s', 'uptime'],
39
- ['%7s', 'version'] ]
40
-
41
- def setup(opt)
42
- opt.on("-q", '--qps', "show qps") {|v| @qps = v}
43
- opt.on("-w", '--wait=SECOND', "specify wait time for repeat(second)") {|v| @wait = v.to_i}
44
- opt.on("-c", '--count=REPEATTIME', "specify repeat count") {|v| @count = v.to_i}
45
- opt.on("-d", '--delimiter=CHAR', "spedify delimiter") {|v| @delimiter = v}
31
+ HeaderConfigs = [
32
+ ['hostname:port', {}],
33
+ ['state', {}],
34
+ ['role', {}],
35
+ ['partition', {:align => :right}],
36
+ ['balance', {:align => :right}],
37
+ ['items', {:align => :right}],
38
+ ['conn', {:align => :right}],
39
+ ['behind', {:align => :right}],
40
+ ['hit', {:align => :right}],
41
+ ['size', {:align => :right}],
42
+ ['uptime', {:align => :right}],
43
+ ['version', {:align => :right}],
44
+ ]
45
+ HeaderConfigQpss = [
46
+ ['qps', {:align => :right}],
47
+ ['qps-r', {:align => :right}],
48
+ ['qps-w', {:align => :right}],
49
+ ]
50
+
51
+ def setup
52
+ super
53
+ set_option_index_server
54
+ @optp.on("-q", '--qps', "show qps") {|v| @qps = v}
55
+ @optp.on("-w", '--wait=SECOND', "specify wait time for repeat(second)") {|v| @wait = v.to_i}
56
+ @optp.on("-c", '--count=REPEATTIME', "specify repeat count") {|v| @count = v.to_i}
57
+ @optp.on("-d", '--delimiter=CHAR', "specify delimiter") {|v| @delimiter = v}
46
58
  end
47
59
 
48
60
  def initialize
@@ -53,19 +65,18 @@ module Flare
53
65
  @cont = true
54
66
  @delimiter = ' '
55
67
  end
56
-
68
+
57
69
  def interrupt
58
70
  puts "INTERRUPTED"
59
71
  @cont = false
60
72
  end
61
73
 
62
- def execute(config, *args)
74
+ def execute(config, args)
75
+ parse_index_server(config, args)
63
76
  nodes = {}
64
77
  threads = {}
65
- header = Marshal.load(Marshal.dump(HeaderConfig))
66
- header << ['%5.5s', 'qps'] << ['%5.5s', 'qps-r'] << ['%5.5s', 'qps-w'] if @qps
67
78
 
68
- Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout]) do |s|
79
+ Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
69
80
  nodes = s.stats_nodes
70
81
  unless nodes
71
82
  error "Invalid index server."
@@ -79,64 +90,11 @@ module Flare
79
90
  queue = {}
80
91
 
81
92
  nodes.each do |hostname_port,data|
82
- hostname, port = hostname_port.split(":", 2)
83
93
  queue[hostname_port] = SizedQueue.new(1)
84
94
  worker_threads << Thread.new(queue[hostname_port]) do |q|
85
- s = nil
86
- while @cont
87
- stats_data = nil
88
- begin
89
- s = Flare::Tools::Stats.open(hostname, data['port'], config[:timeout])
90
- stats = s.stats
91
- time = Time.now
92
- behind = threads[hostname_port].has_key?('behind') ? threads[hostname_port]['behind'] : "-"
93
- uptime_short = short_desc_of_second(stats['uptime'])
94
- hit_rate = if stats.has_key?('cmd_get') && stats['cmd_get'] != "0"
95
- cmd_get = stats['cmd_get'].to_f
96
- get_hits = stats['get_hits'].to_f
97
- (get_hits / cmd_get * 100.0).round
98
- else
99
- "-"
100
- end
101
- size = stats['bytes'] == "0" ? "-" : (stats['bytes'].to_i / 1024 / 1024 / 1024) # gigabyte
102
- stats_data = {
103
- :hostname => hostname,
104
- :port => port,
105
- :hostname_port => "#{hostname}:#{port}",
106
- :state => data['state'],
107
- :role => data['role'],
108
- :partition => data['partition'] == "-1" ? "-" : data['partition'],
109
- :balance => data['balance'],
110
- :items => stats['curr_items'],
111
- :conn => stats['curr_connections'],
112
- :behind => behind,
113
- :hit_rate => hit_rate,
114
- :size => size,
115
- :uptime => stats['uptime'],
116
- :uptime_short => uptime_short,
117
- :version => stats['version'],
118
- :cmd_get => stats['cmd_get'],
119
- :cmd_set => stats['cmd_set'],
120
- :time => time,
121
- }
122
- rescue Errno::ECONNREFUSED => e
123
- rescue => e
124
- begin
125
- s.close unless s.nil?
126
- rescue => close_error
127
- end
128
- s = nil
129
- end
130
- if stats_data.nil?
131
- stats_data = {
132
- :hostname_port => "#{hostname}:#{port}",
133
- }
134
- end
135
- q.push stats_data
136
- end
137
- s.close unless s.nil?
138
- end # Thread.new
139
- end # nodes.each
95
+ enqueueing_node_stats(q, threads, config, hostname_port, data)
96
+ end
97
+ end
140
98
 
141
99
  query_prev = {} if @qps
142
100
 
@@ -144,7 +102,11 @@ module Flare
144
102
  interruptible {sleep 1}
145
103
  end
146
104
 
147
- s = Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], config[:timeout])
105
+ s = Flare::Tools::IndexServer.open(
106
+ @index_server_entity.host,
107
+ @index_server_entity.port,
108
+ @timeout
109
+ )
148
110
  unless s
149
111
  error "Couldn't connect to the index server."
150
112
  return S_NG
@@ -164,62 +126,39 @@ module Flare
164
126
  nodes.each do |k, n|
165
127
  max_nodekey_length = k.length if k.length > max_nodekey_length
166
128
  end
167
- header[0][0] = "%-#{max_nodekey_length}.#{max_nodekey_length}s"
168
- format = header.map {|x| x[0]}.join(@delimiter)
169
- label = format % header.map{|x| x[1]}.flatten
170
- puts label
171
- nodes.each do |k, n|
129
+ table = Table.new
130
+ add_header_to_table(table, header_configs)
131
+ nodes.each do |k, node|
172
132
  stats_data = queue[k].pop
173
133
  next if (args.size > 0 && !args.include?(k))
174
- stats_data[:state] = n['state']
175
- stats_data[:role] = n['role']
176
- stats_data[:partition] = n['partition']
177
- stats_data[:balance] = n['balance']
178
- stats_data[:behind] = (threads.has_key?(k) || threads[k].has_key?('behind')) ? threads[k]['behind'] : "-"
179
- output = [:hostname_port, :state, :role, :partition, :balance, :items,
180
- :conn, :behind, :hit_rate, :size, :uptime_short, :version].map {|x| stats_data[x]}
181
- if @qps
182
- query = {}
183
- query[:query] = stats_data[:cmd_get].to_i+stats_data[:cmd_set].to_i
184
- query[:query_r] = stats_data[:cmd_get].to_i
185
- query[:query_w] = stats_data[:cmd_set].to_i
186
- query[:time] = time = stats_data[:time]
187
- if query_prev.has_key?(k)
188
- duration = (time-query_prev[k][:time]).to_f
189
- [:query, :query_r, :query_w].each do |x|
190
- diff = (query[x]-query_prev[k][x]).to_f
191
- qps = if diff > 0 then diff/duration else 0 end
192
- output << qps
193
- end
194
- else
195
- output << 0 << 0 << 0
196
- end
197
- query_prev[k] = query.dup
198
- end
199
-
200
- puts format % output
134
+ behind = (threads.has_key?(k) || threads[k].has_key?('behind')) ? threads[k]['behind'] : "-"
135
+ r = record(stats_data, node, behind, query_prev, k)
136
+ add_record_to_table(table, header_configs, r)
201
137
  end
202
138
  interruptible {
203
139
  wait_for_stats
204
140
  }
141
+ puts table.prettify
205
142
  end
206
143
  s.close
207
144
 
208
145
  @cont = false
209
146
 
210
147
  queue.each do |k,q|
211
- q.clear
148
+ q.pop until q.empty?
212
149
  end
213
-
150
+
214
151
  interruptible {
215
152
  worker_threads.each do |t|
216
153
  t.join
217
154
  end
218
155
  }
219
-
156
+
220
157
  S_OK
221
158
  end
222
159
 
160
+ private
161
+
223
162
  def wait_for_stats
224
163
  if @qps || @count > 1
225
164
  wait = @wait
@@ -229,7 +168,121 @@ module Flare
229
168
  end
230
169
  end
231
170
  end
232
-
171
+
172
+ def enqueueing_node_stats(q, threads, config, hostname_port, data)
173
+ hostname, port = hostname_port.split(":", 2)
174
+ s = nil
175
+ while @cont
176
+ stats_data = nil
177
+ begin
178
+ s = Flare::Tools::Stats.open(hostname, data['port'], @timeout)
179
+ stats = s.stats
180
+ time = Time.now
181
+ behind = threads[hostname_port].has_key?('behind') ? threads[hostname_port]['behind'] : "-"
182
+ uptime_short = short_desc_of_second(stats['uptime'])
183
+ hit_rate = if stats.has_key?('cmd_get') && stats['cmd_get'] != "0"
184
+ cmd_get = stats['cmd_get'].to_f
185
+ get_hits = stats['get_hits'].to_f
186
+ (get_hits / cmd_get * 100.0).round
187
+ else
188
+ "-"
189
+ end
190
+ size = stats['bytes'] == "0" ? "-" : (stats['bytes'].to_i / 1024 / 1024 / 1024) # gigabyte
191
+ stats_data = {
192
+ :hostname => hostname,
193
+ :port => port,
194
+ :hostname_port => "#{hostname}:#{port}",
195
+ :state => data['state'],
196
+ :role => data['role'],
197
+ :partition => data['partition'] == "-1" ? "-" : data['partition'],
198
+ :balance => data['balance'],
199
+ :items => stats['curr_items'],
200
+ :conn => stats['curr_connections'],
201
+ :behind => behind,
202
+ :hit_rate => hit_rate,
203
+ :size => size,
204
+ :uptime => stats['uptime'],
205
+ :uptime_short => uptime_short,
206
+ :version => stats['version'],
207
+ :cmd_get => stats['cmd_get'],
208
+ :cmd_set => stats['cmd_set'],
209
+ :time => time,
210
+ }
211
+ rescue Errno::ECONNREFUSED => e
212
+ rescue => e
213
+ begin
214
+ s.close unless s.nil?
215
+ rescue => close_error
216
+ error "Socket close failed: #{close_error.inspect}"
217
+ end
218
+ s = nil
219
+ end
220
+ if stats_data.nil?
221
+ stats_data = {
222
+ :hostname_port => "#{hostname}:#{port}",
223
+ }
224
+ end
225
+ q.push stats_data
226
+ end
227
+ s.close unless s.nil?
228
+ end
229
+
230
+ def add_header_to_table(table, header_configs)
231
+ row = Row.new(:separator => @delimiter)
232
+ header_configs.each do |header_config|
233
+ row.add_column(Column.new(header_config[0]))
234
+ end
235
+ table.add_row(row)
236
+ end
237
+
238
+ def add_record_to_table(table, header_configs, record)
239
+ row = Row.new(:separator => @delimiter)
240
+ header_configs.each_with_index do |header_config, index|
241
+ row.add_column(Column.new(record[index], header_config[1]))
242
+ end
243
+ table.add_row(row)
244
+ end
245
+
246
+ # You can override this method to extend stats infos.
247
+ def record(stats_data, node, behind, query_prev, index)
248
+ stats_data[:state] = node['state']
249
+ stats_data[:role] = node['role']
250
+ stats_data[:partition] = node['partition']
251
+ stats_data[:balance] = node['balance']
252
+ stats_data[:behind] = behind
253
+ output = [:hostname_port, :state, :role, :partition, :balance, :items,
254
+ :conn, :behind, :hit_rate, :size, :uptime_short, :version].map {|x| stats_data[x]}
255
+ if @qps
256
+ query = {}
257
+ query[:query] = stats_data[:cmd_get].to_i+stats_data[:cmd_set].to_i
258
+ query[:query_r] = stats_data[:cmd_get].to_i
259
+ query[:query_w] = stats_data[:cmd_set].to_i
260
+ query[:time] = time = stats_data[:time]
261
+ if query_prev.has_key?(index)
262
+ duration = (time-query_prev[index][:time]).to_f
263
+ [:query, :query_r, :query_w].each do |x|
264
+ diff = (query[x]-query_prev[index][x]).to_f
265
+ qps = if diff > 0
266
+ sprintf("%.1f", diff/duration)
267
+ else
268
+ '-'
269
+ end
270
+ output << qps
271
+ end
272
+ else
273
+ output << '-' << '-' << '-'
274
+ end
275
+ query_prev[index] = query.dup
276
+ end
277
+ output
278
+ end
279
+
280
+ # You can override this method to extend stats infos.
281
+ def header_configs
282
+ configs = Marshal.load(Marshal.dump(HeaderConfigs))
283
+ configs += Marshal.load(Marshal.dump(HeaderConfigQpss)) if @qps
284
+ configs
285
+ end
233
286
  end
234
287
  end
235
288
  end