flare-tools 0.4.5.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +24 -0
- data/.travis.yml +15 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +28 -0
- data/History.txt +5 -0
- data/Makefile +25 -0
- data/README.txt +2 -2
- data/Rakefile +36 -34
- data/flare-tools.gemspec +26 -0
- data/lib/flare/entity/server.rb +4 -0
- data/lib/flare/tools.rb +3 -3
- data/lib/flare/tools/cli.rb +1 -0
- data/lib/flare/tools/cli/activate.rb +19 -13
- data/lib/flare/tools/cli/balance.rb +15 -8
- data/lib/flare/tools/cli/deploy.rb +53 -49
- data/lib/flare/tools/cli/dispatch.rb +101 -0
- data/lib/flare/tools/cli/down.rb +19 -14
- data/lib/flare/tools/cli/dump.rb +19 -14
- data/lib/flare/tools/cli/dumpkey.rb +21 -16
- data/lib/flare/tools/cli/flare_admin.rb +16 -72
- data/lib/flare/tools/cli/flare_argv0.rb +7 -51
- data/lib/flare/tools/cli/index.rb +10 -7
- data/lib/flare/tools/cli/index_server_config.rb +94 -0
- data/lib/flare/tools/cli/list.rb +14 -12
- data/lib/flare/tools/cli/master.rb +18 -12
- data/lib/flare/tools/cli/option.rb +55 -0
- data/lib/flare/tools/cli/part.rb +13 -12
- data/lib/flare/tools/cli/ping.rb +11 -13
- data/lib/flare/tools/cli/reconstruct.rb +22 -15
- data/lib/flare/tools/cli/remove.rb +20 -17
- data/lib/flare/tools/cli/restore.rb +20 -20
- data/lib/flare/tools/cli/slave.rb +20 -14
- data/lib/flare/tools/cli/stats.rb +171 -118
- data/lib/flare/tools/cli/sub_command.rb +16 -5
- data/lib/flare/tools/cli/summary.rb +12 -10
- data/lib/flare/tools/cli/threads.rb +15 -9
- data/lib/flare/tools/cli/verify.rb +20 -18
- data/lib/flare/tools/cluster.rb +3 -2
- data/lib/flare/util/constant.rb +3 -0
- data/lib/flare/util/pretty_table.rb +8 -0
- data/lib/flare/util/pretty_table/column.rb +41 -0
- data/lib/flare/util/pretty_table/row.rb +29 -0
- data/lib/flare/util/pretty_table/table.rb +37 -0
- data/package/Rakefile +44 -0
- data/package/flare-tools/Makefile +23 -0
- data/package/flare-tools/debian/changelog +135 -0
- data/package/flare-tools/debian/compat +1 -0
- data/package/flare-tools/debian/control +20 -0
- data/package/flare-tools/debian/copyright +35 -0
- data/package/flare-tools/debian/dirs +2 -0
- data/{.gemtest → package/flare-tools/debian/docs} +0 -0
- data/package/flare-tools/debian/rules +125 -0
- data/test/{test/experimental → experimental}/cache_test.rb +0 -0
- data/test/{test/experimental → experimental}/key_distribution_test.rb +0 -0
- data/test/{test/experimental → experimental}/keychecker_test.rb +0 -0
- data/test/{test/experimental → experimental}/list_test.rb +0 -0
- data/test/{test/extra → extra}/replication_test.rb +0 -0
- data/test/{test/integration → integration}/cli_test.rb +9 -7
- data/test/{test/integration → integration}/dump_expired_test.rb +4 -3
- data/test/{test/integration → integration}/dump_test.rb +5 -5
- data/test/{test/integration → integration}/index_server_test.rb +1 -1
- data/test/{test/integration → integration}/node_test.rb +1 -1
- data/test/{test/integration → integration}/partition_test.rb +6 -5
- data/test/{test/integration → integration}/proxy_test.rb +4 -3
- data/test/{test/integration → integration}/stats_test.rb +1 -1
- data/test/integration/subcommands.rb +128 -0
- data/test/{test/system → system}/flare_admin_test.rb +7 -5
- data/test/{test/unit → unit}/bwlimit_test.rb +0 -0
- data/test/{test/unit → unit}/cluster_test.rb +1 -1
- data/test/{test/unit → unit}/daemon_test.rb +0 -0
- data/test/{test/unit → unit}/logger_test.rb +0 -0
- data/test/{test/unit → unit}/tools_test.rb +0 -0
- data/test/unit/util/pretty_table_test.rb +46 -0
- data/test/{test/unit → unit}/util_test.rb +7 -2
- metadata +88 -100
- data/PostInstall.txt +0 -7
- data/lib/flare/tools/cli/cli_util.rb +0 -77
- data/lib/flare/util/command_line.rb +0 -79
@@ -5,11 +5,14 @@
|
|
5
5
|
|
6
6
|
require 'flare/util/logging'
|
7
7
|
require 'flare/util/interruption'
|
8
|
+
require 'flare/tools/cli/option'
|
8
9
|
|
9
10
|
module Flare
|
10
11
|
module Tools
|
11
12
|
module Cli
|
12
13
|
class SubCommand
|
14
|
+
include Flare::Tools::Cli::Option
|
15
|
+
|
13
16
|
@@myname = {}
|
14
17
|
@@desc = {}
|
15
18
|
@@usage = {}
|
@@ -52,14 +55,22 @@ module Flare
|
|
52
55
|
self.to_sym.to_s
|
53
56
|
end
|
54
57
|
|
55
|
-
def
|
58
|
+
def initialize
|
59
|
+
option_init
|
56
60
|
end
|
57
61
|
|
58
|
-
def setup
|
59
|
-
|
62
|
+
def setup
|
63
|
+
set_option_global
|
60
64
|
end
|
61
|
-
|
62
|
-
def
|
65
|
+
|
66
|
+
def execute_subcommand(config, args)
|
67
|
+
setup
|
68
|
+
rest_args = parse_options(config, args)
|
69
|
+
|
70
|
+
execute(config, rest_args)
|
71
|
+
end
|
72
|
+
|
73
|
+
def execute
|
63
74
|
raise "execute"
|
64
75
|
end
|
65
76
|
|
@@ -8,19 +8,18 @@ require 'flare/tools/index_server'
|
|
8
8
|
require 'flare/tools/cli/sub_command'
|
9
9
|
require 'flare/tools/common'
|
10
10
|
require 'flare/util/conversion'
|
11
|
+
require 'flare/tools/cli/index_server_config'
|
11
12
|
|
12
|
-
#
|
13
13
|
module Flare
|
14
14
|
module Tools
|
15
15
|
module Cli
|
16
16
|
|
17
|
-
# == Description
|
18
|
-
#
|
19
17
|
class Summary < SubCommand
|
20
18
|
include Flare::Util::Conversion
|
21
19
|
include Flare::Util::Logging
|
22
20
|
include Flare::Tools::Common
|
23
|
-
|
21
|
+
include Flare::Tools::Cli::IndexServerConfig
|
22
|
+
|
24
23
|
myname :summary
|
25
24
|
desc "show the summary of a flare cluster."
|
26
25
|
usage "summary [hostname:port] ..."
|
@@ -33,19 +32,22 @@ module Flare
|
|
33
32
|
['%16s', 'size'],
|
34
33
|
['%13s', 'items'] ]
|
35
34
|
|
36
|
-
def setup
|
35
|
+
def setup
|
36
|
+
super
|
37
|
+
set_option_index_server
|
37
38
|
end
|
38
39
|
|
39
40
|
def initialize
|
40
41
|
super
|
41
42
|
end
|
42
|
-
|
43
|
-
def execute(config,
|
43
|
+
|
44
|
+
def execute(config, args)
|
45
|
+
parse_index_server(config, args)
|
44
46
|
nodes = {}
|
45
47
|
threads = {}
|
46
48
|
header = HeaderConfig
|
47
49
|
|
48
|
-
Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port],
|
50
|
+
Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
|
49
51
|
nodes = s.stats_nodes
|
50
52
|
unless nodes
|
51
53
|
error "Invalid index server."
|
@@ -69,7 +71,7 @@ module Flare
|
|
69
71
|
|
70
72
|
nodes.each do |hostname_port,data|
|
71
73
|
hostname, port = hostname_port.split(":", 2)
|
72
|
-
Flare::Tools::Stats.open(hostname, data['port'],
|
74
|
+
Flare::Tools::Stats.open(hostname, data['port'], @timeout) do |s|
|
73
75
|
stats = s.stats
|
74
76
|
p = data['partition'].to_i
|
75
77
|
total_parts = p+1 if p+1 > total_parts if data['role'] == 'master'
|
@@ -85,7 +87,7 @@ module Flare
|
|
85
87
|
label = format % header.map{|x| x[1]}.flatten
|
86
88
|
puts label
|
87
89
|
puts(format % [name, total_parts, total_nodes, total_masters, total_slaves, total_bytes, total_items])
|
88
|
-
|
90
|
+
|
89
91
|
S_OK
|
90
92
|
end
|
91
93
|
|
@@ -7,6 +7,7 @@ require 'flare/tools/index_server'
|
|
7
7
|
require 'flare/util/conversion'
|
8
8
|
require 'flare/tools/common'
|
9
9
|
require 'flare/tools/cli/sub_command'
|
10
|
+
require 'flare/tools/cli/index_server_config'
|
10
11
|
|
11
12
|
module Flare
|
12
13
|
module Tools
|
@@ -14,18 +15,23 @@ module Flare
|
|
14
15
|
class Threads < SubCommand
|
15
16
|
include Flare::Util::Conversion
|
16
17
|
include Flare::Tools::Common
|
18
|
+
include Flare::Tools::Cli::IndexServerConfig
|
17
19
|
|
18
20
|
myname :threads
|
19
21
|
desc "show the list of threads in a flare cluster."
|
20
22
|
usage "threads [hostname:port]"
|
21
|
-
|
22
|
-
def setup
|
23
|
+
|
24
|
+
def setup
|
25
|
+
super
|
26
|
+
set_option_index_server
|
23
27
|
end
|
24
28
|
|
25
29
|
def initialize
|
30
|
+
super
|
26
31
|
end
|
27
32
|
|
28
|
-
def execute(config,
|
33
|
+
def execute(config, args)
|
34
|
+
parse_index_server(config, args)
|
29
35
|
header = [
|
30
36
|
['%5s', 'id'],
|
31
37
|
['%-32s', 'peer'],
|
@@ -49,14 +55,14 @@ module Flare
|
|
49
55
|
error "invalid arguments: "+args.join(' ')
|
50
56
|
return S_NG
|
51
57
|
end
|
52
|
-
|
58
|
+
|
53
59
|
threads = []
|
54
|
-
|
55
|
-
Flare::Tools::Stats.open(hostname, port,
|
60
|
+
|
61
|
+
Flare::Tools::Stats.open(hostname, port, @timeout) do |s|
|
56
62
|
threads = s.stats_threads
|
57
63
|
threads = threads.sort_by{|key,val| [val['peer'], key]}
|
58
64
|
end
|
59
|
-
|
65
|
+
|
60
66
|
puts format % header.map{|x| x[1]}.flatten
|
61
67
|
|
62
68
|
threads.each do |thread_id, data|
|
@@ -68,10 +74,10 @@ module Flare
|
|
68
74
|
data['queue'],
|
69
75
|
]
|
70
76
|
end
|
71
|
-
|
77
|
+
|
72
78
|
S_OK
|
73
79
|
end
|
74
|
-
|
80
|
+
|
75
81
|
end
|
76
82
|
end
|
77
83
|
end
|
@@ -9,26 +9,27 @@ require 'flare/util/conversion'
|
|
9
9
|
require 'flare/util/key_resolver'
|
10
10
|
require 'flare/util/hash_function'
|
11
11
|
require 'flare/tools/cli/sub_command'
|
12
|
+
require 'flare/tools/cli/index_server_config'
|
12
13
|
require 'digest/md5'
|
13
14
|
|
14
15
|
|
15
|
-
#
|
16
16
|
module Flare
|
17
17
|
module Tools
|
18
18
|
module Cli
|
19
19
|
|
20
|
-
# == Description
|
21
|
-
#
|
22
20
|
class Verify < SubCommand
|
23
21
|
include Flare::Util::Conversion
|
24
22
|
include Flare::Util::HashFunction
|
23
|
+
include Flare::Tools::Cli::IndexServerConfig
|
25
24
|
|
26
25
|
myname :verify
|
27
26
|
desc "verify the cluster. (experimental)"
|
28
27
|
usage "verify"
|
29
|
-
|
30
|
-
def setup
|
31
|
-
|
28
|
+
|
29
|
+
def setup
|
30
|
+
super
|
31
|
+
set_option_index_server
|
32
|
+
@optp.on('--key-hash-algorithm=TYPE', "key hash algorithm") do |v|
|
32
33
|
case @key_hash_algorithm = v.to_sym
|
33
34
|
when :simple, :crc32
|
34
35
|
else
|
@@ -36,12 +37,12 @@ module Flare
|
|
36
37
|
exit
|
37
38
|
end
|
38
39
|
end
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
@optp.on('--use-test-data', "store test data") {|v| @use_test_data = true}
|
41
|
+
@optp.on('--debug', "use debug mode") {|v| @debug = true}
|
42
|
+
@optp.on('--64bit', "(experimental) 64bit mode") {|v| @word_size = 64}
|
43
|
+
@optp.on('--verbose', "use verbose mode") {|v| @verbose = true}
|
44
|
+
@optp.on('--meta', "use meta command") {|v| @meta = true}
|
45
|
+
@optp.on('--quiet', "use quiet mode") {|v| @quiet = true}
|
45
46
|
end
|
46
47
|
|
47
48
|
def initialize
|
@@ -57,12 +58,13 @@ module Flare
|
|
57
58
|
@quiet = false
|
58
59
|
end
|
59
60
|
|
60
|
-
def execute(config,
|
61
|
+
def execute(config, args)
|
62
|
+
parse_index_server(config, args)
|
61
63
|
keys = {}
|
62
64
|
cout = STDERR
|
63
65
|
status = S_OK
|
64
66
|
info "connecting to index ..."
|
65
|
-
Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port],
|
67
|
+
Flare::Tools::IndexServer.open(config[:index_server_hostname], config[:index_server_port], @timeout) do |s|
|
66
68
|
nodes = s.stats_nodes.sort_by{|key, val| [val['partition'].to_i, val['role'], key]}
|
67
69
|
|
68
70
|
# meta
|
@@ -111,7 +113,7 @@ module Flare
|
|
111
113
|
|
112
114
|
if @use_test_data
|
113
115
|
info "storing test data ..."
|
114
|
-
Flare::Tools::Node.open(hostname0, port0.to_i,
|
116
|
+
Flare::Tools::Node.open(hostname0, port0.to_i, @timeout) do |n|
|
115
117
|
(1..10000).each do |i|
|
116
118
|
key = ".test."+Digest::MD5.new.update(i.to_s).to_s
|
117
119
|
n.set(key, i.to_s)
|
@@ -123,7 +125,7 @@ module Flare
|
|
123
125
|
nodes.each do |nodekey,val|
|
124
126
|
hostname, port = nodekey.split(":", 2)
|
125
127
|
partition = val['partition'].to_i
|
126
|
-
Flare::Tools::Node.open(hostname, port.to_i,
|
128
|
+
Flare::Tools::Node.open(hostname, port.to_i, @timeout) do |n|
|
127
129
|
cout.write "checking #{nodekey} ... "
|
128
130
|
msg = "OK"
|
129
131
|
interruptible do
|
@@ -178,7 +180,7 @@ module Flare
|
|
178
180
|
error "failed: not found #{remain} keys" if remain > 0
|
179
181
|
|
180
182
|
# delete
|
181
|
-
Flare::Tools::Node.open(hostname0, port0.to_i,
|
183
|
+
Flare::Tools::Node.open(hostname0, port0.to_i, @timeout) do |n|
|
182
184
|
keys.each do |k,v|
|
183
185
|
n.delete(k)
|
184
186
|
end
|
@@ -194,7 +196,7 @@ module Flare
|
|
194
196
|
end
|
195
197
|
status
|
196
198
|
end
|
197
|
-
|
199
|
+
|
198
200
|
end
|
199
201
|
end
|
200
202
|
end
|
data/lib/flare/tools/cluster.rb
CHANGED
@@ -128,8 +128,8 @@ module Flare
|
|
128
128
|
true
|
129
129
|
when RoleMaster
|
130
130
|
# if the partition has at least one active slave, one of the slaves will take over the master.
|
131
|
-
slaves_in_partition(node[StatPartition]).inject(false) do |r,slave_nodekey|
|
132
|
-
node_stat(slave_nodekey)[State] == StateActive
|
131
|
+
slaves_in_partition(node[StatPartition]).inject(false) do |r, slave_nodekey|
|
132
|
+
r || node_stat(slave_nodekey)[State] == StateActive
|
133
133
|
end
|
134
134
|
else
|
135
135
|
error "unknown role: #{node[Role]}"
|
@@ -252,6 +252,7 @@ module Flare
|
|
252
252
|
<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
|
253
253
|
<!DOCTYPE boost_serialization>
|
254
254
|
<boost_serialization signature="serialization::archive" version="4">
|
255
|
+
<version>0</version>
|
255
256
|
<node_map#{serattr_(node_map_id)}>
|
256
257
|
\t<count>#{@nodes.size}</count>
|
257
258
|
\t<item_version>0</item_version>
|
data/lib/flare/util/constant.rb
CHANGED
@@ -0,0 +1,8 @@
|
|
1
|
+
# -*- coding: utf-8; -*-
|
2
|
+
# Authors:: Yuya YAGUCHI <yuya.yaguchi@gree.net>
|
3
|
+
# Copyright:: Copyright (C) GREE, Inc. 2014.
|
4
|
+
# License:: MIT-style
|
5
|
+
|
6
|
+
require 'flare/util/pretty_table/table.rb'
|
7
|
+
require 'flare/util/pretty_table/column.rb'
|
8
|
+
require 'flare/util/pretty_table/row.rb'
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# -*- coding: utf-8; -*-
|
2
|
+
# Authors:: Yuya YAGUCHI <yuya.yaguchi@gree.net>
|
3
|
+
# Copyright:: Copyright (C) GREE, Inc. 2014.
|
4
|
+
# License:: MIT-style
|
5
|
+
|
6
|
+
module Flare
|
7
|
+
module Util
|
8
|
+
module PrettyTable
|
9
|
+
class Column
|
10
|
+
def initialize(text, options = {})
|
11
|
+
@text = text.to_s
|
12
|
+
@align = options[:align] || :left
|
13
|
+
end
|
14
|
+
|
15
|
+
def width
|
16
|
+
@text.size
|
17
|
+
end
|
18
|
+
|
19
|
+
def prettify(column_width)
|
20
|
+
if self.width >= column_width
|
21
|
+
return @text
|
22
|
+
end
|
23
|
+
|
24
|
+
padding = padding(column_width - self.width)
|
25
|
+
case @align
|
26
|
+
when :left then
|
27
|
+
@text + padding
|
28
|
+
else # @align == :right
|
29
|
+
padding + @text
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def padding(length)
|
36
|
+
' ' * length
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- coding: utf-8; -*-
|
2
|
+
# Authors:: Yuya YAGUCHI <yuya.yaguchi@gree.net>
|
3
|
+
# Copyright:: Copyright (C) GREE, Inc. 2014.
|
4
|
+
# License:: MIT-style
|
5
|
+
|
6
|
+
module Flare
|
7
|
+
module Util
|
8
|
+
module PrettyTable
|
9
|
+
class Row
|
10
|
+
attr_reader :columns
|
11
|
+
|
12
|
+
def initialize(option = {})
|
13
|
+
@columns = []
|
14
|
+
@separator = option[:separator] || ' '
|
15
|
+
end
|
16
|
+
|
17
|
+
def add_column(column)
|
18
|
+
@columns << column
|
19
|
+
end
|
20
|
+
|
21
|
+
def prettify(column_widths)
|
22
|
+
@columns \
|
23
|
+
.each_with_index.map {|column, index| column.prettify(column_widths[index]) } \
|
24
|
+
.join(@separator)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# -*- coding: utf-8; -*-
|
2
|
+
# Authors:: Yuya YAGUCHI <yuya.yaguchi@gree.net>
|
3
|
+
# Copyright:: Copyright (C) GREE, Inc. 2014.
|
4
|
+
# License:: MIT-style
|
5
|
+
|
6
|
+
module Flare
|
7
|
+
module Util
|
8
|
+
module PrettyTable
|
9
|
+
class Table
|
10
|
+
def initialize
|
11
|
+
@rows = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def add_row(row)
|
15
|
+
@rows << row
|
16
|
+
end
|
17
|
+
|
18
|
+
def prettify
|
19
|
+
column_widths = max_column_widths
|
20
|
+
@rows \
|
21
|
+
.map {|row| row.prettify(column_widths) } \
|
22
|
+
.join("\n")
|
23
|
+
end
|
24
|
+
|
25
|
+
def max_column_widths
|
26
|
+
widths = []
|
27
|
+
@rows.each do |row|
|
28
|
+
row.columns.each_with_index do |column, index|
|
29
|
+
widths[index] = [column.width, (widths[index] || 0)].max
|
30
|
+
end
|
31
|
+
end
|
32
|
+
widths
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/package/Rakefile
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
|
2
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
|
3
|
+
|
4
|
+
require 'flare/tools'
|
5
|
+
|
6
|
+
flaretools = "flare-tools"
|
7
|
+
|
8
|
+
task :default => [:build]
|
9
|
+
|
10
|
+
task :build do |t|
|
11
|
+
sh "(cd #{flaretools} && debuild -us -uc)"
|
12
|
+
end
|
13
|
+
|
14
|
+
task :debclean do
|
15
|
+
sh "(cd #{flaretools} && debclean)"
|
16
|
+
end
|
17
|
+
|
18
|
+
task :change do
|
19
|
+
puts "================================="
|
20
|
+
puts " Flare::Tools::VERSION = #{Flare::Tools::VERSION}"
|
21
|
+
puts "================================="
|
22
|
+
sh "(cd #{flaretools} && dch -i)"
|
23
|
+
end
|
24
|
+
|
25
|
+
def previous version
|
26
|
+
prev = version.split('.').map{|v| v.to_i}
|
27
|
+
prev[2] -= 1
|
28
|
+
prev.join('.')
|
29
|
+
end
|
30
|
+
|
31
|
+
task :autochange do
|
32
|
+
v = Flare::Tools::VERSION
|
33
|
+
since = previous(v)
|
34
|
+
sh "git-dch --debian-branch='(no branch)' --new-version #{version} --since=#{since}"
|
35
|
+
end
|
36
|
+
|
37
|
+
task :clean => [:debclean] do
|
38
|
+
sh "rm -f *.dsc *.tar.gz *.build *.changes"
|
39
|
+
end
|
40
|
+
|
41
|
+
task :cleanall => [:clean] do
|
42
|
+
sh "rm -f flare-tools/debian/changelog.dch"
|
43
|
+
end
|
44
|
+
|