kcluster 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/kcluster +255 -0
- data/lib/nothing.rb +0 -0
- metadata +83 -0
data/bin/kcluster
ADDED
@@ -0,0 +1,255 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.push(File.dirname($0))
|
4
|
+
require 'optparse'
|
5
|
+
require 'socket'
|
6
|
+
require 'net/http'
|
7
|
+
require 'json'
|
8
|
+
|
9
|
+
# FIXME this really only works inside twitter. it would be nice to support
|
10
|
+
# a general serverset interface (or opensource colony).
|
11
|
+
$colony_host = ENV["COLONY_HOST"] || "colony.smf1.twitter.com"
|
12
|
+
$colony_port = ENV["COLONY_PORT"] || 9080
|
13
|
+
|
14
|
+
$options = {
|
15
|
+
:config_filename => ENV['HOME'] + "/.kestrel_cluster",
|
16
|
+
:server_list => [],
|
17
|
+
:port => 22133,
|
18
|
+
:verbose => false,
|
19
|
+
}
|
20
|
+
|
21
|
+
def verbose(s)
|
22
|
+
puts s if $options[:verbose]
|
23
|
+
end
|
24
|
+
|
25
|
+
def fetch_stats(host, port, data)
|
26
|
+
verbose "--- Fetching stats from #{host}:#{port}"
|
27
|
+
sock = TCPSocket.open(host, port)
|
28
|
+
sock.puts("stats")
|
29
|
+
done = false
|
30
|
+
while !done && line = sock.gets.chomp
|
31
|
+
if (line == 'END') then
|
32
|
+
done = true
|
33
|
+
elsif line =~ /STAT queue_([\w\+\-]+) (\d+)/
|
34
|
+
key = $1
|
35
|
+
value = $2.to_i
|
36
|
+
(stat, queue_name) = case key
|
37
|
+
when /([\w\+\-]+)_total_items/ then [:total_items, $1]
|
38
|
+
when /([\w\+\-]+)_expired_items/ then [:expired_items, $1]
|
39
|
+
when /([\w\+\-]+)_mem_items/ then [:mem_items, $1]
|
40
|
+
when /([\w\+\-]+)_items/ then [:items, $1]
|
41
|
+
when /([\w\+\-]+)_mem_bytes/ then [:mem_bytes, $1]
|
42
|
+
when /([\w\+\-]+)_bytes/ then [:bytes, $1]
|
43
|
+
when /([\w\+\-]+)_age/ then [:age, $1]
|
44
|
+
end
|
45
|
+
|
46
|
+
if (queue_name)
|
47
|
+
queue_name = queue_name.split('+', 2).first if $options[:rollup_fanouts]
|
48
|
+
|
49
|
+
if (stat == :age)
|
50
|
+
data[:min_age][queue_name] = value if value < data[:min_age][queue_name]
|
51
|
+
data[:max_age][queue_name] = value if value > data[:max_age][queue_name]
|
52
|
+
else
|
53
|
+
data[stat][queue_name] += value
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
sock.close
|
59
|
+
end
|
60
|
+
|
61
|
+
def fetch_all
|
62
|
+
data = Hash.new do |h, data_type|
|
63
|
+
h[data_type] = Hash.new do |h, k|
|
64
|
+
h[k] = (data_type == :min_age) ? 2**31 : 0
|
65
|
+
end
|
66
|
+
end
|
67
|
+
$options[:server_list].each do |server|
|
68
|
+
begin
|
69
|
+
fetch_stats(server, $options[:port], data)
|
70
|
+
rescue => e
|
71
|
+
puts "Could not connect to host #{server}: #{e}"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
data
|
75
|
+
end
|
76
|
+
|
77
|
+
def sort_data(data)
|
78
|
+
rv = {}
|
79
|
+
data.each { |key, v| rv[key] = v.sort_by { |h| [ h[1], h[0] ] } }
|
80
|
+
rv
|
81
|
+
end
|
82
|
+
|
83
|
+
def report(data, key)
|
84
|
+
puts ""
|
85
|
+
format = "%14s %s\n"
|
86
|
+
printf(format, key, "queue")
|
87
|
+
printf(format, "============", "====================")
|
88
|
+
stats = data[key] || {}
|
89
|
+
stats.each { |queue, value| printf("%14d %s\n", value, queue) }
|
90
|
+
end
|
91
|
+
|
92
|
+
def report_all(data, keys)
|
93
|
+
keys.each { |key| report(data, key) }
|
94
|
+
puts ""
|
95
|
+
end
|
96
|
+
|
97
|
+
def delete_all(queue_name)
|
98
|
+
$options[:server_list].each do |server|
|
99
|
+
print "--- Deleting queue #{queue_name} from #{server}:#{$options[:port]} ... "
|
100
|
+
STDOUT.flush
|
101
|
+
sock = TCPSocket.open(server, $options[:port])
|
102
|
+
sock.puts("delete " + queue_name)
|
103
|
+
puts sock.readline.chomp
|
104
|
+
sock.close
|
105
|
+
end
|
106
|
+
puts "Done."
|
107
|
+
end
|
108
|
+
|
109
|
+
def flush_all(queue_name)
|
110
|
+
$options[:server_list].each do |server|
|
111
|
+
print "--- Flushing queue #{queue_name} from #{server}:#{$options[:port]} ... "
|
112
|
+
STDOUT.flush
|
113
|
+
sock = TCPSocket.open(server, $options[:port])
|
114
|
+
sock.puts("flush " + queue_name)
|
115
|
+
puts sock.readline.chomp
|
116
|
+
sock.close
|
117
|
+
end
|
118
|
+
puts "Done."
|
119
|
+
end
|
120
|
+
|
121
|
+
def keep_unchanged_data(last, current)
|
122
|
+
rv = {}
|
123
|
+
current.keys.each { |key| rv[key] = keep_unchanged_hash(last[key], current[key]) }
|
124
|
+
rv
|
125
|
+
end
|
126
|
+
|
127
|
+
def keep_unchanged_hash(last, current)
|
128
|
+
rv = {}
|
129
|
+
last.each do |key, value|
|
130
|
+
rv[key] = value if current[key] == value
|
131
|
+
end
|
132
|
+
rv
|
133
|
+
end
|
134
|
+
|
135
|
+
# dump queues that appear to have neither :items nor :total_items changing.
|
136
|
+
def find_stale(rounds)
|
137
|
+
puts "Check 1..."
|
138
|
+
last = fetch_all
|
139
|
+
(1...rounds).each do |i|
|
140
|
+
sleep 5
|
141
|
+
puts "Check #{i + 1}..."
|
142
|
+
current = fetch_all
|
143
|
+
last = keep_unchanged_data(last, current)
|
144
|
+
end
|
145
|
+
|
146
|
+
stale = (last[:total_items].keys & last[:items].keys).sort!
|
147
|
+
sorted = sort_data(last)
|
148
|
+
|
149
|
+
puts ""
|
150
|
+
puts "stale queue prediction"
|
151
|
+
puts "======================"
|
152
|
+
printf("%11s %11s %s\n", "total_items", "items", "queue")
|
153
|
+
printf("%11s %11s %s\n", "-----------", "-----------", "--------------------")
|
154
|
+
stale.each do |queue_name|
|
155
|
+
items = last[:items][queue_name]
|
156
|
+
total_items = last[:total_items][queue_name]
|
157
|
+
printf("%11d %11d %s\n", total_items, items, queue_name)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
|
162
|
+
# ----------
|
163
|
+
|
164
|
+
parser = OptionParser.new do |opts|
|
165
|
+
opts.banner = "Usage: #{$0} [options] <command>"
|
166
|
+
opts.separator "Example: #{$0} -f kestrel max_age"
|
167
|
+
|
168
|
+
opts.on("-f", "--file=FILENAME", "load kestrel server list from file (use '-' for stdin) (default: #{$options[:config_filename]})") do |filename|
|
169
|
+
$options[:config_filename] = filename
|
170
|
+
end
|
171
|
+
opts.on("-p", "--port=N", "use port (default: #{$options[:port]})") do |port|
|
172
|
+
$options[:port] = port.to_i
|
173
|
+
end
|
174
|
+
opts.on("-r", "--role=ROLE", "load kestrel server list from colony audubon role (requires colony)") do |role|
|
175
|
+
$options[:role] = role
|
176
|
+
end
|
177
|
+
opts.on("-v", "--verbose", "print verbose debugging info as we work") do
|
178
|
+
$options[:verbose] = true
|
179
|
+
end
|
180
|
+
opts.on("-F", "--rollup-fanouts", "roll up stats for fanout queues into a single count") do
|
181
|
+
$options[:rollup_fanouts] = true
|
182
|
+
end
|
183
|
+
|
184
|
+
opts.separator ""
|
185
|
+
opts.separator "Commands:"
|
186
|
+
opts.separator " all show all of the stats summaries below (items, etc)"
|
187
|
+
opts.separator " items show item counts per queue"
|
188
|
+
opts.separator " bytes show byte counts per queue"
|
189
|
+
opts.separator " mem show in-memory byte counts per queue"
|
190
|
+
opts.separator " min_age show minimum age (msec) per queue"
|
191
|
+
opts.separator " max_age show maximum age (msec) per queue"
|
192
|
+
opts.separator ""
|
193
|
+
opts.separator " delete <name> remove a queue across the cluster"
|
194
|
+
opts.separator " flush <name> drain a queue across the cluster"
|
195
|
+
opts.separator " stale [rounds] try to detect stale queues (default: 5 rounds)"
|
196
|
+
opts.separator ""
|
197
|
+
end
|
198
|
+
|
199
|
+
parser.parse!(ARGV)
|
200
|
+
|
201
|
+
if ARGV.size < 1
|
202
|
+
puts
|
203
|
+
puts parser
|
204
|
+
exit 1
|
205
|
+
end
|
206
|
+
|
207
|
+
if $options[:role]
|
208
|
+
data = Net::HTTP.get URI.parse("http://#{$colony_host}:#{$colony_port}/query?q=mo+audubon.role.#{$options[:role]}")
|
209
|
+
$options[:server_list] = JSON.parse(data).values.flatten
|
210
|
+
else
|
211
|
+
server_file = $options[:config_filename] == "-" ? STDIN : File.open($options[:config_filename], "r")
|
212
|
+
$options[:server_list] = server_file.readlines.map { |line| line.chomp }
|
213
|
+
server_file.close
|
214
|
+
end
|
215
|
+
|
216
|
+
command = ARGV[0].downcase
|
217
|
+
|
218
|
+
if command == "all"
|
219
|
+
$show = [ :items, :bytes, :mem_bytes, :min_age, :max_age ]
|
220
|
+
elsif command == "items"
|
221
|
+
$show = [ :items ]
|
222
|
+
elsif command == "bytes"
|
223
|
+
$show = [ :bytes ]
|
224
|
+
elsif command == "mem"
|
225
|
+
$show = [ :mem_bytes ]
|
226
|
+
elsif command == "min_age"
|
227
|
+
$show = [ :min_age ]
|
228
|
+
elsif command == "max_age"
|
229
|
+
$show = [ :max_age ]
|
230
|
+
elsif command == "delete" && ARGV.size == 2
|
231
|
+
queue_name = ARGV[1]
|
232
|
+
print "Really delete #{queue_name}? (yes/no) "
|
233
|
+
STDOUT.flush
|
234
|
+
delete_all(queue_name) if STDIN.gets.chomp == 'yes'
|
235
|
+
exit 0
|
236
|
+
elsif command == "flush" && ARGV.size == 2
|
237
|
+
queue_name = ARGV[1]
|
238
|
+
print "Really flush #{queue_name}? (yes/no) "
|
239
|
+
STDOUT.flush
|
240
|
+
flush_all(queue_name) if STDIN.gets.chomp == 'yes'
|
241
|
+
exit 0
|
242
|
+
elsif command == "stale"
|
243
|
+
rounds = ARGV[1] ? ARGV[1].to_i : 5
|
244
|
+
find_stale(rounds)
|
245
|
+
exit 0
|
246
|
+
end
|
247
|
+
|
248
|
+
if $show
|
249
|
+
report_all(sort_data(fetch_all), $show)
|
250
|
+
exit 0
|
251
|
+
else
|
252
|
+
puts
|
253
|
+
puts parser
|
254
|
+
exit 1
|
255
|
+
end
|
data/lib/nothing.rb
ADDED
File without changes
|
metadata
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: kcluster
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 19
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 1
|
9
|
+
- 0
|
10
|
+
version: 1.1.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Robey Pointer
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-03-02 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: json
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 3
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
version: "0"
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
description: |
|
35
|
+
A handy little CLI tool for browsing quick stats across your kestrel cluster.
|
36
|
+
Kcluster collects stats from each machine in a cluster, then summarizes them
|
37
|
+
into one table.
|
38
|
+
|
39
|
+
email: robeypointer@gmail.com
|
40
|
+
executables:
|
41
|
+
- kcluster
|
42
|
+
extensions: []
|
43
|
+
|
44
|
+
extra_rdoc_files: []
|
45
|
+
|
46
|
+
files:
|
47
|
+
- lib/nothing.rb
|
48
|
+
- bin/kcluster
|
49
|
+
homepage: http://robey.github.com/kestrel
|
50
|
+
licenses: []
|
51
|
+
|
52
|
+
post_install_message:
|
53
|
+
rdoc_options: []
|
54
|
+
|
55
|
+
require_paths:
|
56
|
+
- lib
|
57
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
hash: 3
|
63
|
+
segments:
|
64
|
+
- 0
|
65
|
+
version: "0"
|
66
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
67
|
+
none: false
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
hash: 3
|
72
|
+
segments:
|
73
|
+
- 0
|
74
|
+
version: "0"
|
75
|
+
requirements: []
|
76
|
+
|
77
|
+
rubyforge_project:
|
78
|
+
rubygems_version: 1.8.15
|
79
|
+
signing_key:
|
80
|
+
specification_version: 3
|
81
|
+
summary: Kestrel cluster management tool
|
82
|
+
test_files: []
|
83
|
+
|