nchan_tools 0.1.1 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/nchan-benchmark +30 -186
- data/exe/nchan-pub +1 -1
- data/exe/nchan-redis-debug +63 -0
- data/exe/nchan-sub +3 -3
- data/lib/nchan_tools/benchmark.rb +241 -0
- data/lib/nchan_tools/pubsub.rb +37 -24
- data/lib/nchan_tools/rdsck.rb +111 -0
- data/lib/nchan_tools/version.rb +1 -1
- data/nchan_tools.gemspec +3 -3
- metadata +10 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 07a8230e83733a4948ee9cc9df50eea8b698f2762e7f70a054f9708c98bb7ea1
|
4
|
+
data.tar.gz: 16ab41364f02c7996441a8f461bd969d9c11af023936e64f0be1f912f81ece88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f2f505b79a22588f37f38f12aa0c461cf5d9db9f3c89375f6a70a9a0df3b8a62f2b778425fbb71bc4b7eb9b8b4fdf275c08cf365dd9d89b57e5396ed5312e1e
|
7
|
+
data.tar.gz: 6c4bfe8a215370d37319144375b633bbbb868d1f50ac4dc1f6c2736dce057f78edd30e3995d7be34a5534d80f0ee4cd50f5d0633b63da085c28620c24751a979
|
data/exe/nchan-benchmark
CHANGED
@@ -1,21 +1,44 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'securerandom'
|
4
|
-
require 'nchan_tools/
|
4
|
+
require 'nchan_tools/benchmark'
|
5
5
|
require "optparse"
|
6
6
|
require 'timers'
|
7
7
|
require 'json'
|
8
|
-
require "pry"
|
9
8
|
require "HDRHistogram"
|
10
9
|
|
11
10
|
verbose = false
|
11
|
+
save_csv = false
|
12
|
+
csv_columns = NchanTools::Benchmark::CSV_COLUMNS_DEFAULT
|
13
|
+
init_args = {}
|
12
14
|
|
13
15
|
opt_parser=OptionParser.new do |opts|
|
14
16
|
opts.on("-v", "--verbose", "somewhat rather extraneously wordful output") do
|
15
17
|
verbose = true
|
16
18
|
end
|
19
|
+
opts.on("--csv FILENAME", "Append results to file in CSV format") do |f|
|
20
|
+
save_csv = f
|
21
|
+
end
|
22
|
+
opts.on("--csv-columns col1,col2,...", "csv columns list") do |f|
|
23
|
+
csv_columns = f.split(/\W+/).map(&:to_sym)
|
24
|
+
end
|
25
|
+
opts.on("-t", "--time TIME", "Time to run benchmark") do |v|
|
26
|
+
init_args[:time] = v
|
27
|
+
end
|
28
|
+
opts.on("-r", "--msgrate NUMBER", "Message publishing rate per minute per channel") do |v|
|
29
|
+
init_args[:messages_per_channel_per_minute] = v
|
30
|
+
end
|
31
|
+
opts.on("-p", "--msgpadding NUMBER", "Message padding, in bytes") do |v|
|
32
|
+
init_args[:message_padding_bytes] = v
|
33
|
+
end
|
34
|
+
opts.on("-c", "--channels NUMBER", "Number of channels") do |v|
|
35
|
+
init_args[:channels] = v
|
36
|
+
end
|
37
|
+
opts.on("-s", "--subscribers NUMBER", "Subscribers per channel") do |v|
|
38
|
+
init_args[:subscribers_per_channel] = v
|
39
|
+
end
|
17
40
|
end
|
18
|
-
opt_parser.banner="Usage:
|
41
|
+
opt_parser.banner="Usage: nchan-benchmark [options] url1 url2 url3..."
|
19
42
|
opt_parser.parse!
|
20
43
|
|
21
44
|
urls = []
|
@@ -23,192 +46,13 @@ urls += ARGV
|
|
23
46
|
begin
|
24
47
|
urls += STDIN.read_nonblock(100000).split /\s*\n+\s*/
|
25
48
|
rescue IO::WaitReadable
|
49
|
+
rescue EOFError
|
26
50
|
end
|
27
51
|
|
28
52
|
urls.uniq!
|
29
53
|
|
30
|
-
|
31
|
-
def initialize(urls)
|
32
|
-
@urls = urls
|
33
|
-
@n = urls.count
|
34
|
-
@initializing = 0
|
35
|
-
@ready = 0
|
36
|
-
@running = 0
|
37
|
-
@finished = 0
|
38
|
-
@subs = []
|
39
|
-
@results = {}
|
40
|
-
@failed = {}
|
41
|
-
|
42
|
-
@hdrh_publish = nil
|
43
|
-
@hdrh_receive = nil
|
44
|
-
|
45
|
-
subs = []
|
46
|
-
end
|
47
|
-
|
48
|
-
def run
|
49
|
-
puts "connecting to #{@n} Nchan server#{@n > 1 ? "s" : ""}..."
|
50
|
-
@urls.each do |url|
|
51
|
-
sub = Subscriber.new(url, 1, client: :websocket, timeout: 900000, extra_headers: {"Accept" => "text/x-json-hdrhistogram"})
|
52
|
-
sub.on_failure do |err|
|
53
|
-
unless @results[sub]
|
54
|
-
unless @results[sub.url]
|
55
|
-
@failed[sub] = true
|
56
|
-
abort err, sub
|
57
|
-
end
|
58
|
-
end
|
59
|
-
false
|
60
|
-
end
|
61
|
-
sub.on_message do |msg|
|
62
|
-
msg = msg.to_s
|
63
|
-
case msg
|
64
|
-
when "READY"
|
65
|
-
puts " #{sub.url} ok"
|
66
|
-
@ready +=1
|
67
|
-
if @ready == @n
|
68
|
-
control :run
|
69
|
-
puts "start benchmark..."
|
70
|
-
end
|
71
|
-
when "RUNNING"
|
72
|
-
puts " #{sub.url} running"
|
73
|
-
when /^RESULTS\n/
|
74
|
-
msg = msg[8..-1]
|
75
|
-
parsed = JSON.parse msg
|
76
|
-
@results[sub.url] = parsed
|
77
|
-
1+1
|
78
|
-
else
|
79
|
-
binding.pry
|
80
|
-
1+1
|
81
|
-
end
|
82
|
-
end
|
83
|
-
@subs << sub
|
84
|
-
sub.run
|
85
|
-
sub.wait :ready, 1
|
86
|
-
if @failed[sub]
|
87
|
-
puts " #{sub.url} failed"
|
88
|
-
else
|
89
|
-
puts " #{sub.url} ok"
|
90
|
-
end
|
91
|
-
end
|
92
|
-
return if @failed.count > 0
|
93
|
-
puts "initializing benchmark..."
|
94
|
-
control :initialize
|
95
|
-
self.wait
|
96
|
-
puts "finished."
|
97
|
-
puts ""
|
98
|
-
end
|
99
|
-
|
100
|
-
def wait
|
101
|
-
@subs.each &:wait
|
102
|
-
end
|
103
|
-
|
104
|
-
def control(msg)
|
105
|
-
@subs.each { |sub| sub.client.send_data msg.to_s }
|
106
|
-
end
|
107
|
-
|
108
|
-
def abort(err, src_sub = nil)
|
109
|
-
puts " #{err}"
|
110
|
-
@subs.each do |sub|
|
111
|
-
sub.terminate unless sub == src_sub
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
def hdrhistogram_stats(name, histogram)
|
116
|
-
fmt = <<-END.gsub(/^ {6}/, '')
|
117
|
-
%s
|
118
|
-
min: %.3fms
|
119
|
-
avg: %.3fms
|
120
|
-
99%%ile: %.3fms
|
121
|
-
max: %.3fms
|
122
|
-
stddev: %.3fms
|
123
|
-
samples: %d
|
124
|
-
END
|
125
|
-
fmt % [ name,
|
126
|
-
histogram.min, histogram.mean, histogram.percentile(99.0), histogram.max, histogram.stddev, histogram.count
|
127
|
-
]
|
128
|
-
end
|
129
|
-
|
130
|
-
def results
|
131
|
-
channels = 0
|
132
|
-
runtime = []
|
133
|
-
subscribers = 0
|
134
|
-
message_length = []
|
135
|
-
messages_sent = 0
|
136
|
-
messages_send_confirmed = 0
|
137
|
-
messages_send_unconfirmed = 0
|
138
|
-
messages_send_failed = 0
|
139
|
-
messages_received = 0
|
140
|
-
messages_unreceived = 0
|
141
|
-
hdrh_publish = nil
|
142
|
-
hdrh_receive = nil
|
143
|
-
@results.each do |url, data|
|
144
|
-
channels += data["channels"]
|
145
|
-
runtime << data["run_time_sec"]
|
146
|
-
subscribers += data["subscribers"]
|
147
|
-
message_length << data["message_length"]
|
148
|
-
messages_sent += data["messages"]["sent"]
|
149
|
-
messages_send_confirmed += data["messages"]["send_confirmed"]
|
150
|
-
messages_send_unconfirmed += data["messages"]["send_unconfirmed"]
|
151
|
-
messages_send_failed += data["messages"]["send_failed"]
|
152
|
-
messages_received += data["messages"]["received"]
|
153
|
-
messages_unreceived += data["messages"]["unreceived"]
|
154
|
-
|
155
|
-
if data["message_publishing_histogram"]
|
156
|
-
hdrh = HDRHistogram.unserialize(data["message_publishing_histogram"], unit: :ms, multiplier: 0.001)
|
157
|
-
if hdrh_publish
|
158
|
-
hdrh_publish.merge! hdrh
|
159
|
-
else
|
160
|
-
hdrh_publish = hdrh
|
161
|
-
end
|
162
|
-
end
|
163
|
-
if data["message_delivery_histogram"]
|
164
|
-
hdrh = HDRHistogram.unserialize(data["message_delivery_histogram"], unit: :ms, multiplier: 0.001)
|
165
|
-
if hdrh_receive
|
166
|
-
hdrh_receive.merge! hdrh
|
167
|
-
else
|
168
|
-
hdrh_receive = hdrh
|
169
|
-
end
|
170
|
-
end
|
171
|
-
end
|
172
|
-
|
173
|
-
message_length.uniq!
|
174
|
-
runtime.uniq!
|
175
|
-
|
176
|
-
fmt = <<-END.gsub(/^ {6}/, '')
|
177
|
-
Nchan servers: %d
|
178
|
-
runtime: %s
|
179
|
-
channels: %d
|
180
|
-
subscribers: %d
|
181
|
-
subscribers per channel: %.1f
|
182
|
-
messages:
|
183
|
-
length: %s
|
184
|
-
sent: %d
|
185
|
-
send_confirmed: %d
|
186
|
-
send_unconfirmed: %d
|
187
|
-
send_failed: %d
|
188
|
-
received: %d
|
189
|
-
unreceived: %d
|
190
|
-
send rate: %.3f/sec
|
191
|
-
receive rate: %.3f/sec
|
192
|
-
send rate per channel: %.3f/min
|
193
|
-
receive rate per subscriber: %.3f/min
|
194
|
-
END
|
195
|
-
out = fmt % [
|
196
|
-
@n, runtime.join(","), channels, subscribers, subscribers.to_f/channels,
|
197
|
-
message_length.join(","), messages_sent, messages_send_confirmed, messages_send_unconfirmed, messages_send_failed,
|
198
|
-
messages_received, messages_unreceived,
|
199
|
-
messages_sent.to_f/runtime.max,
|
200
|
-
messages_received.to_f/runtime.max,
|
201
|
-
(messages_sent.to_f* 60)/(runtime.max*channels),
|
202
|
-
(messages_received.to_f * 60)/(runtime.max * subscribers)
|
203
|
-
]
|
204
|
-
|
205
|
-
out << hdrhistogram_stats("message publishing latency", hdrh_publish) if hdrh_publish
|
206
|
-
out << hdrhistogram_stats("message delivery latency", hdrh_receive) if hdrh_receive
|
207
|
-
|
208
|
-
puts out
|
209
|
-
end
|
210
|
-
end
|
211
|
-
|
212
|
-
benchan = Benchan.new urls
|
54
|
+
benchan = NchanTools::Benchmark.new urls, init_args
|
213
55
|
benchan.run
|
214
56
|
benchan.results
|
57
|
+
benchan.append_csv_file(save_csv, csv_columns) if save_csv
|
58
|
+
|
data/exe/nchan-pub
CHANGED
@@ -54,7 +54,7 @@ puts "Publishing to #{url}."
|
|
54
54
|
|
55
55
|
loopmsg=("\r"*20) + "sending message #"
|
56
56
|
|
57
|
-
pub = Publisher.new url, nostore: true, timeout: timeout, verbose: verbose, websocket: websocket
|
57
|
+
pub = NchanTools::Publisher.new url, nostore: true, timeout: timeout, verbose: verbose, websocket: websocket
|
58
58
|
pub.accept=accept
|
59
59
|
pub.nofail=true
|
60
60
|
repeat=true
|
@@ -0,0 +1,63 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "redis"
|
4
|
+
require "optparse"
|
5
|
+
require 'nchan_tools/rdsck'
|
6
|
+
|
7
|
+
$opt = {
|
8
|
+
url: "redis://127.0.0.1:6379/",
|
9
|
+
verbose: false,
|
10
|
+
command: nil
|
11
|
+
}
|
12
|
+
|
13
|
+
def dbg(...)
|
14
|
+
if $opt[:verbose]
|
15
|
+
print("# ")
|
16
|
+
puts(...)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
opt_parser=OptionParser.new do |opts|
|
21
|
+
opts.on("--url", "--url REDIS_URL (#{$opt[:url]})", "Redis server and port..") do |v|
|
22
|
+
$opt[:url]=v
|
23
|
+
end
|
24
|
+
opts.on("-q", "--quiet", "output only results without any other information") do
|
25
|
+
$opt[:quiet]=false
|
26
|
+
end
|
27
|
+
opts.on("--list-channels", "list all Nchan channels on Redis server or cluster") do |v|
|
28
|
+
$opt[:command]=:filter_channels
|
29
|
+
end
|
30
|
+
opts.on("--filter-channels-min-subscribers=[NUMBER]") do |v|
|
31
|
+
$opt[:command]=:filter_channels
|
32
|
+
$opt[:min_subscribers]=v.to_i
|
33
|
+
end
|
34
|
+
end
|
35
|
+
opt_parser.banner= <<~EOB
|
36
|
+
Debugging tools for the Redis server or cluster backing Nchan.
|
37
|
+
Usage: nchan-redis-debug [options]
|
38
|
+
|
39
|
+
WARNING: --list-channels and --filter-channels-* options for this tool
|
40
|
+
use the Redis SCAN command. This may increase CPU load on the Redis
|
41
|
+
server and may affect the latency of Nchan requests.
|
42
|
+
USE THESE OPTIONS WITH GREAT CARE
|
43
|
+
|
44
|
+
Example:
|
45
|
+
nchan-redis-debug --url redis:// --filter-channels-min-subscribers=10
|
46
|
+
EOB
|
47
|
+
opt_parser.parse!
|
48
|
+
|
49
|
+
rdsck = Rdsck.new $opt
|
50
|
+
if not rdsck.connect
|
51
|
+
STDERR.puts "failed to connect to #{$opt[:url]}"
|
52
|
+
exit 1
|
53
|
+
end
|
54
|
+
|
55
|
+
case $opt[:command]
|
56
|
+
when :filter_channels
|
57
|
+
puts "# scanning for channels #{$opt[:min_subscribers] && "with subscribers >= #{$opt[:min_subscribers]}"}"
|
58
|
+
chans = rdsck.filter_channels(min_subscribers: $opt[:min_subscribers])
|
59
|
+
puts "# found #{chans.count} channel#{chans.count != 1 && "s"}#{chans.count == 0 ? "." : ":"}"
|
60
|
+
puts chans.join("\n")
|
61
|
+
else
|
62
|
+
puts "Nothing to do"
|
63
|
+
end
|
data/exe/nchan-sub
CHANGED
@@ -33,7 +33,7 @@ opt_parser=OptionParser.new do |opts|
|
|
33
33
|
opts.on("-p", "--parallel NUM (#{par})", "number of parallel clients"){|v| par = v.to_i}
|
34
34
|
opts.on("-t", "--timeout SEC (#{opt[:timeout]})", "Long-poll timeout"){|v| opt[:timeout] = v}
|
35
35
|
opts.on("-q", "--quit STRING (#{opt[:quit_message]})", "Quit message"){|v| opt[:quit_message] = v}
|
36
|
-
opts.on("-c", "--client STRING (#{opt[:client]})", "sub client (one of #{Subscriber::Client.unique_aliases.join ', '})") do |v|
|
36
|
+
opts.on("-c", "--client STRING (#{opt[:client]})", "sub client (one of #{NchanTools::Subscriber::Client.unique_aliases.join ', '})") do |v|
|
37
37
|
opt[:client] = v.to_sym
|
38
38
|
end
|
39
39
|
opts.on("--content-type", "show received content-type"){|v| print_content_type = true}
|
@@ -68,7 +68,7 @@ if origin
|
|
68
68
|
opt[:extra_headers]['Origin'] = origin
|
69
69
|
end
|
70
70
|
|
71
|
-
sub = Subscriber.new url, par, opt
|
71
|
+
sub = NchanTools::Subscriber.new url, par, opt
|
72
72
|
|
73
73
|
|
74
74
|
NOMSGF="\r"*30 + "Received message %i, len:%i"
|
@@ -111,7 +111,7 @@ sub.on_message do |msg|
|
|
111
111
|
end
|
112
112
|
|
113
113
|
sub.on_failure do |err_msg|
|
114
|
-
if Subscriber::IntervalPollClient === sub.client
|
114
|
+
if NchanTools::Subscriber::IntervalPollClient === sub.client
|
115
115
|
unless err_msg.match(/\(code 304\)/)
|
116
116
|
false
|
117
117
|
end
|
@@ -0,0 +1,241 @@
|
|
1
|
+
require 'nchan_tools/pubsub'
|
2
|
+
require 'securerandom'
|
3
|
+
require 'timers'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module NchanTools
|
7
|
+
class Benchmark
|
8
|
+
CSV_COLUMNS_ALL=%i[servers runtime channels channels_K channels_M subscribers message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed messages_received messages_unreceived messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber message_publishing_avg message_publishing_99th message_publishing_max message_publishing_stddev message_publishing_count message_delivery_avg message_delivery_99th message_delivery_max message_delivery_stddev message_delivery_count]
|
9
|
+
CSV_COLUMNS_DEFAULT=%i[servers runtime channels subscribers message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed messages_received messages_unreceived messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber message_publishing_avg message_publishing_99th message_publishing_max message_publishing_stddev message_publishing_count message_delivery_avg message_delivery_99th message_delivery_max message_delivery_stddev message_delivery_count]
|
10
|
+
class BenchmarkError < StandardError
|
11
|
+
end
|
12
|
+
def initialize(urls, init_args=nil)
|
13
|
+
@urls = urls
|
14
|
+
@n = urls.count
|
15
|
+
@initializing = 0
|
16
|
+
@ready = 0
|
17
|
+
@running = 0
|
18
|
+
@finished = 0
|
19
|
+
@subs = []
|
20
|
+
@results = {}
|
21
|
+
@failed = {}
|
22
|
+
|
23
|
+
@init_args = init_args
|
24
|
+
@histograms = {}
|
25
|
+
|
26
|
+
subs = []
|
27
|
+
end
|
28
|
+
|
29
|
+
def run
|
30
|
+
puts "connecting to #{@n} Nchan server#{@n > 1 ? "s" : ""}..."
|
31
|
+
@urls.each do |url|
|
32
|
+
sub = NchanTools::Subscriber.new(url, 1, client: :websocket, timeout: 900000, extra_headers: {"Accept" => "text/x-json-hdrhistogram"})
|
33
|
+
sub.on_failure do |err|
|
34
|
+
unless @results[sub]
|
35
|
+
unless @results[sub.url]
|
36
|
+
@failed[sub] = true
|
37
|
+
abort err, sub
|
38
|
+
end
|
39
|
+
end
|
40
|
+
false
|
41
|
+
end
|
42
|
+
sub.on_message do |msg|
|
43
|
+
msg = msg.to_s
|
44
|
+
case msg
|
45
|
+
when /^READY/
|
46
|
+
puts " #{sub.url} ok"
|
47
|
+
@ready +=1
|
48
|
+
if @ready == @n
|
49
|
+
puts "start benchmark..."
|
50
|
+
control :run
|
51
|
+
end
|
52
|
+
when /^RUNNING/
|
53
|
+
puts " #{sub.url} running"
|
54
|
+
when /^RESULTS\n/
|
55
|
+
msg = msg[8..-1]
|
56
|
+
parsed = JSON.parse msg
|
57
|
+
|
58
|
+
#backwards-compatible histogram fields
|
59
|
+
parsed["histograms"]||={}
|
60
|
+
if parsed[:message_publishing_histogram] then
|
61
|
+
parsed[:histograms]["message publishing"]=parsed[:message_publishing_histogram]
|
62
|
+
end
|
63
|
+
if parsed[:message_delivery_histogram] then
|
64
|
+
parsed[:histograms]["message delivery"]=parsed[:message_delivery_histogram]
|
65
|
+
end
|
66
|
+
|
67
|
+
@results[sub.url] = parsed
|
68
|
+
@results[sub.url]["raw"] = msg if @results[sub.url]
|
69
|
+
sub.client.send_close
|
70
|
+
when /^INITIALIZING/
|
71
|
+
#do nothing
|
72
|
+
else
|
73
|
+
raise BenchmarkError, "unexpected server response: #{msg}"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
@subs << sub
|
77
|
+
sub.run
|
78
|
+
sub.wait :ready, 1
|
79
|
+
if @failed[sub]
|
80
|
+
puts " #{sub.url} failed"
|
81
|
+
else
|
82
|
+
puts " #{sub.url} ok"
|
83
|
+
end
|
84
|
+
end
|
85
|
+
return if @failed.count > 0
|
86
|
+
puts "initializing benchmark..."
|
87
|
+
control :init
|
88
|
+
self.wait
|
89
|
+
puts "finished."
|
90
|
+
puts ""
|
91
|
+
end
|
92
|
+
|
93
|
+
def wait
|
94
|
+
@subs.each &:wait
|
95
|
+
end
|
96
|
+
|
97
|
+
def control(msg)
|
98
|
+
if @init_args && (msg.to_sym ==:init || msg.to_sym ==:initialize)
|
99
|
+
msg = "#{msg.to_s} #{@init_args.map{|k,v| "#{k}=#{v}"}.join(" ")}"
|
100
|
+
end
|
101
|
+
@subs.each { |sub| sub.client.send_data msg.to_s }
|
102
|
+
end
|
103
|
+
|
104
|
+
def abort(err, src_sub = nil)
|
105
|
+
puts " #{err}"
|
106
|
+
@subs.each do |sub|
|
107
|
+
sub.terminate unless sub == src_sub
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def hdrhistogram_stats(name, histogram)
|
112
|
+
fmt = <<-END.gsub(/^ {6}/, '')
|
113
|
+
%s
|
114
|
+
min: %.3fms
|
115
|
+
avg: %.3fms
|
116
|
+
99%%ile: %.3fms
|
117
|
+
max: %.3fms
|
118
|
+
stddev: %.3fms
|
119
|
+
samples: %d
|
120
|
+
END
|
121
|
+
fmt % [ name,
|
122
|
+
histogram.min, histogram.mean, histogram.percentile(99.0), histogram.max, histogram.stddev, histogram.count
|
123
|
+
]
|
124
|
+
end
|
125
|
+
|
126
|
+
def results
|
127
|
+
@channels = 0
|
128
|
+
@runtime = []
|
129
|
+
@subscribers = 0
|
130
|
+
@message_length = []
|
131
|
+
@messages_sent = 0
|
132
|
+
@messages_send_confirmed = 0
|
133
|
+
@messages_send_unconfirmed = 0
|
134
|
+
@messages_send_failed = 0
|
135
|
+
@messages_received = 0
|
136
|
+
@messages_unreceived = 0
|
137
|
+
@histograms = {}
|
138
|
+
@results.each do |url, data|
|
139
|
+
@channels += data["channels"]
|
140
|
+
@runtime << data["run_time_sec"]
|
141
|
+
@subscribers += data["subscribers"]
|
142
|
+
@message_length << data["message_length"]
|
143
|
+
@messages_sent += data["messages"]["sent"]
|
144
|
+
@messages_send_confirmed += data["messages"]["send_confirmed"]
|
145
|
+
@messages_send_unconfirmed += data["messages"]["send_unconfirmed"]
|
146
|
+
@messages_send_failed += data["messages"]["send_failed"]
|
147
|
+
@messages_received += data["messages"]["received"]
|
148
|
+
@messages_unreceived += data["messages"]["unreceived"]
|
149
|
+
if data["histograms"]
|
150
|
+
data["histograms"].each do |name, str|
|
151
|
+
name = name.to_sym
|
152
|
+
hdrh = HDRHistogram.unserialize(str, unit: :ms, multiplier: 0.001)
|
153
|
+
if @histograms[name]
|
154
|
+
@histograms[name].merge! hdrh
|
155
|
+
else
|
156
|
+
@histograms[name] = hdrh
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
@message_length = @message_length.inject(0, :+).to_f / @message_length.size
|
163
|
+
@runtime = @runtime.inject(0, :+).to_f / @runtime.size
|
164
|
+
|
165
|
+
fmt = <<-END.gsub(/^ {6}/, '')
|
166
|
+
Nchan servers: %d
|
167
|
+
runtime: %d
|
168
|
+
channels: %d
|
169
|
+
subscribers: %d
|
170
|
+
subscribers per channel: %.1f
|
171
|
+
messages:
|
172
|
+
length: %d
|
173
|
+
sent: %d
|
174
|
+
send_confirmed: %d
|
175
|
+
send_unconfirmed: %d
|
176
|
+
send_failed: %d
|
177
|
+
received: %d
|
178
|
+
unreceived: %d
|
179
|
+
send rate: %.3f/sec
|
180
|
+
receive rate: %.3f/sec
|
181
|
+
send rate per channel: %.3f/min
|
182
|
+
receive rate per subscriber: %.3f/min
|
183
|
+
END
|
184
|
+
out = fmt % [
|
185
|
+
@n, @runtime, @channels, @subscribers, @subscribers.to_f/@channels,
|
186
|
+
@message_length, @messages_sent, @messages_send_confirmed, @messages_send_unconfirmed, @messages_send_failed,
|
187
|
+
@messages_received, @messages_unreceived,
|
188
|
+
@messages_sent.to_f/@runtime,
|
189
|
+
@messages_received.to_f/@runtime,
|
190
|
+
(@messages_sent.to_f* 60)/(@runtime * @channels),
|
191
|
+
(@messages_received.to_f * 60)/(@runtime * @subscribers)
|
192
|
+
]
|
193
|
+
@histograms.each do |name, histogram|
|
194
|
+
out << hdrhistogram_stats("#{name} latency:", histogram)
|
195
|
+
end
|
196
|
+
|
197
|
+
puts out
|
198
|
+
end
|
199
|
+
|
200
|
+
def append_csv_file(file, columns=Benchmark::CSV_COLUMNS_DEFAULT)
|
201
|
+
require "csv"
|
202
|
+
write_headers = File.zero?(file) || !File.exists?(file)
|
203
|
+
headers = columns
|
204
|
+
vals = {
|
205
|
+
servers: @n,
|
206
|
+
runtime: @runtime,
|
207
|
+
channels: @channels,
|
208
|
+
channels_K: @channels/1000.0,
|
209
|
+
channels_M: @channels/1000000.0,
|
210
|
+
subscribers: @subscribers * @channels,
|
211
|
+
message_length: @message_length,
|
212
|
+
messages_sent: @messages_sent,
|
213
|
+
messages_send_confirmed: @messages_send_confirmed,
|
214
|
+
messages_send_unconfirmed: @messages_send_unconfirmed,
|
215
|
+
messages_send_failed: @messages_send_failed,
|
216
|
+
messages_received: @messages_received,
|
217
|
+
messages_unreceived: @messages_unreceived,
|
218
|
+
messages_send_rate: @messages_sent.to_f/@runtime,
|
219
|
+
messages_receive_rate: @messages_received.to_f/@runtime,
|
220
|
+
messages_send_rate_per_channel: (@messages_sent.to_f* 60)/(@runtime * @channels),
|
221
|
+
messages_receive_rate_per_subscriber: (@messages_received.to_f * 60)/(@runtime * @subscribers * @channels)
|
222
|
+
}
|
223
|
+
@histograms.each do |name, histogram|
|
224
|
+
vals["#{name}_avg".to_sym]=histogram.mean
|
225
|
+
vals["#{name}_95th".to_sym]=histogram.percentile(95.0)
|
226
|
+
vals["#{name}_99th".to_sym]=histogram.percentile(99.0)
|
227
|
+
vals["#{name}_max".to_sym]=histogram.max
|
228
|
+
vals["#{name}_stddev".to_sym]=histogram.stddev
|
229
|
+
vals["#{name}_count".to_sym]=histogram.count
|
230
|
+
end
|
231
|
+
|
232
|
+
row = []
|
233
|
+
headers.each { |header| row << (vals[header.to_sym] || "-")}
|
234
|
+
|
235
|
+
csv = CSV.open(file, "a", {headers: headers, write_headers: write_headers})
|
236
|
+
csv << row
|
237
|
+
csv.flush
|
238
|
+
csv.close
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|
data/lib/nchan_tools/pubsub.rb
CHANGED
@@ -3,7 +3,7 @@ require 'typhoeus'
|
|
3
3
|
require 'json'
|
4
4
|
require 'oga'
|
5
5
|
require 'yaml'
|
6
|
-
|
6
|
+
|
7
7
|
require 'celluloid/current'
|
8
8
|
require 'date'
|
9
9
|
Typhoeus::Config.memoize = false
|
@@ -46,15 +46,12 @@ module URI
|
|
46
46
|
u
|
47
47
|
end
|
48
48
|
end
|
49
|
-
|
50
|
-
$seq = 0
|
49
|
+
module NchanTools
|
51
50
|
class Message
|
52
51
|
attr_accessor :content_type, :message, :times_seen, :etag, :last_modified, :eventsource_event
|
53
52
|
def initialize(msg, last_modified=nil, etag=nil)
|
54
53
|
@times_seen=1
|
55
54
|
@message, @last_modified, @etag = msg, last_modified, etag
|
56
|
-
$seq+=1
|
57
|
-
@seq = $seq
|
58
55
|
@idhist = []
|
59
56
|
end
|
60
57
|
def serverside_id
|
@@ -248,7 +245,7 @@ class Subscriber
|
|
248
245
|
end
|
249
246
|
end
|
250
247
|
|
251
|
-
class SubscriberError <
|
248
|
+
class SubscriberError < StandardError
|
252
249
|
end
|
253
250
|
class Client
|
254
251
|
attr_accessor :concurrency
|
@@ -490,6 +487,14 @@ class Subscriber
|
|
490
487
|
@ws.binary data
|
491
488
|
end
|
492
489
|
|
490
|
+
def send_ping(msg=nil)
|
491
|
+
@ws.ping(msg)
|
492
|
+
end
|
493
|
+
|
494
|
+
def send_close(reason=nil, code=1000)
|
495
|
+
@ws.close(reason, code)
|
496
|
+
end
|
497
|
+
|
493
498
|
def write(data)
|
494
499
|
@sock.write data
|
495
500
|
end
|
@@ -582,11 +587,11 @@ class Subscriber
|
|
582
587
|
end
|
583
588
|
|
584
589
|
bundle.ws.on :ping do |ev|
|
585
|
-
@
|
590
|
+
@subscriber.on(:ping).call ev, bundle
|
586
591
|
end
|
587
592
|
|
588
593
|
bundle.ws.on :pong do |ev|
|
589
|
-
@
|
594
|
+
@subscriber.on(:pong).call ev, bundle
|
590
595
|
end
|
591
596
|
|
592
597
|
bundle.ws.on :error do |ev|
|
@@ -648,13 +653,6 @@ class Subscriber
|
|
648
653
|
end
|
649
654
|
end
|
650
655
|
|
651
|
-
def on_ping
|
652
|
-
@on_ping = Proc.new if block_given?
|
653
|
-
end
|
654
|
-
def on_pong
|
655
|
-
@on_pong = Proc.new if block_given?
|
656
|
-
end
|
657
|
-
|
658
656
|
def listen(bundle)
|
659
657
|
while @ws[bundle]
|
660
658
|
begin
|
@@ -684,10 +682,10 @@ class Subscriber
|
|
684
682
|
private :ws_client
|
685
683
|
|
686
684
|
def send_ping(data=nil)
|
687
|
-
ws_client.
|
685
|
+
ws_client.send_ping data
|
688
686
|
end
|
689
|
-
def send_close(
|
690
|
-
ws_client.send_close
|
687
|
+
def send_close(reason=nil, code=1000)
|
688
|
+
ws_client.send_close reason, code
|
691
689
|
end
|
692
690
|
def send_data(data)
|
693
691
|
ws_client.send_data data
|
@@ -697,17 +695,18 @@ class Subscriber
|
|
697
695
|
end
|
698
696
|
|
699
697
|
def close(bundle)
|
700
|
-
if bundle
|
698
|
+
if bundle then
|
701
699
|
@ws.delete bundle
|
702
700
|
bundle.sock.close unless bundle.sock.closed?
|
703
701
|
end
|
704
702
|
@connected -= 1
|
705
|
-
if @connected <= 0
|
706
|
-
|
703
|
+
if @connected <= 0 then
|
704
|
+
until @ws.count == 0 do
|
705
|
+
sleep 0.1
|
706
|
+
end
|
707
707
|
@cooked.signal true
|
708
708
|
end
|
709
709
|
end
|
710
|
-
|
711
710
|
end
|
712
711
|
|
713
712
|
class LongPollClient < Client
|
@@ -1521,6 +1520,8 @@ class Subscriber
|
|
1521
1520
|
|
1522
1521
|
attr_accessor :url, :client, :messages, :max_round_trips, :quit_message, :errors, :concurrency, :waiting, :finished, :client_class, :log
|
1523
1522
|
def initialize(url, concurrency=1, opt={})
|
1523
|
+
@empty_block = Proc.new {}
|
1524
|
+
@on={}
|
1524
1525
|
@care_about_message_ids=opt[:use_message_id].nil? ? true : opt[:use_message_id]
|
1525
1526
|
@url=url
|
1526
1527
|
@quit_message = opt[:quit_message]
|
@@ -1615,6 +1616,14 @@ class Subscriber
|
|
1615
1616
|
@client.poke until_what, timeout
|
1616
1617
|
end
|
1617
1618
|
|
1619
|
+
def on(evt_name = nil, &block)
|
1620
|
+
if block_given?
|
1621
|
+
@on[evt_name.to_sym] = block
|
1622
|
+
else
|
1623
|
+
@on[evt_name.to_sym] or @empty_block
|
1624
|
+
end
|
1625
|
+
end
|
1626
|
+
|
1618
1627
|
def on_message(msg=nil, bundle=nil, &block)
|
1619
1628
|
#puts "received message #{msg && msg.to_s[0..15]}"
|
1620
1629
|
if block_given?
|
@@ -1646,7 +1655,7 @@ end
|
|
1646
1655
|
class Publisher
|
1647
1656
|
#include Celluloid
|
1648
1657
|
|
1649
|
-
class PublisherError <
|
1658
|
+
class PublisherError < StandardError
|
1650
1659
|
end
|
1651
1660
|
|
1652
1661
|
attr_accessor :messages, :response, :response_code, :response_body, :nofail, :accept, :url, :extra_headers, :verbose, :ws, :channel_info, :channel_info_type
|
@@ -1660,6 +1669,7 @@ class Publisher
|
|
1660
1669
|
@accept = opt[:accept]
|
1661
1670
|
@verbose = opt[:verbose]
|
1662
1671
|
@on_response = opt[:on_response]
|
1672
|
+
@http2 = opt[:http2]
|
1663
1673
|
|
1664
1674
|
@ws_wait_until_response = true
|
1665
1675
|
|
@@ -1800,6 +1810,7 @@ class Publisher
|
|
1800
1810
|
headers = {:'Content-Type' => content_type, :'Accept' => accept}
|
1801
1811
|
headers[:'X-Eventsource-Event'] = eventsource_event if eventsource_event
|
1802
1812
|
headers.merge! @extra_headers if @extra_headers
|
1813
|
+
|
1803
1814
|
post = Typhoeus::Request.new(
|
1804
1815
|
@url,
|
1805
1816
|
headers: headers,
|
@@ -1807,7 +1818,8 @@ class Publisher
|
|
1807
1818
|
body: body,
|
1808
1819
|
timeout: @timeout || PUBLISH_TIMEOUT,
|
1809
1820
|
connecttimeout: @timeout || PUBLISH_TIMEOUT,
|
1810
|
-
verbose: @verbose
|
1821
|
+
verbose: @verbose,
|
1822
|
+
http_version: @http2 ? :httpv2_0 : :none
|
1811
1823
|
)
|
1812
1824
|
if body && @messages
|
1813
1825
|
msg=Message.new body
|
@@ -1895,3 +1907,4 @@ class Publisher
|
|
1895
1907
|
|
1896
1908
|
|
1897
1909
|
end
|
1910
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
class Rdsck
|
2
|
+
attr_accessor :url, :verbose, :namespace
|
3
|
+
attr_accessor :redis, :masters
|
4
|
+
def initialize(opt)
|
5
|
+
@url=opt[:url]
|
6
|
+
@verbose=opt[:verbose]
|
7
|
+
@namespace=opt[:namespace]
|
8
|
+
end
|
9
|
+
|
10
|
+
def cluster?
|
11
|
+
@cluster_mode
|
12
|
+
end
|
13
|
+
|
14
|
+
def connect
|
15
|
+
begin
|
16
|
+
@redis=Redis.new url: $opt[:url]
|
17
|
+
mode = redis.info["redis_mode"]
|
18
|
+
rescue StandardError => e
|
19
|
+
STDERR.puts e.message
|
20
|
+
return false
|
21
|
+
end
|
22
|
+
|
23
|
+
if mode == "cluster"
|
24
|
+
@redis.close
|
25
|
+
begin
|
26
|
+
@redis=Redis.new cluster: [$opt[:url]]
|
27
|
+
@redis.ping
|
28
|
+
rescue StandardError => e
|
29
|
+
STDERR.puts e.message
|
30
|
+
return false
|
31
|
+
end
|
32
|
+
|
33
|
+
@cluster_mode = true
|
34
|
+
@masters = []
|
35
|
+
|
36
|
+
redis.connection.each do |c|
|
37
|
+
node = Redis.new url: c[:id]
|
38
|
+
@masters << node
|
39
|
+
end
|
40
|
+
else
|
41
|
+
@masters = [@redis]
|
42
|
+
end
|
43
|
+
|
44
|
+
dbg "Connected to Redis #{mode == "cluster" ? "cluster" : "server"}"
|
45
|
+
(Array === @redis.connection ? @redis.connection : [@redis.connection]) .each do |v|
|
46
|
+
dbg " #{v[:id]}"
|
47
|
+
end
|
48
|
+
self
|
49
|
+
end
|
50
|
+
|
51
|
+
def key(subkey=nil)
|
52
|
+
k = "{channel:#{$opt[:namespace]}/#{$opt[:channel_id]}}"
|
53
|
+
return subkey ? "#{k}:#{subkey}" : k
|
54
|
+
end
|
55
|
+
|
56
|
+
def info
|
57
|
+
channel_hash=@redis.hgetall key
|
58
|
+
hash_ttl=@redis.ttl key
|
59
|
+
channel_subs=@redis.hgetall key("subscribers")
|
60
|
+
#...
|
61
|
+
end
|
62
|
+
|
63
|
+
def filter_channels(filters={})
|
64
|
+
script = <<~EOF
|
65
|
+
local prev_cursor = ARGV[1]
|
66
|
+
local pattern = ARGV[2]
|
67
|
+
local scan_batch_size = ARGV[3]
|
68
|
+
|
69
|
+
local min_subscribers = ARGV[4] and #ARGV[4] > 0 and tonumber(ARGV[4])
|
70
|
+
|
71
|
+
local cursor, iteration
|
72
|
+
if pattern and #pattern > 0 then
|
73
|
+
cursor, iteration = unpack(redis.call("SCAN", prev_cursor, "MATCH", pattern, "COUNT", scan_batch_size))
|
74
|
+
else
|
75
|
+
cursor, iteration = unpack(redis.call("SCAN", prev_cursor, "COUNT", scan_batch_size))
|
76
|
+
end
|
77
|
+
|
78
|
+
local matched = {}
|
79
|
+
for _, chankey in pairs(iteration) do
|
80
|
+
local match = true
|
81
|
+
if min_subscribers then
|
82
|
+
match = match and (tonumber(redis.call('HGET', chankey, 'fake_subscribers') or 0) >= min_subscribers)
|
83
|
+
end
|
84
|
+
if match then
|
85
|
+
table.insert(matched, chankey)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
return {cursor, matched}
|
90
|
+
EOF
|
91
|
+
|
92
|
+
results = []
|
93
|
+
batch_size=2
|
94
|
+
masters.each do |m|
|
95
|
+
hash = m.script "load", script
|
96
|
+
cursor, pattern = "0", "{channel:*}"
|
97
|
+
loop do
|
98
|
+
cursor, batch_results = m.evalsha hash, keys: [], argv: [cursor, pattern, batch_size, filters[:min_subscribers]]
|
99
|
+
results += batch_results
|
100
|
+
pattern = ""
|
101
|
+
break if cursor.to_i == 0
|
102
|
+
end
|
103
|
+
end
|
104
|
+
results
|
105
|
+
|
106
|
+
results.map! do |key|
|
107
|
+
m = key.match(/^\{channel\:(.*)\}$/)
|
108
|
+
m[1] || key
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
data/lib/nchan_tools/version.rb
CHANGED
data/nchan_tools.gemspec
CHANGED
@@ -34,9 +34,9 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_dependency 'websocket-extensions'
|
35
35
|
spec.add_dependency "permessage_deflate"
|
36
36
|
spec.add_dependency 'http_parser.rb'
|
37
|
-
if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.2.2')
|
38
|
-
|
39
|
-
end
|
37
|
+
if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.2.2')
|
38
|
+
spec.add_dependency 'http-2'
|
39
|
+
end
|
40
40
|
spec.add_development_dependency "pry"
|
41
41
|
spec.add_development_dependency "bundler", "~> 1.16"
|
42
42
|
spec.add_development_dependency "rake", "~> 10.0"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nchan_tools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Leo Ponomarev
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-12-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: typhoeus
|
@@ -212,6 +212,7 @@ email:
|
|
212
212
|
executables:
|
213
213
|
- nchan-benchmark
|
214
214
|
- nchan-pub
|
215
|
+
- nchan-redis-debug
|
215
216
|
- nchan-sub
|
216
217
|
extensions: []
|
217
218
|
extra_rdoc_files: []
|
@@ -226,16 +227,19 @@ files:
|
|
226
227
|
- bin/setup
|
227
228
|
- exe/nchan-benchmark
|
228
229
|
- exe/nchan-pub
|
230
|
+
- exe/nchan-redis-debug
|
229
231
|
- exe/nchan-sub
|
230
232
|
- lib/nchan_tools.rb
|
233
|
+
- lib/nchan_tools/benchmark.rb
|
231
234
|
- lib/nchan_tools/pubsub.rb
|
235
|
+
- lib/nchan_tools/rdsck.rb
|
232
236
|
- lib/nchan_tools/version.rb
|
233
237
|
- nchan_tools.gemspec
|
234
238
|
homepage: https://nchan.io
|
235
239
|
licenses:
|
236
240
|
- WTFPL
|
237
241
|
metadata: {}
|
238
|
-
post_install_message:
|
242
|
+
post_install_message:
|
239
243
|
rdoc_options: []
|
240
244
|
require_paths:
|
241
245
|
- lib
|
@@ -250,9 +254,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
250
254
|
- !ruby/object:Gem::Version
|
251
255
|
version: '0'
|
252
256
|
requirements: []
|
253
|
-
|
254
|
-
|
255
|
-
signing_key:
|
257
|
+
rubygems_version: 3.1.4
|
258
|
+
signing_key:
|
256
259
|
specification_version: 4
|
257
260
|
summary: Development and testing utilities for Nchan
|
258
261
|
test_files: []
|