nchan_tools 0.1.5 → 0.1.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/exe/nchan-benchmark +6 -2
- data/exe/nchan-pub +1 -1
- data/exe/nchan-redis-debug +57 -0
- data/exe/nchan-sub +3 -3
- data/lib/nchan_tools/benchmark.rb +65 -38
- data/lib/nchan_tools/pubsub.rb +32 -17
- data/lib/nchan_tools/rdsck.rb +120 -0
- data/lib/nchan_tools/version.rb +1 -1
- data/nchan_tools.gemspec +6 -5
- metadata +31 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e17f44cb591b9f74d1791e1f9ec8574613fbc840c6aac91953c8a526a8a63006
|
4
|
+
data.tar.gz: 405aabdbc70b14571c6cc545ca13f0ea7e5ad65b130ab37194fee1ed7ccf7d5c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 31e391ad81b9a43519d014a538f77b87b25706a009180b83239eea014c174497d8d7e3c9ca26d173b075cd42cba4e82ab92dc696ce459058080a325c29f808e3
|
7
|
+
data.tar.gz: 66654049eaf6f7f26e16fd98df88b1570a5d80f7168cc2f774e5d7168ae4d42a668b1384c3280fd82b8473f78e63ec4774b005f9971be4db7e9a2d4d65a01264
|
data/exe/nchan-benchmark
CHANGED
@@ -9,7 +9,7 @@ require "HDRHistogram"
|
|
9
9
|
|
10
10
|
verbose = false
|
11
11
|
save_csv = false
|
12
|
-
|
12
|
+
csv_columns = NchanTools::Benchmark::CSV_COLUMNS_DEFAULT
|
13
13
|
init_args = {}
|
14
14
|
|
15
15
|
opt_parser=OptionParser.new do |opts|
|
@@ -19,6 +19,9 @@ opt_parser=OptionParser.new do |opts|
|
|
19
19
|
opts.on("--csv FILENAME", "Append results to file in CSV format") do |f|
|
20
20
|
save_csv = f
|
21
21
|
end
|
22
|
+
opts.on("--csv-columns col1,col2,...", "csv columns list") do |f|
|
23
|
+
csv_columns = f.split(/\W+/).map(&:to_sym)
|
24
|
+
end
|
22
25
|
opts.on("-t", "--time TIME", "Time to run benchmark") do |v|
|
23
26
|
init_args[:time] = v
|
24
27
|
end
|
@@ -43,6 +46,7 @@ urls += ARGV
|
|
43
46
|
begin
|
44
47
|
urls += STDIN.read_nonblock(100000).split /\s*\n+\s*/
|
45
48
|
rescue IO::WaitReadable
|
49
|
+
rescue EOFError
|
46
50
|
end
|
47
51
|
|
48
52
|
urls.uniq!
|
@@ -50,5 +54,5 @@ urls.uniq!
|
|
50
54
|
benchan = NchanTools::Benchmark.new urls, init_args
|
51
55
|
benchan.run
|
52
56
|
benchan.results
|
53
|
-
benchan.append_csv_file(save_csv) if save_csv
|
57
|
+
benchan.append_csv_file(save_csv, csv_columns) if save_csv
|
54
58
|
|
data/exe/nchan-pub
CHANGED
@@ -54,7 +54,7 @@ puts "Publishing to #{url}."
|
|
54
54
|
|
55
55
|
loopmsg=("\r"*20) + "sending message #"
|
56
56
|
|
57
|
-
pub = Publisher.new url, nostore: true, timeout: timeout, verbose: verbose, websocket: websocket
|
57
|
+
pub = NchanTools::Publisher.new url, nostore: true, timeout: timeout, verbose: verbose, websocket: websocket
|
58
58
|
pub.accept=accept
|
59
59
|
pub.nofail=true
|
60
60
|
repeat=true
|
@@ -0,0 +1,57 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "redis"
|
4
|
+
require "optparse"
|
5
|
+
require 'nchan_tools/rdsck'
|
6
|
+
|
7
|
+
opt = {
|
8
|
+
url: "redis://127.0.0.1:6379/",
|
9
|
+
verbose: false,
|
10
|
+
command: nil
|
11
|
+
}
|
12
|
+
|
13
|
+
opt_parser=OptionParser.new do |opts|
|
14
|
+
opts.on("--url", "--url REDIS_URL (#{opt[:url]})", "Redis server and port..") do |v|
|
15
|
+
opt[:url]=v
|
16
|
+
end
|
17
|
+
opts.on("-q", "--quiet", "output only results without any other information") do
|
18
|
+
opt[:quiet]=false
|
19
|
+
opts[:verbose] = !opt[:quiet]
|
20
|
+
end
|
21
|
+
opts.on("--list-channels", "list all Nchan channels on Redis server or cluster") do |v|
|
22
|
+
opt[:command]=:filter_channels
|
23
|
+
end
|
24
|
+
opts.on("--filter-channels-min-subscribers=[NUMBER]") do |v|
|
25
|
+
opt[:command]=:filter_channels
|
26
|
+
opt[:min_subscribers]=v.to_i
|
27
|
+
end
|
28
|
+
end
|
29
|
+
opt_parser.banner= <<~EOB
|
30
|
+
Debugging tools for the Redis server or cluster backing Nchan.
|
31
|
+
Usage: nchan-redis-debug [options]
|
32
|
+
|
33
|
+
WARNING: --list-channels and --filter-channels-* options for this tool
|
34
|
+
use the Redis SCAN command. This may increase CPU load on the Redis
|
35
|
+
server and may affect the latency of Nchan requests.
|
36
|
+
USE THESE OPTIONS WITH GREAT CARE
|
37
|
+
|
38
|
+
Example:
|
39
|
+
nchan-redis-debug --url redis:// --filter-channels-min-subscribers=10
|
40
|
+
EOB
|
41
|
+
opt_parser.parse!
|
42
|
+
|
43
|
+
rdsck = Rdsck.new opt
|
44
|
+
if not rdsck.connect
|
45
|
+
STDERR.puts "failed to connect to #{opt[:url]}"
|
46
|
+
exit 1
|
47
|
+
end
|
48
|
+
|
49
|
+
case opt[:command]
|
50
|
+
when :filter_channels
|
51
|
+
puts "# scanning for channels #{opt[:min_subscribers] && "with subscribers >= #{opt[:min_subscribers]}"}"
|
52
|
+
chans = rdsck.filter_channels(min_subscribers: opt[:min_subscribers])
|
53
|
+
puts "# found #{chans.count} channel#{chans.count != 1 && "s"}#{chans.count == 0 ? "." : ":"}"
|
54
|
+
puts chans.join("\n")
|
55
|
+
else
|
56
|
+
puts "Nothing to do"
|
57
|
+
end
|
data/exe/nchan-sub
CHANGED
@@ -33,7 +33,7 @@ opt_parser=OptionParser.new do |opts|
|
|
33
33
|
opts.on("-p", "--parallel NUM (#{par})", "number of parallel clients"){|v| par = v.to_i}
|
34
34
|
opts.on("-t", "--timeout SEC (#{opt[:timeout]})", "Long-poll timeout"){|v| opt[:timeout] = v}
|
35
35
|
opts.on("-q", "--quit STRING (#{opt[:quit_message]})", "Quit message"){|v| opt[:quit_message] = v}
|
36
|
-
opts.on("-c", "--client STRING (#{opt[:client]})", "sub client (one of #{Subscriber::Client.unique_aliases.join ', '})") do |v|
|
36
|
+
opts.on("-c", "--client STRING (#{opt[:client]})", "sub client (one of #{NchanTools::Subscriber::Client.unique_aliases.join ', '})") do |v|
|
37
37
|
opt[:client] = v.to_sym
|
38
38
|
end
|
39
39
|
opts.on("--content-type", "show received content-type"){|v| print_content_type = true}
|
@@ -68,7 +68,7 @@ if origin
|
|
68
68
|
opt[:extra_headers]['Origin'] = origin
|
69
69
|
end
|
70
70
|
|
71
|
-
sub = Subscriber.new url, par, opt
|
71
|
+
sub = NchanTools::Subscriber.new url, par, opt
|
72
72
|
|
73
73
|
|
74
74
|
NOMSGF="\r"*30 + "Received message %i, len:%i"
|
@@ -111,7 +111,7 @@ sub.on_message do |msg|
|
|
111
111
|
end
|
112
112
|
|
113
113
|
sub.on_failure do |err_msg|
|
114
|
-
if Subscriber::IntervalPollClient === sub.client
|
114
|
+
if NchanTools::Subscriber::IntervalPollClient === sub.client
|
115
115
|
unless err_msg.match(/\(code 304\)/)
|
116
116
|
false
|
117
117
|
end
|
@@ -5,6 +5,8 @@ require 'json'
|
|
5
5
|
|
6
6
|
module NchanTools
|
7
7
|
class Benchmark
|
8
|
+
CSV_COLUMNS_ALL=%i[servers runtime channels channels_K channels_M subscribers message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed messages_received messages_unreceived messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber message_publishing_avg message_publishing_99th message_publishing_max message_publishing_stddev message_publishing_count message_delivery_avg message_delivery_99th message_delivery_max message_delivery_stddev message_delivery_count]
|
9
|
+
CSV_COLUMNS_DEFAULT=%i[servers runtime channels subscribers message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed messages_received messages_unreceived messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber message_publishing_avg message_publishing_99th message_publishing_max message_publishing_stddev message_publishing_count message_delivery_avg message_delivery_99th message_delivery_max message_delivery_stddev message_delivery_count]
|
8
10
|
class BenchmarkError < StandardError
|
9
11
|
end
|
10
12
|
def initialize(urls, init_args=nil)
|
@@ -19,9 +21,7 @@ class Benchmark
|
|
19
21
|
@failed = {}
|
20
22
|
|
21
23
|
@init_args = init_args
|
22
|
-
|
23
|
-
@hdrh_publish = nil
|
24
|
-
@hdrh_receive = nil
|
24
|
+
@histograms = {}
|
25
25
|
|
26
26
|
subs = []
|
27
27
|
end
|
@@ -46,17 +46,27 @@ class Benchmark
|
|
46
46
|
puts " #{sub.url} ok"
|
47
47
|
@ready +=1
|
48
48
|
if @ready == @n
|
49
|
-
control :run
|
50
49
|
puts "start benchmark..."
|
50
|
+
control :run
|
51
51
|
end
|
52
52
|
when /^RUNNING/
|
53
53
|
puts " #{sub.url} running"
|
54
54
|
when /^RESULTS\n/
|
55
55
|
msg = msg[8..-1]
|
56
56
|
parsed = JSON.parse msg
|
57
|
+
|
58
|
+
#backwards-compatible histogram fields
|
59
|
+
parsed["histograms"]||={}
|
60
|
+
if parsed[:message_publishing_histogram] then
|
61
|
+
parsed[:histograms]["message publishing"]=parsed[:message_publishing_histogram]
|
62
|
+
end
|
63
|
+
if parsed[:message_delivery_histogram] then
|
64
|
+
parsed[:histograms]["message delivery"]=parsed[:message_delivery_histogram]
|
65
|
+
end
|
66
|
+
|
57
67
|
@results[sub.url] = parsed
|
58
68
|
@results[sub.url]["raw"] = msg if @results[sub.url]
|
59
|
-
|
69
|
+
sub.client.send_close
|
60
70
|
when /^INITIALIZING/
|
61
71
|
#do nothing
|
62
72
|
else
|
@@ -124,8 +134,7 @@ class Benchmark
|
|
124
134
|
@messages_send_failed = 0
|
125
135
|
@messages_received = 0
|
126
136
|
@messages_unreceived = 0
|
127
|
-
@
|
128
|
-
@hdrh_receive = nil
|
137
|
+
@histograms = {}
|
129
138
|
@results.each do |url, data|
|
130
139
|
@channels += data["channels"]
|
131
140
|
@runtime << data["run_time_sec"]
|
@@ -137,27 +146,21 @@ class Benchmark
|
|
137
146
|
@messages_send_failed += data["messages"]["send_failed"]
|
138
147
|
@messages_received += data["messages"]["received"]
|
139
148
|
@messages_unreceived += data["messages"]["unreceived"]
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
@
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
if data["message_delivery_histogram"]
|
150
|
-
hdrh = HDRHistogram.unserialize(data["message_delivery_histogram"], unit: :ms, multiplier: 0.001)
|
151
|
-
if @hdrh_receive
|
152
|
-
@hdrh_receive.merge! hdrh
|
153
|
-
else
|
154
|
-
@hdrh_receive = hdrh
|
149
|
+
if data["histograms"]
|
150
|
+
data["histograms"].each do |name, str|
|
151
|
+
name = name.to_sym
|
152
|
+
hdrh = HDRHistogram.unserialize(str, unit: :ms, multiplier: 0.001)
|
153
|
+
if @histograms[name]
|
154
|
+
@histograms[name].merge! hdrh
|
155
|
+
else
|
156
|
+
@histograms[name] = hdrh
|
157
|
+
end
|
155
158
|
end
|
156
159
|
end
|
157
160
|
end
|
158
161
|
|
159
|
-
@message_length = @message_length.
|
160
|
-
@runtime = @runtime.
|
162
|
+
@message_length = @message_length.inject(0, :+).to_f / @message_length.size
|
163
|
+
@runtime = @runtime.inject(0, :+).to_f / @runtime.size
|
161
164
|
|
162
165
|
fmt = <<-END.gsub(/^ {6}/, '')
|
163
166
|
Nchan servers: %d
|
@@ -187,26 +190,50 @@ class Benchmark
|
|
187
190
|
(@messages_sent.to_f* 60)/(@runtime * @channels),
|
188
191
|
(@messages_received.to_f * 60)/(@runtime * @subscribers)
|
189
192
|
]
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
+
@histograms.each do |name, histogram|
|
194
|
+
out << hdrhistogram_stats("#{name} latency:", histogram)
|
195
|
+
end
|
193
196
|
|
194
197
|
puts out
|
195
198
|
end
|
196
199
|
|
197
|
-
def append_csv_file(file)
|
200
|
+
def append_csv_file(file, columns=Benchmark::CSV_COLUMNS_DEFAULT)
|
198
201
|
require "csv"
|
199
|
-
write_headers = File.zero?(file)
|
200
|
-
headers =
|
202
|
+
write_headers = File.zero?(file) || !File.exists?(file)
|
203
|
+
headers = columns
|
204
|
+
vals = {
|
205
|
+
servers: @n,
|
206
|
+
runtime: @runtime,
|
207
|
+
channels: @channels,
|
208
|
+
channels_K: @channels/1000.0,
|
209
|
+
channels_M: @channels/1000000.0,
|
210
|
+
subscribers: @subscribers * @channels,
|
211
|
+
message_length: @message_length,
|
212
|
+
messages_sent: @messages_sent,
|
213
|
+
messages_send_confirmed: @messages_send_confirmed,
|
214
|
+
messages_send_unconfirmed: @messages_send_unconfirmed,
|
215
|
+
messages_send_failed: @messages_send_failed,
|
216
|
+
messages_received: @messages_received,
|
217
|
+
messages_unreceived: @messages_unreceived,
|
218
|
+
messages_send_rate: @messages_sent.to_f/@runtime,
|
219
|
+
messages_receive_rate: @messages_received.to_f/@runtime,
|
220
|
+
messages_send_rate_per_channel: (@messages_sent.to_f* 60)/(@runtime * @channels),
|
221
|
+
messages_receive_rate_per_subscriber: (@messages_received.to_f * 60)/(@runtime * @subscribers * @channels)
|
222
|
+
}
|
223
|
+
@histograms.each do |name, histogram|
|
224
|
+
vals["#{name}_avg".to_sym]=histogram.mean
|
225
|
+
vals["#{name}_95th".to_sym]=histogram.percentile(95.0)
|
226
|
+
vals["#{name}_99th".to_sym]=histogram.percentile(99.0)
|
227
|
+
vals["#{name}_max".to_sym]=histogram.max
|
228
|
+
vals["#{name}_stddev".to_sym]=histogram.stddev
|
229
|
+
vals["#{name}_count".to_sym]=histogram.count
|
230
|
+
end
|
231
|
+
|
232
|
+
row = []
|
233
|
+
headers.each { |header| row << (vals[header.to_sym] || "-")}
|
234
|
+
|
201
235
|
csv = CSV.open(file, "a", {headers: headers, write_headers: write_headers})
|
202
|
-
csv <<
|
203
|
-
@message_length, @messages_sent, @messages_send_confirmed, @messages_send_unconfirmed, @messages_send_failed,
|
204
|
-
@messages_received, @messages_unreceived,
|
205
|
-
@messages_sent.to_f/@runtime, @messages_received.to_f/@runtime,
|
206
|
-
(@messages_sent.to_f* 60)/(@runtime * @channels), (@messages_received.to_f * 60)/(@runtime * @subscribers),
|
207
|
-
@hdrh_publish.mean, @hdrh_publish.percentile(99.0), @hdrh_publish.max, @hdrh_publish.stddev, @hdrh_publish.count,
|
208
|
-
@hdrh_receive.mean, @hdrh_receive.percentile(99.0), @hdrh_receive.max, @hdrh_receive.stddev, @hdrh_receive.count
|
209
|
-
]
|
236
|
+
csv << row
|
210
237
|
csv.flush
|
211
238
|
csv.close
|
212
239
|
end
|
data/lib/nchan_tools/pubsub.rb
CHANGED
@@ -487,6 +487,14 @@ class Subscriber
|
|
487
487
|
@ws.binary data
|
488
488
|
end
|
489
489
|
|
490
|
+
def send_ping(msg=nil)
|
491
|
+
@ws.ping(msg)
|
492
|
+
end
|
493
|
+
|
494
|
+
def send_close(reason=nil, code=1000)
|
495
|
+
@ws.close(reason, code)
|
496
|
+
end
|
497
|
+
|
490
498
|
def write(data)
|
491
499
|
@sock.write data
|
492
500
|
end
|
@@ -579,11 +587,11 @@ class Subscriber
|
|
579
587
|
end
|
580
588
|
|
581
589
|
bundle.ws.on :ping do |ev|
|
582
|
-
@
|
590
|
+
@subscriber.on(:ping).call ev, bundle
|
583
591
|
end
|
584
592
|
|
585
593
|
bundle.ws.on :pong do |ev|
|
586
|
-
@
|
594
|
+
@subscriber.on(:pong).call ev, bundle
|
587
595
|
end
|
588
596
|
|
589
597
|
bundle.ws.on :error do |ev|
|
@@ -645,13 +653,6 @@ class Subscriber
|
|
645
653
|
end
|
646
654
|
end
|
647
655
|
|
648
|
-
def on_ping
|
649
|
-
@on_ping = Proc.new if block_given?
|
650
|
-
end
|
651
|
-
def on_pong
|
652
|
-
@on_pong = Proc.new if block_given?
|
653
|
-
end
|
654
|
-
|
655
656
|
def listen(bundle)
|
656
657
|
while @ws[bundle]
|
657
658
|
begin
|
@@ -681,10 +682,10 @@ class Subscriber
|
|
681
682
|
private :ws_client
|
682
683
|
|
683
684
|
def send_ping(data=nil)
|
684
|
-
ws_client.
|
685
|
+
ws_client.send_ping data
|
685
686
|
end
|
686
|
-
def send_close(
|
687
|
-
ws_client.send_close
|
687
|
+
def send_close(reason=nil, code=1000)
|
688
|
+
ws_client.send_close reason, code
|
688
689
|
end
|
689
690
|
def send_data(data)
|
690
691
|
ws_client.send_data data
|
@@ -694,17 +695,18 @@ class Subscriber
|
|
694
695
|
end
|
695
696
|
|
696
697
|
def close(bundle)
|
697
|
-
if bundle
|
698
|
+
if bundle then
|
698
699
|
@ws.delete bundle
|
699
700
|
bundle.sock.close unless bundle.sock.closed?
|
700
701
|
end
|
701
702
|
@connected -= 1
|
702
|
-
if @connected <= 0
|
703
|
-
|
703
|
+
if @connected <= 0 then
|
704
|
+
until @ws.count == 0 do
|
705
|
+
sleep 0.1
|
706
|
+
end
|
704
707
|
@cooked.signal true
|
705
708
|
end
|
706
709
|
end
|
707
|
-
|
708
710
|
end
|
709
711
|
|
710
712
|
class LongPollClient < Client
|
@@ -1518,6 +1520,8 @@ class Subscriber
|
|
1518
1520
|
|
1519
1521
|
attr_accessor :url, :client, :messages, :max_round_trips, :quit_message, :errors, :concurrency, :waiting, :finished, :client_class, :log
|
1520
1522
|
def initialize(url, concurrency=1, opt={})
|
1523
|
+
@empty_block = Proc.new {}
|
1524
|
+
@on={}
|
1521
1525
|
@care_about_message_ids=opt[:use_message_id].nil? ? true : opt[:use_message_id]
|
1522
1526
|
@url=url
|
1523
1527
|
@quit_message = opt[:quit_message]
|
@@ -1612,6 +1616,14 @@ class Subscriber
|
|
1612
1616
|
@client.poke until_what, timeout
|
1613
1617
|
end
|
1614
1618
|
|
1619
|
+
def on(evt_name = nil, &block)
|
1620
|
+
if block_given?
|
1621
|
+
@on[evt_name.to_sym] = block
|
1622
|
+
else
|
1623
|
+
@on[evt_name.to_sym] or @empty_block
|
1624
|
+
end
|
1625
|
+
end
|
1626
|
+
|
1615
1627
|
def on_message(msg=nil, bundle=nil, &block)
|
1616
1628
|
#puts "received message #{msg && msg.to_s[0..15]}"
|
1617
1629
|
if block_given?
|
@@ -1657,6 +1669,7 @@ class Publisher
|
|
1657
1669
|
@accept = opt[:accept]
|
1658
1670
|
@verbose = opt[:verbose]
|
1659
1671
|
@on_response = opt[:on_response]
|
1672
|
+
@http2 = opt[:http2]
|
1660
1673
|
|
1661
1674
|
@ws_wait_until_response = true
|
1662
1675
|
|
@@ -1797,6 +1810,7 @@ class Publisher
|
|
1797
1810
|
headers = {:'Content-Type' => content_type, :'Accept' => accept}
|
1798
1811
|
headers[:'X-Eventsource-Event'] = eventsource_event if eventsource_event
|
1799
1812
|
headers.merge! @extra_headers if @extra_headers
|
1813
|
+
|
1800
1814
|
post = Typhoeus::Request.new(
|
1801
1815
|
@url,
|
1802
1816
|
headers: headers,
|
@@ -1804,7 +1818,8 @@ class Publisher
|
|
1804
1818
|
body: body,
|
1805
1819
|
timeout: @timeout || PUBLISH_TIMEOUT,
|
1806
1820
|
connecttimeout: @timeout || PUBLISH_TIMEOUT,
|
1807
|
-
verbose: @verbose
|
1821
|
+
verbose: @verbose,
|
1822
|
+
http_version: @http2 ? :httpv2_0 : :none
|
1808
1823
|
)
|
1809
1824
|
if body && @messages
|
1810
1825
|
msg=Message.new body
|
@@ -0,0 +1,120 @@
|
|
1
|
+
class Rdsck
|
2
|
+
attr_accessor :url, :verbose, :namespace
|
3
|
+
attr_accessor :redis, :masters
|
4
|
+
|
5
|
+
def dbg(*args)
|
6
|
+
if @verbose
|
7
|
+
print("# ")
|
8
|
+
puts(*args)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(opt)
|
13
|
+
@url=opt[:url]
|
14
|
+
@verbose=opt[:verbose]
|
15
|
+
@namespace=opt[:namespace]
|
16
|
+
@channel_id=opt[:channel_id]
|
17
|
+
end
|
18
|
+
|
19
|
+
def cluster?
|
20
|
+
@cluster_mode
|
21
|
+
end
|
22
|
+
|
23
|
+
def connect
|
24
|
+
begin
|
25
|
+
@redis=Redis.new url: @url
|
26
|
+
mode = redis.info["redis_mode"]
|
27
|
+
rescue StandardError => e
|
28
|
+
STDERR.puts e.message
|
29
|
+
return false
|
30
|
+
end
|
31
|
+
|
32
|
+
if mode == "cluster"
|
33
|
+
@redis.close
|
34
|
+
begin
|
35
|
+
@redis=Redis.new cluster: [@url]
|
36
|
+
@redis.ping
|
37
|
+
rescue StandardError => e
|
38
|
+
STDERR.puts e.message
|
39
|
+
return false
|
40
|
+
end
|
41
|
+
|
42
|
+
@cluster_mode = true
|
43
|
+
@masters = []
|
44
|
+
|
45
|
+
redis.connection.each do |c|
|
46
|
+
node = Redis.new url: c[:id]
|
47
|
+
@masters << node
|
48
|
+
end
|
49
|
+
else
|
50
|
+
@masters = [@redis]
|
51
|
+
end
|
52
|
+
|
53
|
+
dbg "Connected to Redis #{mode == "cluster" ? "cluster" : "server"}"
|
54
|
+
(Array === @redis.connection ? @redis.connection : [@redis.connection]) .each do |v|
|
55
|
+
dbg " #{v[:id]}"
|
56
|
+
end
|
57
|
+
self
|
58
|
+
end
|
59
|
+
|
60
|
+
def key(subkey=nil)
|
61
|
+
k = "{channel:#{@namespace}/#{@channel_id}}"
|
62
|
+
return subkey ? "#{k}:#{subkey}" : k
|
63
|
+
end
|
64
|
+
|
65
|
+
def info
|
66
|
+
channel_hash=@redis.hgetall key
|
67
|
+
hash_ttl=@redis.ttl key
|
68
|
+
channel_subs=@redis.hgetall key("subscribers")
|
69
|
+
#...
|
70
|
+
end
|
71
|
+
|
72
|
+
def filter_channels(filters={})
|
73
|
+
script = <<~EOF
|
74
|
+
local prev_cursor = ARGV[1]
|
75
|
+
local pattern = ARGV[2]
|
76
|
+
local scan_batch_size = ARGV[3]
|
77
|
+
|
78
|
+
local min_subscribers = ARGV[4] and #ARGV[4] > 0 and tonumber(ARGV[4])
|
79
|
+
|
80
|
+
local cursor, iteration
|
81
|
+
if pattern and #pattern > 0 then
|
82
|
+
cursor, iteration = unpack(redis.call("SCAN", prev_cursor, "MATCH", pattern, "COUNT", scan_batch_size))
|
83
|
+
else
|
84
|
+
cursor, iteration = unpack(redis.call("SCAN", prev_cursor, "COUNT", scan_batch_size))
|
85
|
+
end
|
86
|
+
|
87
|
+
local matched = {}
|
88
|
+
for _, chankey in pairs(iteration) do
|
89
|
+
local match = true
|
90
|
+
if min_subscribers then
|
91
|
+
match = match and (tonumber(redis.call('HGET', chankey, 'fake_subscribers') or 0) >= min_subscribers)
|
92
|
+
end
|
93
|
+
if match then
|
94
|
+
table.insert(matched, chankey)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
return {cursor, matched}
|
99
|
+
EOF
|
100
|
+
|
101
|
+
results = []
|
102
|
+
batch_size=500
|
103
|
+
masters.each do |m|
|
104
|
+
hash = m.script "load", script
|
105
|
+
cursor, pattern = "0", "{channel:*}"
|
106
|
+
loop do
|
107
|
+
cursor, batch_results = m.evalsha hash, keys: [], argv: [cursor, pattern, batch_size, filters[:min_subscribers]]
|
108
|
+
results += batch_results
|
109
|
+
pattern = ""
|
110
|
+
break if cursor.to_i == 0
|
111
|
+
end
|
112
|
+
end
|
113
|
+
results
|
114
|
+
|
115
|
+
results.map! do |key|
|
116
|
+
m = key.match(/^\{channel\:(.*)\}$/)
|
117
|
+
m[1] || key
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
data/lib/nchan_tools/version.rb
CHANGED
data/nchan_tools.gemspec
CHANGED
@@ -29,15 +29,16 @@ Gem::Specification.new do |spec|
|
|
29
29
|
spec.add_dependency "celluloid"
|
30
30
|
spec.add_dependency "celluloid-io"
|
31
31
|
spec.add_dependency "HDRHistogram"
|
32
|
+
spec.add_dependency "redis"
|
32
33
|
|
33
34
|
spec.add_dependency "websocket-driver"
|
34
35
|
spec.add_dependency 'websocket-extensions'
|
35
36
|
spec.add_dependency "permessage_deflate"
|
36
37
|
spec.add_dependency 'http_parser.rb'
|
37
|
-
if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.2.2')
|
38
|
-
|
39
|
-
end
|
38
|
+
if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.2.2')
|
39
|
+
spec.add_dependency 'http-2'
|
40
|
+
end
|
40
41
|
spec.add_development_dependency "pry"
|
41
|
-
spec.add_development_dependency "bundler"
|
42
|
-
spec.add_development_dependency "rake"
|
42
|
+
spec.add_development_dependency "bundler"
|
43
|
+
spec.add_development_dependency "rake"
|
43
44
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nchan_tools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Leo Ponomarev
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: typhoeus
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: redis
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: websocket-driver
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -182,36 +196,37 @@ dependencies:
|
|
182
196
|
name: bundler
|
183
197
|
requirement: !ruby/object:Gem::Requirement
|
184
198
|
requirements:
|
185
|
-
- - "
|
199
|
+
- - ">="
|
186
200
|
- !ruby/object:Gem::Version
|
187
|
-
version: '
|
201
|
+
version: '0'
|
188
202
|
type: :development
|
189
203
|
prerelease: false
|
190
204
|
version_requirements: !ruby/object:Gem::Requirement
|
191
205
|
requirements:
|
192
|
-
- - "
|
206
|
+
- - ">="
|
193
207
|
- !ruby/object:Gem::Version
|
194
|
-
version: '
|
208
|
+
version: '0'
|
195
209
|
- !ruby/object:Gem::Dependency
|
196
210
|
name: rake
|
197
211
|
requirement: !ruby/object:Gem::Requirement
|
198
212
|
requirements:
|
199
|
-
- - "
|
213
|
+
- - ">="
|
200
214
|
- !ruby/object:Gem::Version
|
201
|
-
version: '
|
215
|
+
version: '0'
|
202
216
|
type: :development
|
203
217
|
prerelease: false
|
204
218
|
version_requirements: !ruby/object:Gem::Requirement
|
205
219
|
requirements:
|
206
|
-
- - "
|
220
|
+
- - ">="
|
207
221
|
- !ruby/object:Gem::Version
|
208
|
-
version: '
|
222
|
+
version: '0'
|
209
223
|
description: publishing, subscribing, testing, and benchmarking utilities for Nchan.
|
210
224
|
email:
|
211
225
|
- leo@nchan.io
|
212
226
|
executables:
|
213
227
|
- nchan-benchmark
|
214
228
|
- nchan-pub
|
229
|
+
- nchan-redis-debug
|
215
230
|
- nchan-sub
|
216
231
|
extensions: []
|
217
232
|
extra_rdoc_files: []
|
@@ -226,17 +241,19 @@ files:
|
|
226
241
|
- bin/setup
|
227
242
|
- exe/nchan-benchmark
|
228
243
|
- exe/nchan-pub
|
244
|
+
- exe/nchan-redis-debug
|
229
245
|
- exe/nchan-sub
|
230
246
|
- lib/nchan_tools.rb
|
231
247
|
- lib/nchan_tools/benchmark.rb
|
232
248
|
- lib/nchan_tools/pubsub.rb
|
249
|
+
- lib/nchan_tools/rdsck.rb
|
233
250
|
- lib/nchan_tools/version.rb
|
234
251
|
- nchan_tools.gemspec
|
235
252
|
homepage: https://nchan.io
|
236
253
|
licenses:
|
237
254
|
- WTFPL
|
238
255
|
metadata: {}
|
239
|
-
post_install_message:
|
256
|
+
post_install_message:
|
240
257
|
rdoc_options: []
|
241
258
|
require_paths:
|
242
259
|
- lib
|
@@ -251,9 +268,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
251
268
|
- !ruby/object:Gem::Version
|
252
269
|
version: '0'
|
253
270
|
requirements: []
|
254
|
-
|
255
|
-
|
256
|
-
signing_key:
|
271
|
+
rubygems_version: 3.1.4
|
272
|
+
signing_key:
|
257
273
|
specification_version: 4
|
258
274
|
summary: Development and testing utilities for Nchan
|
259
275
|
test_files: []
|