nchan_tools 0.1.1 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 03c7684cee5fb57143e8411351aea95d4bcb989cf8f2d66ebe716655f8f07b09
4
- data.tar.gz: ed22f3cdd7eb55acab2ad29684daacd4e09edf0a3ffda1c4e2d5309affa896f3
3
+ metadata.gz: 07a8230e83733a4948ee9cc9df50eea8b698f2762e7f70a054f9708c98bb7ea1
4
+ data.tar.gz: 16ab41364f02c7996441a8f461bd969d9c11af023936e64f0be1f912f81ece88
5
5
  SHA512:
6
- metadata.gz: 3009d9a05e5f8d6d81302cf81389be78ef165621fd433fd08beb1d16d0dcba5fa7cbcc6dd56a83a86a6d4756dcd609fa33136031bb4a50077f0d811405aef0ce
7
- data.tar.gz: cae127fd3166e2442625f3a177e6060561fb8fb0056dc63cc14f03b89404c80a87e1cfd9ee2ed3a0078f91e6665c4b090492f257e5e308368af1cff32a0e23c1
6
+ metadata.gz: 8f2f505b79a22588f37f38f12aa0c461cf5d9db9f3c89375f6a70a9a0df3b8a62f2b778425fbb71bc4b7eb9b8b4fdf275c08cf365dd9d89b57e5396ed5312e1e
7
+ data.tar.gz: 6c4bfe8a215370d37319144375b633bbbb868d1f50ac4dc1f6c2736dce057f78edd30e3995d7be34a5534d80f0ee4cd50f5d0633b63da085c28620c24751a979
@@ -1,21 +1,44 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
3
  require 'securerandom'
4
- require 'nchan_tools/pubsub'
4
+ require 'nchan_tools/benchmark'
5
5
  require "optparse"
6
6
  require 'timers'
7
7
  require 'json'
8
- require "pry"
9
8
  require "HDRHistogram"
10
9
 
11
10
  verbose = false
11
+ save_csv = false
12
+ csv_columns = NchanTools::Benchmark::CSV_COLUMNS_DEFAULT
13
+ init_args = {}
12
14
 
13
15
  opt_parser=OptionParser.new do |opts|
14
16
  opts.on("-v", "--verbose", "somewhat rather extraneously wordful output") do
15
17
  verbose = true
16
18
  end
19
+ opts.on("--csv FILENAME", "Append results to file in CSV format") do |f|
20
+ save_csv = f
21
+ end
22
+ opts.on("--csv-columns col1,col2,...", "csv columns list") do |f|
23
+ csv_columns = f.split(/\W+/).map(&:to_sym)
24
+ end
25
+ opts.on("-t", "--time TIME", "Time to run benchmark") do |v|
26
+ init_args[:time] = v
27
+ end
28
+ opts.on("-r", "--msgrate NUMBER", "Message publishing rate per minute per channel") do |v|
29
+ init_args[:messages_per_channel_per_minute] = v
30
+ end
31
+ opts.on("-p", "--msgpadding NUMBER", "Message padding, in bytes") do |v|
32
+ init_args[:message_padding_bytes] = v
33
+ end
34
+ opts.on("-c", "--channels NUMBER", "Number of channels") do |v|
35
+ init_args[:channels] = v
36
+ end
37
+ opts.on("-s", "--subscribers NUMBER", "Subscribers per channel") do |v|
38
+ init_args[:subscribers_per_channel] = v
39
+ end
17
40
  end
18
- opt_parser.banner="Usage: benchan.rb [options] url1 url2 url3..."
41
+ opt_parser.banner="Usage: nchan-benchmark [options] url1 url2 url3..."
19
42
  opt_parser.parse!
20
43
 
21
44
  urls = []
@@ -23,192 +46,13 @@ urls += ARGV
23
46
  begin
24
47
  urls += STDIN.read_nonblock(100000).split /\s*\n+\s*/
25
48
  rescue IO::WaitReadable
49
+ rescue EOFError
26
50
  end
27
51
 
28
52
  urls.uniq!
29
53
 
30
- class Benchan
31
- def initialize(urls)
32
- @urls = urls
33
- @n = urls.count
34
- @initializing = 0
35
- @ready = 0
36
- @running = 0
37
- @finished = 0
38
- @subs = []
39
- @results = {}
40
- @failed = {}
41
-
42
- @hdrh_publish = nil
43
- @hdrh_receive = nil
44
-
45
- subs = []
46
- end
47
-
48
- def run
49
- puts "connecting to #{@n} Nchan server#{@n > 1 ? "s" : ""}..."
50
- @urls.each do |url|
51
- sub = Subscriber.new(url, 1, client: :websocket, timeout: 900000, extra_headers: {"Accept" => "text/x-json-hdrhistogram"})
52
- sub.on_failure do |err|
53
- unless @results[sub]
54
- unless @results[sub.url]
55
- @failed[sub] = true
56
- abort err, sub
57
- end
58
- end
59
- false
60
- end
61
- sub.on_message do |msg|
62
- msg = msg.to_s
63
- case msg
64
- when "READY"
65
- puts " #{sub.url} ok"
66
- @ready +=1
67
- if @ready == @n
68
- control :run
69
- puts "start benchmark..."
70
- end
71
- when "RUNNING"
72
- puts " #{sub.url} running"
73
- when /^RESULTS\n/
74
- msg = msg[8..-1]
75
- parsed = JSON.parse msg
76
- @results[sub.url] = parsed
77
- 1+1
78
- else
79
- binding.pry
80
- 1+1
81
- end
82
- end
83
- @subs << sub
84
- sub.run
85
- sub.wait :ready, 1
86
- if @failed[sub]
87
- puts " #{sub.url} failed"
88
- else
89
- puts " #{sub.url} ok"
90
- end
91
- end
92
- return if @failed.count > 0
93
- puts "initializing benchmark..."
94
- control :initialize
95
- self.wait
96
- puts "finished."
97
- puts ""
98
- end
99
-
100
- def wait
101
- @subs.each &:wait
102
- end
103
-
104
- def control(msg)
105
- @subs.each { |sub| sub.client.send_data msg.to_s }
106
- end
107
-
108
- def abort(err, src_sub = nil)
109
- puts " #{err}"
110
- @subs.each do |sub|
111
- sub.terminate unless sub == src_sub
112
- end
113
- end
114
-
115
- def hdrhistogram_stats(name, histogram)
116
- fmt = <<-END.gsub(/^ {6}/, '')
117
- %s
118
- min: %.3fms
119
- avg: %.3fms
120
- 99%%ile: %.3fms
121
- max: %.3fms
122
- stddev: %.3fms
123
- samples: %d
124
- END
125
- fmt % [ name,
126
- histogram.min, histogram.mean, histogram.percentile(99.0), histogram.max, histogram.stddev, histogram.count
127
- ]
128
- end
129
-
130
- def results
131
- channels = 0
132
- runtime = []
133
- subscribers = 0
134
- message_length = []
135
- messages_sent = 0
136
- messages_send_confirmed = 0
137
- messages_send_unconfirmed = 0
138
- messages_send_failed = 0
139
- messages_received = 0
140
- messages_unreceived = 0
141
- hdrh_publish = nil
142
- hdrh_receive = nil
143
- @results.each do |url, data|
144
- channels += data["channels"]
145
- runtime << data["run_time_sec"]
146
- subscribers += data["subscribers"]
147
- message_length << data["message_length"]
148
- messages_sent += data["messages"]["sent"]
149
- messages_send_confirmed += data["messages"]["send_confirmed"]
150
- messages_send_unconfirmed += data["messages"]["send_unconfirmed"]
151
- messages_send_failed += data["messages"]["send_failed"]
152
- messages_received += data["messages"]["received"]
153
- messages_unreceived += data["messages"]["unreceived"]
154
-
155
- if data["message_publishing_histogram"]
156
- hdrh = HDRHistogram.unserialize(data["message_publishing_histogram"], unit: :ms, multiplier: 0.001)
157
- if hdrh_publish
158
- hdrh_publish.merge! hdrh
159
- else
160
- hdrh_publish = hdrh
161
- end
162
- end
163
- if data["message_delivery_histogram"]
164
- hdrh = HDRHistogram.unserialize(data["message_delivery_histogram"], unit: :ms, multiplier: 0.001)
165
- if hdrh_receive
166
- hdrh_receive.merge! hdrh
167
- else
168
- hdrh_receive = hdrh
169
- end
170
- end
171
- end
172
-
173
- message_length.uniq!
174
- runtime.uniq!
175
-
176
- fmt = <<-END.gsub(/^ {6}/, '')
177
- Nchan servers: %d
178
- runtime: %s
179
- channels: %d
180
- subscribers: %d
181
- subscribers per channel: %.1f
182
- messages:
183
- length: %s
184
- sent: %d
185
- send_confirmed: %d
186
- send_unconfirmed: %d
187
- send_failed: %d
188
- received: %d
189
- unreceived: %d
190
- send rate: %.3f/sec
191
- receive rate: %.3f/sec
192
- send rate per channel: %.3f/min
193
- receive rate per subscriber: %.3f/min
194
- END
195
- out = fmt % [
196
- @n, runtime.join(","), channels, subscribers, subscribers.to_f/channels,
197
- message_length.join(","), messages_sent, messages_send_confirmed, messages_send_unconfirmed, messages_send_failed,
198
- messages_received, messages_unreceived,
199
- messages_sent.to_f/runtime.max,
200
- messages_received.to_f/runtime.max,
201
- (messages_sent.to_f* 60)/(runtime.max*channels),
202
- (messages_received.to_f * 60)/(runtime.max * subscribers)
203
- ]
204
-
205
- out << hdrhistogram_stats("message publishing latency", hdrh_publish) if hdrh_publish
206
- out << hdrhistogram_stats("message delivery latency", hdrh_receive) if hdrh_receive
207
-
208
- puts out
209
- end
210
- end
211
-
212
- benchan = Benchan.new urls
54
+ benchan = NchanTools::Benchmark.new urls, init_args
213
55
  benchan.run
214
56
  benchan.results
57
+ benchan.append_csv_file(save_csv, csv_columns) if save_csv
58
+
@@ -54,7 +54,7 @@ puts "Publishing to #{url}."
54
54
 
55
55
  loopmsg=("\r"*20) + "sending message #"
56
56
 
57
- pub = Publisher.new url, nostore: true, timeout: timeout, verbose: verbose, websocket: websocket
57
+ pub = NchanTools::Publisher.new url, nostore: true, timeout: timeout, verbose: verbose, websocket: websocket
58
58
  pub.accept=accept
59
59
  pub.nofail=true
60
60
  repeat=true
@@ -0,0 +1,63 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "redis"
4
+ require "optparse"
5
+ require 'nchan_tools/rdsck'
6
+
7
+ $opt = {
8
+ url: "redis://127.0.0.1:6379/",
9
+ verbose: false,
10
+ command: nil
11
+ }
12
+
13
+ def dbg(...)
14
+ if $opt[:verbose]
15
+ print("# ")
16
+ puts(...)
17
+ end
18
+ end
19
+
20
+ opt_parser=OptionParser.new do |opts|
21
+ opts.on("--url", "--url REDIS_URL (#{$opt[:url]})", "Redis server and port..") do |v|
22
+ $opt[:url]=v
23
+ end
24
+ opts.on("-q", "--quiet", "output only results without any other information") do
25
+ $opt[:quiet]=false
26
+ end
27
+ opts.on("--list-channels", "list all Nchan channels on Redis server or cluster") do |v|
28
+ $opt[:command]=:filter_channels
29
+ end
30
+ opts.on("--filter-channels-min-subscribers=[NUMBER]") do |v|
31
+ $opt[:command]=:filter_channels
32
+ $opt[:min_subscribers]=v.to_i
33
+ end
34
+ end
35
+ opt_parser.banner= <<~EOB
36
+ Debugging tools for the Redis server or cluster backing Nchan.
37
+ Usage: nchan-redis-debug [options]
38
+
39
+ WARNING: --list-channels and --filter-channels-* options for this tool
40
+ use the Redis SCAN command. This may increase CPU load on the Redis
41
+ server and may affect the latency of Nchan requests.
42
+ USE THESE OPTIONS WITH GREAT CARE
43
+
44
+ Example:
45
+ nchan-redis-debug --url redis:// --filter-channels-min-subscribers=10
46
+ EOB
47
+ opt_parser.parse!
48
+
49
+ rdsck = Rdsck.new $opt
50
+ if not rdsck.connect
51
+ STDERR.puts "failed to connect to #{$opt[:url]}"
52
+ exit 1
53
+ end
54
+
55
+ case $opt[:command]
56
+ when :filter_channels
57
+ puts "# scanning for channels #{$opt[:min_subscribers] && "with subscribers >= #{$opt[:min_subscribers]}"}"
58
+ chans = rdsck.filter_channels(min_subscribers: $opt[:min_subscribers])
59
+ puts "# found #{chans.count} channel#{chans.count != 1 && "s"}#{chans.count == 0 ? "." : ":"}"
60
+ puts chans.join("\n")
61
+ else
62
+ puts "Nothing to do"
63
+ end
@@ -33,7 +33,7 @@ opt_parser=OptionParser.new do |opts|
33
33
  opts.on("-p", "--parallel NUM (#{par})", "number of parallel clients"){|v| par = v.to_i}
34
34
  opts.on("-t", "--timeout SEC (#{opt[:timeout]})", "Long-poll timeout"){|v| opt[:timeout] = v}
35
35
  opts.on("-q", "--quit STRING (#{opt[:quit_message]})", "Quit message"){|v| opt[:quit_message] = v}
36
- opts.on("-c", "--client STRING (#{opt[:client]})", "sub client (one of #{Subscriber::Client.unique_aliases.join ', '})") do |v|
36
+ opts.on("-c", "--client STRING (#{opt[:client]})", "sub client (one of #{NchanTools::Subscriber::Client.unique_aliases.join ', '})") do |v|
37
37
  opt[:client] = v.to_sym
38
38
  end
39
39
  opts.on("--content-type", "show received content-type"){|v| print_content_type = true}
@@ -68,7 +68,7 @@ if origin
68
68
  opt[:extra_headers]['Origin'] = origin
69
69
  end
70
70
 
71
- sub = Subscriber.new url, par, opt
71
+ sub = NchanTools::Subscriber.new url, par, opt
72
72
 
73
73
 
74
74
  NOMSGF="\r"*30 + "Received message %i, len:%i"
@@ -111,7 +111,7 @@ sub.on_message do |msg|
111
111
  end
112
112
 
113
113
  sub.on_failure do |err_msg|
114
- if Subscriber::IntervalPollClient === sub.client
114
+ if NchanTools::Subscriber::IntervalPollClient === sub.client
115
115
  unless err_msg.match(/\(code 304\)/)
116
116
  false
117
117
  end
@@ -0,0 +1,241 @@
1
+ require 'nchan_tools/pubsub'
2
+ require 'securerandom'
3
+ require 'timers'
4
+ require 'json'
5
+
6
+ module NchanTools
7
+ class Benchmark
8
+ CSV_COLUMNS_ALL=%i[servers runtime channels channels_K channels_M subscribers message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed messages_received messages_unreceived messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber message_publishing_avg message_publishing_99th message_publishing_max message_publishing_stddev message_publishing_count message_delivery_avg message_delivery_99th message_delivery_max message_delivery_stddev message_delivery_count]
9
+ CSV_COLUMNS_DEFAULT=%i[servers runtime channels subscribers message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed messages_received messages_unreceived messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber message_publishing_avg message_publishing_99th message_publishing_max message_publishing_stddev message_publishing_count message_delivery_avg message_delivery_99th message_delivery_max message_delivery_stddev message_delivery_count]
10
+ class BenchmarkError < StandardError
11
+ end
12
+ def initialize(urls, init_args=nil)
13
+ @urls = urls
14
+ @n = urls.count
15
+ @initializing = 0
16
+ @ready = 0
17
+ @running = 0
18
+ @finished = 0
19
+ @subs = []
20
+ @results = {}
21
+ @failed = {}
22
+
23
+ @init_args = init_args
24
+ @histograms = {}
25
+
26
+ subs = []
27
+ end
28
+
29
+ def run
30
+ puts "connecting to #{@n} Nchan server#{@n > 1 ? "s" : ""}..."
31
+ @urls.each do |url|
32
+ sub = NchanTools::Subscriber.new(url, 1, client: :websocket, timeout: 900000, extra_headers: {"Accept" => "text/x-json-hdrhistogram"})
33
+ sub.on_failure do |err|
34
+ unless @results[sub]
35
+ unless @results[sub.url]
36
+ @failed[sub] = true
37
+ abort err, sub
38
+ end
39
+ end
40
+ false
41
+ end
42
+ sub.on_message do |msg|
43
+ msg = msg.to_s
44
+ case msg
45
+ when /^READY/
46
+ puts " #{sub.url} ok"
47
+ @ready +=1
48
+ if @ready == @n
49
+ puts "start benchmark..."
50
+ control :run
51
+ end
52
+ when /^RUNNING/
53
+ puts " #{sub.url} running"
54
+ when /^RESULTS\n/
55
+ msg = msg[8..-1]
56
+ parsed = JSON.parse msg
57
+
58
+ #backwards-compatible histogram fields
59
+ parsed["histograms"]||={}
60
+ if parsed[:message_publishing_histogram] then
61
+ parsed[:histograms]["message publishing"]=parsed[:message_publishing_histogram]
62
+ end
63
+ if parsed[:message_delivery_histogram] then
64
+ parsed[:histograms]["message delivery"]=parsed[:message_delivery_histogram]
65
+ end
66
+
67
+ @results[sub.url] = parsed
68
+ @results[sub.url]["raw"] = msg if @results[sub.url]
69
+ sub.client.send_close
70
+ when /^INITIALIZING/
71
+ #do nothing
72
+ else
73
+ raise BenchmarkError, "unexpected server response: #{msg}"
74
+ end
75
+ end
76
+ @subs << sub
77
+ sub.run
78
+ sub.wait :ready, 1
79
+ if @failed[sub]
80
+ puts " #{sub.url} failed"
81
+ else
82
+ puts " #{sub.url} ok"
83
+ end
84
+ end
85
+ return if @failed.count > 0
86
+ puts "initializing benchmark..."
87
+ control :init
88
+ self.wait
89
+ puts "finished."
90
+ puts ""
91
+ end
92
+
93
+ def wait
94
+ @subs.each &:wait
95
+ end
96
+
97
+ def control(msg)
98
+ if @init_args && (msg.to_sym ==:init || msg.to_sym ==:initialize)
99
+ msg = "#{msg.to_s} #{@init_args.map{|k,v| "#{k}=#{v}"}.join(" ")}"
100
+ end
101
+ @subs.each { |sub| sub.client.send_data msg.to_s }
102
+ end
103
+
104
+ def abort(err, src_sub = nil)
105
+ puts " #{err}"
106
+ @subs.each do |sub|
107
+ sub.terminate unless sub == src_sub
108
+ end
109
+ end
110
+
111
+ def hdrhistogram_stats(name, histogram)
112
+ fmt = <<-END.gsub(/^ {6}/, '')
113
+ %s
114
+ min: %.3fms
115
+ avg: %.3fms
116
+ 99%%ile: %.3fms
117
+ max: %.3fms
118
+ stddev: %.3fms
119
+ samples: %d
120
+ END
121
+ fmt % [ name,
122
+ histogram.min, histogram.mean, histogram.percentile(99.0), histogram.max, histogram.stddev, histogram.count
123
+ ]
124
+ end
125
+
126
+ def results
127
+ @channels = 0
128
+ @runtime = []
129
+ @subscribers = 0
130
+ @message_length = []
131
+ @messages_sent = 0
132
+ @messages_send_confirmed = 0
133
+ @messages_send_unconfirmed = 0
134
+ @messages_send_failed = 0
135
+ @messages_received = 0
136
+ @messages_unreceived = 0
137
+ @histograms = {}
138
+ @results.each do |url, data|
139
+ @channels += data["channels"]
140
+ @runtime << data["run_time_sec"]
141
+ @subscribers += data["subscribers"]
142
+ @message_length << data["message_length"]
143
+ @messages_sent += data["messages"]["sent"]
144
+ @messages_send_confirmed += data["messages"]["send_confirmed"]
145
+ @messages_send_unconfirmed += data["messages"]["send_unconfirmed"]
146
+ @messages_send_failed += data["messages"]["send_failed"]
147
+ @messages_received += data["messages"]["received"]
148
+ @messages_unreceived += data["messages"]["unreceived"]
149
+ if data["histograms"]
150
+ data["histograms"].each do |name, str|
151
+ name = name.to_sym
152
+ hdrh = HDRHistogram.unserialize(str, unit: :ms, multiplier: 0.001)
153
+ if @histograms[name]
154
+ @histograms[name].merge! hdrh
155
+ else
156
+ @histograms[name] = hdrh
157
+ end
158
+ end
159
+ end
160
+ end
161
+
162
+ @message_length = @message_length.inject(0, :+).to_f / @message_length.size
163
+ @runtime = @runtime.inject(0, :+).to_f / @runtime.size
164
+
165
+ fmt = <<-END.gsub(/^ {6}/, '')
166
+ Nchan servers: %d
167
+ runtime: %d
168
+ channels: %d
169
+ subscribers: %d
170
+ subscribers per channel: %.1f
171
+ messages:
172
+ length: %d
173
+ sent: %d
174
+ send_confirmed: %d
175
+ send_unconfirmed: %d
176
+ send_failed: %d
177
+ received: %d
178
+ unreceived: %d
179
+ send rate: %.3f/sec
180
+ receive rate: %.3f/sec
181
+ send rate per channel: %.3f/min
182
+ receive rate per subscriber: %.3f/min
183
+ END
184
+ out = fmt % [
185
+ @n, @runtime, @channels, @subscribers, @subscribers.to_f/@channels,
186
+ @message_length, @messages_sent, @messages_send_confirmed, @messages_send_unconfirmed, @messages_send_failed,
187
+ @messages_received, @messages_unreceived,
188
+ @messages_sent.to_f/@runtime,
189
+ @messages_received.to_f/@runtime,
190
+ (@messages_sent.to_f* 60)/(@runtime * @channels),
191
+ (@messages_received.to_f * 60)/(@runtime * @subscribers)
192
+ ]
193
+ @histograms.each do |name, histogram|
194
+ out << hdrhistogram_stats("#{name} latency:", histogram)
195
+ end
196
+
197
+ puts out
198
+ end
199
+
200
+ def append_csv_file(file, columns=Benchmark::CSV_COLUMNS_DEFAULT)
201
+ require "csv"
202
+ write_headers = File.zero?(file) || !File.exists?(file)
203
+ headers = columns
204
+ vals = {
205
+ servers: @n,
206
+ runtime: @runtime,
207
+ channels: @channels,
208
+ channels_K: @channels/1000.0,
209
+ channels_M: @channels/1000000.0,
210
+ subscribers: @subscribers * @channels,
211
+ message_length: @message_length,
212
+ messages_sent: @messages_sent,
213
+ messages_send_confirmed: @messages_send_confirmed,
214
+ messages_send_unconfirmed: @messages_send_unconfirmed,
215
+ messages_send_failed: @messages_send_failed,
216
+ messages_received: @messages_received,
217
+ messages_unreceived: @messages_unreceived,
218
+ messages_send_rate: @messages_sent.to_f/@runtime,
219
+ messages_receive_rate: @messages_received.to_f/@runtime,
220
+ messages_send_rate_per_channel: (@messages_sent.to_f* 60)/(@runtime * @channels),
221
+ messages_receive_rate_per_subscriber: (@messages_received.to_f * 60)/(@runtime * @subscribers * @channels)
222
+ }
223
+ @histograms.each do |name, histogram|
224
+ vals["#{name}_avg".to_sym]=histogram.mean
225
+ vals["#{name}_95th".to_sym]=histogram.percentile(95.0)
226
+ vals["#{name}_99th".to_sym]=histogram.percentile(99.0)
227
+ vals["#{name}_max".to_sym]=histogram.max
228
+ vals["#{name}_stddev".to_sym]=histogram.stddev
229
+ vals["#{name}_count".to_sym]=histogram.count
230
+ end
231
+
232
+ row = []
233
+ headers.each { |header| row << (vals[header.to_sym] || "-")}
234
+
235
+ csv = CSV.open(file, "a", {headers: headers, write_headers: write_headers})
236
+ csv << row
237
+ csv.flush
238
+ csv.close
239
+ end
240
+ end
241
+ end
@@ -3,7 +3,7 @@ require 'typhoeus'
3
3
  require 'json'
4
4
  require 'oga'
5
5
  require 'yaml'
6
- require 'pry'
6
+
7
7
  require 'celluloid/current'
8
8
  require 'date'
9
9
  Typhoeus::Config.memoize = false
@@ -46,15 +46,12 @@ module URI
46
46
  u
47
47
  end
48
48
  end
49
-
50
- $seq = 0
49
+ module NchanTools
51
50
  class Message
52
51
  attr_accessor :content_type, :message, :times_seen, :etag, :last_modified, :eventsource_event
53
52
  def initialize(msg, last_modified=nil, etag=nil)
54
53
  @times_seen=1
55
54
  @message, @last_modified, @etag = msg, last_modified, etag
56
- $seq+=1
57
- @seq = $seq
58
55
  @idhist = []
59
56
  end
60
57
  def serverside_id
@@ -248,7 +245,7 @@ class Subscriber
248
245
  end
249
246
  end
250
247
 
251
- class SubscriberError < Exception
248
+ class SubscriberError < StandardError
252
249
  end
253
250
  class Client
254
251
  attr_accessor :concurrency
@@ -490,6 +487,14 @@ class Subscriber
490
487
  @ws.binary data
491
488
  end
492
489
 
490
+ def send_ping(msg=nil)
491
+ @ws.ping(msg)
492
+ end
493
+
494
+ def send_close(reason=nil, code=1000)
495
+ @ws.close(reason, code)
496
+ end
497
+
493
498
  def write(data)
494
499
  @sock.write data
495
500
  end
@@ -582,11 +587,11 @@ class Subscriber
582
587
  end
583
588
 
584
589
  bundle.ws.on :ping do |ev|
585
- @on_ping.call if @on_ping
590
+ @subscriber.on(:ping).call ev, bundle
586
591
  end
587
592
 
588
593
  bundle.ws.on :pong do |ev|
589
- @on_pong.call if @on_pong
594
+ @subscriber.on(:pong).call ev, bundle
590
595
  end
591
596
 
592
597
  bundle.ws.on :error do |ev|
@@ -648,13 +653,6 @@ class Subscriber
648
653
  end
649
654
  end
650
655
 
651
- def on_ping
652
- @on_ping = Proc.new if block_given?
653
- end
654
- def on_pong
655
- @on_pong = Proc.new if block_given?
656
- end
657
-
658
656
  def listen(bundle)
659
657
  while @ws[bundle]
660
658
  begin
@@ -684,10 +682,10 @@ class Subscriber
684
682
  private :ws_client
685
683
 
686
684
  def send_ping(data=nil)
687
- ws_client.ping data
685
+ ws_client.send_ping data
688
686
  end
689
- def send_close(code=1000, reason=nil)
690
- ws_client.send_close code, reason
687
+ def send_close(reason=nil, code=1000)
688
+ ws_client.send_close reason, code
691
689
  end
692
690
  def send_data(data)
693
691
  ws_client.send_data data
@@ -697,17 +695,18 @@ class Subscriber
697
695
  end
698
696
 
699
697
  def close(bundle)
700
- if bundle
698
+ if bundle then
701
699
  @ws.delete bundle
702
700
  bundle.sock.close unless bundle.sock.closed?
703
701
  end
704
702
  @connected -= 1
705
- if @connected <= 0
706
- binding.pry unless @ws.count == 0
703
+ if @connected <= 0 then
704
+ until @ws.count == 0 do
705
+ sleep 0.1
706
+ end
707
707
  @cooked.signal true
708
708
  end
709
709
  end
710
-
711
710
  end
712
711
 
713
712
  class LongPollClient < Client
@@ -1521,6 +1520,8 @@ class Subscriber
1521
1520
 
1522
1521
  attr_accessor :url, :client, :messages, :max_round_trips, :quit_message, :errors, :concurrency, :waiting, :finished, :client_class, :log
1523
1522
  def initialize(url, concurrency=1, opt={})
1523
+ @empty_block = Proc.new {}
1524
+ @on={}
1524
1525
  @care_about_message_ids=opt[:use_message_id].nil? ? true : opt[:use_message_id]
1525
1526
  @url=url
1526
1527
  @quit_message = opt[:quit_message]
@@ -1615,6 +1616,14 @@ class Subscriber
1615
1616
  @client.poke until_what, timeout
1616
1617
  end
1617
1618
 
1619
+ def on(evt_name = nil, &block)
1620
+ if block_given?
1621
+ @on[evt_name.to_sym] = block
1622
+ else
1623
+ @on[evt_name.to_sym] or @empty_block
1624
+ end
1625
+ end
1626
+
1618
1627
  def on_message(msg=nil, bundle=nil, &block)
1619
1628
  #puts "received message #{msg && msg.to_s[0..15]}"
1620
1629
  if block_given?
@@ -1646,7 +1655,7 @@ end
1646
1655
  class Publisher
1647
1656
  #include Celluloid
1648
1657
 
1649
- class PublisherError < Exception
1658
+ class PublisherError < StandardError
1650
1659
  end
1651
1660
 
1652
1661
  attr_accessor :messages, :response, :response_code, :response_body, :nofail, :accept, :url, :extra_headers, :verbose, :ws, :channel_info, :channel_info_type
@@ -1660,6 +1669,7 @@ class Publisher
1660
1669
  @accept = opt[:accept]
1661
1670
  @verbose = opt[:verbose]
1662
1671
  @on_response = opt[:on_response]
1672
+ @http2 = opt[:http2]
1663
1673
 
1664
1674
  @ws_wait_until_response = true
1665
1675
 
@@ -1800,6 +1810,7 @@ class Publisher
1800
1810
  headers = {:'Content-Type' => content_type, :'Accept' => accept}
1801
1811
  headers[:'X-Eventsource-Event'] = eventsource_event if eventsource_event
1802
1812
  headers.merge! @extra_headers if @extra_headers
1813
+
1803
1814
  post = Typhoeus::Request.new(
1804
1815
  @url,
1805
1816
  headers: headers,
@@ -1807,7 +1818,8 @@ class Publisher
1807
1818
  body: body,
1808
1819
  timeout: @timeout || PUBLISH_TIMEOUT,
1809
1820
  connecttimeout: @timeout || PUBLISH_TIMEOUT,
1810
- verbose: @verbose
1821
+ verbose: @verbose,
1822
+ http_version: @http2 ? :httpv2_0 : :none
1811
1823
  )
1812
1824
  if body && @messages
1813
1825
  msg=Message.new body
@@ -1895,3 +1907,4 @@ class Publisher
1895
1907
 
1896
1908
 
1897
1909
  end
1910
+ end
@@ -0,0 +1,111 @@
1
+ class Rdsck
2
+ attr_accessor :url, :verbose, :namespace
3
+ attr_accessor :redis, :masters
4
+ def initialize(opt)
5
+ @url=opt[:url]
6
+ @verbose=opt[:verbose]
7
+ @namespace=opt[:namespace]
8
+ end
9
+
10
+ def cluster?
11
+ @cluster_mode
12
+ end
13
+
14
+ def connect
15
+ begin
16
+ @redis=Redis.new url: $opt[:url]
17
+ mode = redis.info["redis_mode"]
18
+ rescue StandardError => e
19
+ STDERR.puts e.message
20
+ return false
21
+ end
22
+
23
+ if mode == "cluster"
24
+ @redis.close
25
+ begin
26
+ @redis=Redis.new cluster: [$opt[:url]]
27
+ @redis.ping
28
+ rescue StandardError => e
29
+ STDERR.puts e.message
30
+ return false
31
+ end
32
+
33
+ @cluster_mode = true
34
+ @masters = []
35
+
36
+ redis.connection.each do |c|
37
+ node = Redis.new url: c[:id]
38
+ @masters << node
39
+ end
40
+ else
41
+ @masters = [@redis]
42
+ end
43
+
44
+ dbg "Connected to Redis #{mode == "cluster" ? "cluster" : "server"}"
45
+ (Array === @redis.connection ? @redis.connection : [@redis.connection]) .each do |v|
46
+ dbg " #{v[:id]}"
47
+ end
48
+ self
49
+ end
50
+
51
+ def key(subkey=nil)
52
+ k = "{channel:#{$opt[:namespace]}/#{$opt[:channel_id]}}"
53
+ return subkey ? "#{k}:#{subkey}" : k
54
+ end
55
+
56
+ def info
57
+ channel_hash=@redis.hgetall key
58
+ hash_ttl=@redis.ttl key
59
+ channel_subs=@redis.hgetall key("subscribers")
60
+ #...
61
+ end
62
+
63
+ def filter_channels(filters={})
64
+ script = <<~EOF
65
+ local prev_cursor = ARGV[1]
66
+ local pattern = ARGV[2]
67
+ local scan_batch_size = ARGV[3]
68
+
69
+ local min_subscribers = ARGV[4] and #ARGV[4] > 0 and tonumber(ARGV[4])
70
+
71
+ local cursor, iteration
72
+ if pattern and #pattern > 0 then
73
+ cursor, iteration = unpack(redis.call("SCAN", prev_cursor, "MATCH", pattern, "COUNT", scan_batch_size))
74
+ else
75
+ cursor, iteration = unpack(redis.call("SCAN", prev_cursor, "COUNT", scan_batch_size))
76
+ end
77
+
78
+ local matched = {}
79
+ for _, chankey in pairs(iteration) do
80
+ local match = true
81
+ if min_subscribers then
82
+ match = match and (tonumber(redis.call('HGET', chankey, 'fake_subscribers') or 0) >= min_subscribers)
83
+ end
84
+ if match then
85
+ table.insert(matched, chankey)
86
+ end
87
+ end
88
+
89
+ return {cursor, matched}
90
+ EOF
91
+
92
+ results = []
93
+ batch_size=2
94
+ masters.each do |m|
95
+ hash = m.script "load", script
96
+ cursor, pattern = "0", "{channel:*}"
97
+ loop do
98
+ cursor, batch_results = m.evalsha hash, keys: [], argv: [cursor, pattern, batch_size, filters[:min_subscribers]]
99
+ results += batch_results
100
+ pattern = ""
101
+ break if cursor.to_i == 0
102
+ end
103
+ end
104
+ results
105
+
106
+ results.map! do |key|
107
+ m = key.match(/^\{channel\:(.*)\}$/)
108
+ m[1] || key
109
+ end
110
+ end
111
+ end
@@ -1,3 +1,3 @@
1
1
  module NchanTools
2
- VERSION = "0.1.1"
2
+ VERSION = "0.1.6"
3
3
  end
@@ -34,9 +34,9 @@ Gem::Specification.new do |spec|
34
34
  spec.add_dependency 'websocket-extensions'
35
35
  spec.add_dependency "permessage_deflate"
36
36
  spec.add_dependency 'http_parser.rb'
37
- if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.2.2')
38
- spec.add_dependency 'http-2'
39
- end
37
+ if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.2.2')
38
+ spec.add_dependency 'http-2'
39
+ end
40
40
  spec.add_development_dependency "pry"
41
41
  spec.add_development_dependency "bundler", "~> 1.16"
42
42
  spec.add_development_dependency "rake", "~> 10.0"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nchan_tools
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Leo Ponomarev
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-09-30 00:00:00.000000000 Z
11
+ date: 2020-12-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: typhoeus
@@ -212,6 +212,7 @@ email:
212
212
  executables:
213
213
  - nchan-benchmark
214
214
  - nchan-pub
215
+ - nchan-redis-debug
215
216
  - nchan-sub
216
217
  extensions: []
217
218
  extra_rdoc_files: []
@@ -226,16 +227,19 @@ files:
226
227
  - bin/setup
227
228
  - exe/nchan-benchmark
228
229
  - exe/nchan-pub
230
+ - exe/nchan-redis-debug
229
231
  - exe/nchan-sub
230
232
  - lib/nchan_tools.rb
233
+ - lib/nchan_tools/benchmark.rb
231
234
  - lib/nchan_tools/pubsub.rb
235
+ - lib/nchan_tools/rdsck.rb
232
236
  - lib/nchan_tools/version.rb
233
237
  - nchan_tools.gemspec
234
238
  homepage: https://nchan.io
235
239
  licenses:
236
240
  - WTFPL
237
241
  metadata: {}
238
- post_install_message:
242
+ post_install_message:
239
243
  rdoc_options: []
240
244
  require_paths:
241
245
  - lib
@@ -250,9 +254,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
250
254
  - !ruby/object:Gem::Version
251
255
  version: '0'
252
256
  requirements: []
253
- rubyforge_project:
254
- rubygems_version: 2.7.7
255
- signing_key:
257
+ rubygems_version: 3.1.4
258
+ signing_key:
256
259
  specification_version: 4
257
260
  summary: Development and testing utilities for Nchan
258
261
  test_files: []