nchan_tools 0.1.4 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe6f6c40e0e15751d878e26903dd3c9812919069b8f338e0463529e836c69ae2
-  data.tar.gz: 41b7845122c2ef1792f03f8bdb9533276c2fb5b7507d53ac670b05f3eebbf2cd
+  metadata.gz: ec4e4cc624f57a6d6a6684bf71db8d75b58a4e2e823e258490b9be66eb5de396
+  data.tar.gz: bea9e679190cff5749804e8f8e3ef14ca825edd197ef0381d4c8225f87a07ca8
 SHA512:
-  metadata.gz: 34eb95c4a0ff0c54018ec9a1671880816582a3a7c3e3b0d1eca69c64349d78c119f5bec9daf240266ef3f1c6b4c04359bc3b7ee0cef746be1bfd881b210d7298
-  data.tar.gz: be8316e86a2b32d411fb485d73658071e713dfad3396d75f649852d2158e7b1f85b40a0bce4e6d12ac296c000886ac5c518369b52f5131270fe14a40db66ca03
+  metadata.gz: 38c2d66f98c4eb30a48968197d16513ba080d68bdaf417f884fd4af72f7d63ab0788c46554d636f1a5e6c344861e7c08d885432e9e5e6d44cdb0bd1b630bc4de
+  data.tar.gz: 02fb75182dacf15033c6d5fe3b0191b9c62063aec06fe67942ba2b6c0559368ce86af93b0ab507f3924e524a180e6bed9b2a7dd7b77fbdeca3af702abb141807

data/exe/nchan-benchmark

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'securerandom'
-require 'nchan_tools/pubsub'
+require 'nchan_tools/benchmark'
 require "optparse"
 require 'timers'
 require 'json'
@@ -9,7 +9,7 @@ require "HDRHistogram"
 
 verbose = false
 save_csv = false
-
+csv_columns = NchanTools::Benchmark::CSV_COLUMNS_DEFAULT
 init_args = {}
 
 opt_parser=OptionParser.new do |opts|
@@ -19,6 +19,9 @@ opt_parser=OptionParser.new do |opts|
   opts.on("--csv FILENAME", "Append results to file in CSV format") do |f|
     save_csv = f
   end
+  opts.on("--csv-columns col1,col2,...", "csv columns list") do |f|
+    csv_columns = f.split(/\W+/).map(&:to_sym)
+  end
   opts.on("-t", "--time TIME", "Time to run benchmark") do |v|
     init_args[:time] = v
   end
@@ -43,225 +46,13 @@ urls += ARGV
 begin
   urls += STDIN.read_nonblock(100000).split /\s*\n+\s*/
 rescue IO::WaitReadable
+rescue EOFError
 end
 
 urls.uniq!
 
-class Benchan
-  class BenchmarkError < StandardError
-  end
-  def initialize(urls, init_args=nil)
-    @urls = urls
-    @n = urls.count
-    @initializing = 0
-    @ready = 0
-    @running = 0
-    @finished = 0
-    @subs = []
-    @results = {}
-    @failed = {}
-    
-    @init_args = init_args
-    
-    @hdrh_publish = nil
-    @hdrh_receive = nil
-    
-    subs = []
-  end
-  
-  def run
-    puts "connecting to #{@n} Nchan server#{@n > 1 ? "s" : ""}..."
-    @urls.each do |url|
-      sub = Subscriber.new(url, 1, client: :websocket, timeout: 900000, extra_headers: {"Accept" => "text/x-json-hdrhistogram"})
-      sub.on_failure do |err|
-        unless @results[sub]
-          unless @results[sub.url]
-            @failed[sub] = true
-            abort err, sub
-          end
-        end
-        false
-      end
-      sub.on_message do |msg|
-        msg = msg.to_s
-        case msg
-        when /^READY/
-          puts   "  #{sub.url} ok"
-          @ready +=1
-          if @ready == @n
-            control :run
-            puts "start benchmark..."
-          end
-        when /^RUNNING/
-          puts   "  #{sub.url} running"
-        when /^RESULTS\n/
-          msg = msg[8..-1]
-          parsed = JSON.parse msg
-          @results[sub.url] = parsed
-          @results[sub.url]["raw"] = msg if @results[sub.url]
-          1+1
-        when /^INITIALIZING/
-          #do nothing
-        else
-          raise BenchmarkError, "unexpected server response: #{msg}"
-        end
-      end
-      @subs << sub
-      sub.run
-      sub.wait :ready, 1
-      if @failed[sub]
-        puts "  #{sub.url} failed"
-      else
-        puts "  #{sub.url} ok"
-      end
-    end
-    return if @failed.count > 0
-    puts "initializing benchmark..."
-    control :init
-    self.wait
-    puts "finished."
-    puts ""
-  end
-  
-  def wait
-    @subs.each &:wait
-  end
-  
-  def control(msg)
-    if @init_args && (msg.to_sym ==:init || msg.to_sym ==:initialize)
-      msg = "#{msg.to_s} #{@init_args.map{|k,v| "#{k}=#{v}"}.join(" ")}"
-    end
-    @subs.each { |sub| sub.client.send_data msg.to_s }
-  end
-  
-  def abort(err, src_sub = nil)
-    puts "  #{err}"
-    @subs.each do |sub|
-      sub.terminate unless sub == src_sub
-    end
-  end
-  
-  def hdrhistogram_stats(name, histogram)
-    fmt = <<-END.gsub(/^ {6}/, '')
-      %s
-        min:                         %.3fms
-        avg:                         %.3fms
-        99%%ile:                      %.3fms
-        max:                         %.3fms
-        stddev:                      %.3fms
-        samples:                     %d
-    END
-    fmt % [ name,
-      histogram.min, histogram.mean, histogram.percentile(99.0), histogram.max, histogram.stddev, histogram.count
-    ]
-  end
-  
-  def results
-    @channels = 0
-    @runtime = []
-    @subscribers = 0
-    @message_length = []
-    @messages_sent = 0
-    @messages_send_confirmed = 0
-    @messages_send_unconfirmed = 0
-    @messages_send_failed = 0
-    @messages_received = 0
-    @messages_unreceived = 0
-    @hdrh_publish = nil
-    @hdrh_receive = nil
-    @results.each do |url, data|
-      @channels += data["channels"]
-      @runtime << data["run_time_sec"]
-      @subscribers += data["subscribers"]
-      @message_length << data["message_length"]
-      @messages_sent += data["messages"]["sent"]
-      @messages_send_confirmed += data["messages"]["send_confirmed"]
-      @messages_send_unconfirmed += data["messages"]["send_unconfirmed"]
-      @messages_send_failed += data["messages"]["send_failed"]
-      @messages_received += data["messages"]["received"]
-      @messages_unreceived += data["messages"]["unreceived"]
-      
-      if data["message_publishing_histogram"]
-        hdrh = HDRHistogram.unserialize(data["message_publishing_histogram"], unit: :ms, multiplier: 0.001)
-        if @hdrh_publish
-          @hdrh_publish.merge! hdrh
-        else
-          @hdrh_publish = hdrh
-        end
-      end
-      if data["message_delivery_histogram"]
-        hdrh = HDRHistogram.unserialize(data["message_delivery_histogram"], unit: :ms, multiplier: 0.001)
-        if @hdrh_receive
-          @hdrh_receive.merge! hdrh
-        else
-          @hdrh_receive = hdrh
-        end
-      end
-    end
-    
-    @message_length = @message_length.sum.to_f / @message_length.size
-    @runtime = @runtime.sum.to_f / @runtime.size
-    
-    fmt = <<-END.gsub(/^ {6}/, '')
-      Nchan servers:                 %d
-      runtime:                       %d
-      channels:                      %d
-      subscribers:                   %d
-      subscribers per channel:       %.1f
-      messages:
-        length:                      %d
-        sent:                        %d
-        send_confirmed:              %d
-        send_unconfirmed:            %d
-        send_failed:                 %d
-        received:                    %d
-        unreceived:                  %d
-        send rate:                   %.3f/sec
-        receive rate:                %.3f/sec
-        send rate per channel:       %.3f/min
-        receive rate per subscriber: %.3f/min
-    END
-    out = fmt % [
-      @n, @runtime, @channels, @subscribers, @subscribers.to_f/@channels,
-      @message_length, @messages_sent, @messages_send_confirmed, @messages_send_unconfirmed, @messages_send_failed, 
-      @messages_received, @messages_unreceived,
-      @messages_sent.to_f/@runtime,
-      @messages_received.to_f/@runtime,
-      (@messages_sent.to_f* 60)/(@runtime * @channels),
-      (@messages_received.to_f * 60)/(@runtime * @subscribers)
-    ]
-    
-    out << hdrhistogram_stats("message publishing latency", @hdrh_publish) if @hdrh_publish
-    out << hdrhistogram_stats("message delivery latency", @hdrh_receive) if @hdrh_receive
-    
-    puts out
-  end
-  
-  def append_csv_file(file)
-    require "csv"
-    write_headers = File.zero?(file)
-    headers = %i[servers runtime channels subscribers 
-      message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed
-      messages_send_received messages_send_unreceived 
-      messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber
-      message_publishing_response_avg message_publishing_response_99percentile message_publishing_response_stddev message_publishing_response_count
-      message_delivery_avg message_delivery_99percentile message_delivery_stddev message_delivery_count]
-    csv = CSV.open(file, "a", {headers: headers, write_headers: write_headers})
-    csv << [@n, @runtime, @channels, @subscribers, 
-      @message_length, @messages_sent, @messages_send_confirmed, @messages_send_unconfirmed, @messages_send_failed, 
-      @messages_received, @messages_unreceived,
-      @messages_sent.to_f/@runtime, @messages_received.to_f/@runtime,
-      (@messages_sent.to_f* 60)/(@runtime * @channels), (@messages_received.to_f * 60)/(@runtime * @subscribers),
-      @hdrh_publish.mean, @hdrh_publish.percentile(99.0), @hdrh_publish.max, @hdrh_publish.stddev, @hdrh_publish.count,
-      @hdrh_receive.mean, @hdrh_receive.percentile(99.0), @hdrh_receive.max, @hdrh_receive.stddev, @hdrh_receive.count
-    ]
-    csv.flush
-    csv.close
-  end
-end
-
-benchan = Benchan.new urls, init_args
+benchan = NchanTools::Benchmark.new urls, init_args
 benchan.run
 benchan.results
-benchan.append_csv_file(save_csv) if save_csv
+benchan.append_csv_file(save_csv, csv_columns) if save_csv
 

data/exe/nchan-pub

@@ -54,7 +54,7 @@ puts "Publishing to #{url}."
 
 loopmsg=("\r"*20) + "sending message #"
 
-pub = Publisher.new url, nostore: true, timeout: timeout, verbose: verbose, websocket: websocket
+pub = NchanTools::Publisher.new url, nostore: true, timeout: timeout, verbose: verbose, websocket: websocket
 pub.accept=accept
 pub.nofail=true
 repeat=true

data/exe/nchan-redis-debug

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require "redis"
+require "optparse"
+require 'nchan_tools/rdsck'
+
+$opt = {
+  url: "redis://127.0.0.1:6379/",
+  verbose: false,
+  command: nil
+}
+
+opt_parser=OptionParser.new do |opts|
+  opts.on("--url", "--url REDIS_URL (#{$opt[:url]})", "Redis server and port..") do |v|
+    $opt[:url]=v
+  end
+  opts.on("-q", "--quiet", "output only results without any other information") do
+    $opt[:quiet]=false
+  end
+  opts.on("--list-channels", "list all Nchan channels on Redis server or cluster") do |v|
+    $opt[:command]=:filter_channels
+  end
+  opts.on("--filter-channels-min-subscribers=[NUMBER]") do |v|
+    $opt[:command]=:filter_channels
+    $opt[:min_subscribers]=v.to_i
+  end
+end
+opt_parser.banner= <<~EOB
+  Debugging tools for the Redis server or cluster backing Nchan.
+  Usage: nchan-redis-debug [options]
+  
+  WARNING: --list-channels and --filter-channels-* options for this tool
+    use the Redis SCAN command. This may increase CPU load on the Redis 
+    server and may affect the latency of Nchan requests.
+    USE THESE OPTIONS WITH GREAT CARE
+    
+  Example:
+    nchan-redis-debug --url redis:// --filter-channels-min-subscribers=10
+EOB
+opt_parser.parse!
+
+rdsck = Rdsck.new $opt
+if not rdsck.connect
+  STDERR.puts "failed to connect to #{$opt[:url]}"
+  exit 1
+end
+
+case $opt[:command]
+when :filter_channels
+  puts "# scanning for channels #{$opt[:min_subscribers] && "with subscribers >= #{$opt[:min_subscribers]}"}"
+  chans = rdsck.filter_channels(min_subscribers: $opt[:min_subscribers])
+  puts "# found #{chans.count} channel#{chans.count != 1 && "s"}#{chans.count == 0 ? "." : ":"}"
+  puts chans.join("\n")
+else
+  puts "Nothing to do"
+end

data/exe/nchan-sub

@@ -33,7 +33,7 @@ opt_parser=OptionParser.new do |opts|
   opts.on("-p", "--parallel NUM (#{par})", "number of parallel clients"){|v| par = v.to_i}
   opts.on("-t", "--timeout SEC (#{opt[:timeout]})", "Long-poll timeout"){|v| opt[:timeout] = v}
   opts.on("-q", "--quit STRING (#{opt[:quit_message]})", "Quit message"){|v| opt[:quit_message] = v}
-  opts.on("-c", "--client STRING (#{opt[:client]})", "sub client (one of #{Subscriber::Client.unique_aliases.join ', '})") do |v|
+  opts.on("-c", "--client STRING (#{opt[:client]})", "sub client (one of #{NchanTools::Subscriber::Client.unique_aliases.join ', '})") do |v|
     opt[:client] = v.to_sym
   end
   opts.on("--content-type", "show received content-type"){|v| print_content_type = true}
@@ -68,7 +68,7 @@ if origin
   opt[:extra_headers]['Origin'] = origin
 end
 
-sub = Subscriber.new url, par, opt
+sub = NchanTools::Subscriber.new url, par, opt
 
 
 NOMSGF="\r"*30 + "Received message %i, len:%i"
@@ -111,7 +111,7 @@ sub.on_message do |msg|
 end
 
 sub.on_failure do |err_msg|
-  if Subscriber::IntervalPollClient === sub.client
+  if NchanTools::Subscriber::IntervalPollClient === sub.client
     unless err_msg.match(/\(code 304\)/)
       false
     end 

data/lib/nchan_tools/benchmark.rb

@@ -0,0 +1,241 @@
+require 'nchan_tools/pubsub'
+require 'securerandom'
+require 'timers'
+require 'json'
+
+module NchanTools
+class Benchmark
+  CSV_COLUMNS_ALL=%i[servers runtime channels channels_K channels_M subscribers message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed messages_received messages_unreceived messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber message_publishing_avg message_publishing_99th message_publishing_max message_publishing_stddev message_publishing_count message_delivery_avg message_delivery_99th message_delivery_max message_delivery_stddev message_delivery_count]
+  CSV_COLUMNS_DEFAULT=%i[servers runtime channels subscribers message_length messages_sent messages_send_confirmed messages_send_unconfirmed messages_send_failed messages_received messages_unreceived messages_send_rate messages_receive_rate messages_send_rate_per_channel messages_receive_rate_per_subscriber message_publishing_avg message_publishing_99th message_publishing_max message_publishing_stddev message_publishing_count message_delivery_avg message_delivery_99th message_delivery_max message_delivery_stddev message_delivery_count]
+  class BenchmarkError < StandardError
+  end
+  def initialize(urls, init_args=nil)
+    @urls = urls
+    @n = urls.count
+    @initializing = 0
+    @ready = 0
+    @running = 0
+    @finished = 0
+    @subs = []
+    @results = {}
+    @failed = {}
+    
+    @init_args = init_args
+    @histograms = {}
+    
+    subs = []
+  end
+  
+  def run
+    puts "connecting to #{@n} Nchan server#{@n > 1 ? "s" : ""}..."
+    @urls.each do |url|
+      sub = NchanTools::Subscriber.new(url, 1, client: :websocket, timeout: 900000, extra_headers: {"Accept" => "text/x-json-hdrhistogram"})
+      sub.on_failure do |err|
+        unless @results[sub]
+          unless @results[sub.url]
+            @failed[sub] = true
+            abort err, sub
+          end
+        end
+        false
+      end
+      sub.on_message do |msg|
+        msg = msg.to_s
+        case msg
+        when /^READY/
+          puts   "  #{sub.url} ok"
+          @ready +=1
+          if @ready == @n
+            puts "start benchmark..."
+            control :run
+          end
+        when /^RUNNING/
+          puts   "  #{sub.url} running"
+        when /^RESULTS\n/
+          msg = msg[8..-1]
+          parsed = JSON.parse msg
+          
+          #backwards-compatible histogram fields
+          parsed["histograms"]||={}
+          if parsed[:message_publishing_histogram] then
+            parsed[:histograms]["message publishing"]=parsed[:message_publishing_histogram]
+          end
+          if parsed[:message_delivery_histogram] then
+            parsed[:histograms]["message delivery"]=parsed[:message_delivery_histogram]
+          end
+          
+          @results[sub.url] = parsed
+          @results[sub.url]["raw"] = msg if @results[sub.url]
+          sub.client.send_close
+        when /^INITIALIZING/
+          #do nothing
+        else
+          raise BenchmarkError, "unexpected server response: #{msg}"
+        end
+      end
+      @subs << sub
+      sub.run
+      sub.wait :ready, 1
+      if @failed[sub]
+        puts "  #{sub.url} failed"
+      else
+        puts "  #{sub.url} ok"
+      end
+    end
+    return if @failed.count > 0
+    puts "initializing benchmark..."
+    control :init
+    self.wait
+    puts "finished."
+    puts ""
+  end
+  
+  def wait
+    @subs.each &:wait
+  end
+  
+  def control(msg)
+    if @init_args && (msg.to_sym ==:init || msg.to_sym ==:initialize)
+      msg = "#{msg.to_s} #{@init_args.map{|k,v| "#{k}=#{v}"}.join(" ")}"
+    end
+    @subs.each { |sub| sub.client.send_data msg.to_s }
+  end
+  
+  def abort(err, src_sub = nil)
+    puts "  #{err}"
+    @subs.each do |sub|
+      sub.terminate unless sub == src_sub
+    end
+  end
+  
+  def hdrhistogram_stats(name, histogram)
+    fmt = <<-END.gsub(/^ {6}/, '')
+      %s
+        min:                         %.3fms
+        avg:                         %.3fms
+        99%%ile:                      %.3fms
+        max:                         %.3fms
+        stddev:                      %.3fms
+        samples:                     %d
+    END
+    fmt % [ name,
+      histogram.min, histogram.mean, histogram.percentile(99.0), histogram.max, histogram.stddev, histogram.count
+    ]
+  end
+  
+  def results
+    @channels = 0
+    @runtime = []
+    @subscribers = 0
+    @message_length = []
+    @messages_sent = 0
+    @messages_send_confirmed = 0
+    @messages_send_unconfirmed = 0
+    @messages_send_failed = 0
+    @messages_received = 0
+    @messages_unreceived = 0
+    @histograms = {}
+    @results.each do |url, data|
+      @channels += data["channels"]
+      @runtime << data["run_time_sec"]
+      @subscribers += data["subscribers"]
+      @message_length << data["message_length"]
+      @messages_sent += data["messages"]["sent"]
+      @messages_send_confirmed += data["messages"]["send_confirmed"]
+      @messages_send_unconfirmed += data["messages"]["send_unconfirmed"]
+      @messages_send_failed += data["messages"]["send_failed"]
+      @messages_received += data["messages"]["received"]
+      @messages_unreceived += data["messages"]["unreceived"]
+      if data["histograms"]
+        data["histograms"].each do |name, str|
+          name = name.to_sym
+          hdrh = HDRHistogram.unserialize(str, unit: :ms, multiplier: 0.001)
+          if @histograms[name]
+            @histograms[name].merge! hdrh
+          else
+            @histograms[name] = hdrh
+          end
+        end
+      end
+    end
+    
+    @message_length = @message_length.inject(0, :+).to_f / @message_length.size
+    @runtime = @runtime.inject(0, :+).to_f / @runtime.size
+    
+    fmt = <<-END.gsub(/^ {6}/, '')
+      Nchan servers:                 %d
+      runtime:                       %d
+      channels:                      %d
+      subscribers:                   %d
+      subscribers per channel:       %.1f
+      messages:
+        length:                      %d
+        sent:                        %d
+        send_confirmed:              %d
+        send_unconfirmed:            %d
+        send_failed:                 %d
+        received:                    %d
+        unreceived:                  %d
+        send rate:                   %.3f/sec
+        receive rate:                %.3f/sec
+        send rate per channel:       %.3f/min
+        receive rate per subscriber: %.3f/min
+    END
+    out = fmt % [
+      @n, @runtime, @channels, @subscribers, @subscribers.to_f/@channels,
+      @message_length, @messages_sent, @messages_send_confirmed, @messages_send_unconfirmed, @messages_send_failed, 
+      @messages_received, @messages_unreceived,
+      @messages_sent.to_f/@runtime,
+      @messages_received.to_f/@runtime,
+      (@messages_sent.to_f* 60)/(@runtime * @channels),
+      (@messages_received.to_f * 60)/(@runtime * @subscribers)
+    ]
+    @histograms.each do |name, histogram|
+      out << hdrhistogram_stats("#{name} latency:", histogram)
+    end
+    
+    puts out
+  end
+  
+  def append_csv_file(file, columns=Benchmark::CSV_COLUMNS_DEFAULT)
+    require "csv"
+    write_headers = File.zero?(file) || !File.exists?(file)
+    headers = columns
+    vals = {
+      servers: @n,
+      runtime: @runtime,
+      channels: @channels,
+      channels_K: @channels/1000.0,
+      channels_M: @channels/1000000.0,
+      subscribers: @subscribers * @channels,
+      message_length: @message_length,
+      messages_sent: @messages_sent,
+      messages_send_confirmed: @messages_send_confirmed,
+      messages_send_unconfirmed: @messages_send_unconfirmed,
+      messages_send_failed: @messages_send_failed,
+      messages_received: @messages_received,
+      messages_unreceived: @messages_unreceived,
+      messages_send_rate: @messages_sent.to_f/@runtime,
+      messages_receive_rate: @messages_received.to_f/@runtime,
+      messages_send_rate_per_channel: (@messages_sent.to_f* 60)/(@runtime * @channels),
+      messages_receive_rate_per_subscriber: (@messages_received.to_f * 60)/(@runtime * @subscribers * @channels)
+    }
+    @histograms.each do |name, histogram|
+      vals["#{name}_avg".to_sym]=histogram.mean
+      vals["#{name}_95th".to_sym]=histogram.percentile(95.0)
+      vals["#{name}_99th".to_sym]=histogram.percentile(99.0)
+      vals["#{name}_max".to_sym]=histogram.max
+      vals["#{name}_stddev".to_sym]=histogram.stddev
+      vals["#{name}_count".to_sym]=histogram.count
+    end
+    
+    row = []
+    headers.each { |header| row << (vals[header.to_sym] || "-")}
+    
+    csv = CSV.open(file, "a", {headers: headers, write_headers: write_headers})
+    csv << row
+    csv.flush
+    csv.close
+  end
+end
+end

data/lib/nchan_tools/pubsub.rb

@@ -3,7 +3,7 @@ require 'typhoeus'
 require 'json'
 require 'oga'
 require 'yaml'
-require 'pry'
+
 require 'celluloid/current'
 require 'date'
 Typhoeus::Config.memoize = false
@@ -46,15 +46,12 @@ module URI
     u
   end
 end
-
-$seq = 0
+module NchanTools
 class Message
   attr_accessor :content_type, :message, :times_seen, :etag, :last_modified, :eventsource_event
   def initialize(msg, last_modified=nil, etag=nil)
     @times_seen=1
     @message, @last_modified, @etag = msg, last_modified, etag
-    $seq+=1
-    @seq = $seq
     @idhist = []
   end
   def serverside_id
@@ -490,6 +487,14 @@ class Subscriber
         @ws.binary data
       end
       
+      def send_ping(msg=nil)
+        @ws.ping(msg)
+      end
+      
+      def send_close(reason=nil, code=1000)
+        @ws.close(reason, code)
+      end
+      
       def write(data)
         @sock.write data
       end
@@ -582,11 +587,11 @@ class Subscriber
         end
         
         bundle.ws.on :ping do |ev|
-          @on_ping.call if @on_ping
+          @subscriber.on(:ping).call ev, bundle
         end
         
         bundle.ws.on :pong do |ev|
-          @on_pong.call if @on_pong
+          @subscriber.on(:pong).call ev, bundle
         end
         
         bundle.ws.on :error do |ev|
@@ -648,13 +653,6 @@ class Subscriber
       end
     end
     
-    def on_ping
-      @on_ping = Proc.new if block_given?
-    end
-    def on_pong
-      @on_pong = Proc.new if block_given?
-    end
-    
     def listen(bundle)
       while @ws[bundle]
         begin
@@ -684,10 +682,10 @@ class Subscriber
     private :ws_client
     
     def send_ping(data=nil)
-      ws_client.ping data
+      ws_client.send_ping data
     end
-    def send_close(code=1000, reason=nil)
-      ws_client.send_close code, reason
+    def send_close(reason=nil, code=1000)
+      ws_client.send_close reason, code
     end
     def send_data(data)
       ws_client.send_data data
@@ -697,17 +695,18 @@ class Subscriber
     end
     
     def close(bundle)
-      if bundle
+      if bundle then
         @ws.delete bundle
         bundle.sock.close unless bundle.sock.closed?
       end
       @connected -= 1
-      if @connected <= 0
-        binding.pry unless @ws.count == 0
+      if @connected <= 0 then
+        until @ws.count == 0 do
+          sleep 0.1
+        end
         @cooked.signal true
       end
     end
-    
   end
   
   class LongPollClient < Client
@@ -1521,6 +1520,8 @@ class Subscriber
 
   attr_accessor :url, :client, :messages, :max_round_trips, :quit_message, :errors, :concurrency, :waiting, :finished, :client_class, :log
   def initialize(url, concurrency=1, opt={})
+    @empty_block = Proc.new {}
+    @on={}
     @care_about_message_ids=opt[:use_message_id].nil? ? true : opt[:use_message_id]
     @url=url
     @quit_message = opt[:quit_message]
@@ -1615,6 +1616,14 @@ class Subscriber
     @client.poke until_what, timeout
   end
 
+  def on(evt_name = nil, &block)
+    if block_given?
+      @on[evt_name.to_sym] = block
+    else
+      @on[evt_name.to_sym] or @empty_block
+    end
+  end
+  
   def on_message(msg=nil, bundle=nil, &block)
     #puts "received message #{msg && msg.to_s[0..15]}"
     if block_given?
@@ -1660,6 +1669,7 @@ class Publisher
     @accept = opt[:accept]
     @verbose = opt[:verbose]
     @on_response = opt[:on_response]
+    @http2 = opt[:http2]
     
     @ws_wait_until_response = true
     
@@ -1800,6 +1810,7 @@ class Publisher
     headers = {:'Content-Type' => content_type, :'Accept' => accept}
     headers[:'X-Eventsource-Event'] = eventsource_event if eventsource_event
     headers.merge! @extra_headers if @extra_headers
+    
     post = Typhoeus::Request.new(
       @url,
       headers: headers,
@@ -1807,7 +1818,8 @@ class Publisher
       body: body,
       timeout: @timeout || PUBLISH_TIMEOUT,
       connecttimeout: @timeout || PUBLISH_TIMEOUT,
-      verbose: @verbose
+      verbose: @verbose,
+      http_version: @http2 ? :httpv2_0 : :none
     )
     if body && @messages
       msg=Message.new body
@@ -1895,3 +1907,4 @@ class Publisher
 
   
 end
+end

data/lib/nchan_tools/rdsck.rb

@@ -0,0 +1,119 @@
+class Rdsck
+  attr_accessor :url, :verbose, :namespace
+  attr_accessor :redis, :masters
+  
+  def dbg(*args)
+    if $opt[:verbose]
+      print("# ")
+      puts(*args)
+    end
+  end
+  
+  def initialize(opt)
+    @url=opt[:url]
+    @verbose=opt[:verbose]
+    @namespace=opt[:namespace]
+  end
+  
+  def cluster?
+    @cluster_mode
+  end
+  
+  def connect
+    begin
+      @redis=Redis.new url: $opt[:url]
+      mode = redis.info["redis_mode"]
+    rescue StandardError => e
+      STDERR.puts e.message
+      return false
+    end
+
+    if mode == "cluster"
+      @redis.close
+      begin
+        @redis=Redis.new cluster: [$opt[:url]]
+        @redis.ping
+      rescue StandardError => e
+        STDERR.puts e.message
+        return false
+      end
+      
+      @cluster_mode = true
+      @masters = []
+      
+      redis.connection.each do |c|
+        node = Redis.new url: c[:id]
+        @masters << node
+      end
+    else
+      @masters = [@redis]
+    end
+    
+    dbg "Connected to Redis #{mode == "cluster" ? "cluster" : "server"}"
+    (Array === @redis.connection ? @redis.connection : [@redis.connection]) .each do |v|
+      dbg "  #{v[:id]}"
+    end
+    self
+  end
+  
+  def key(subkey=nil)
+    k = "{channel:#{$opt[:namespace]}/#{$opt[:channel_id]}}"
+    return subkey ? "#{k}:#{subkey}" : k
+  end
+  
+  def info
+    channel_hash=@redis.hgetall key
+    hash_ttl=@redis.ttl key
+    channel_subs=@redis.hgetall key("subscribers")
+    #...
+  end
+  
+  def filter_channels(filters={})
+    script = <<~EOF
+      local prev_cursor = ARGV[1]
+      local pattern = ARGV[2]
+      local scan_batch_size = ARGV[3]
+      
+      local min_subscribers = ARGV[4] and #ARGV[4] > 0 and tonumber(ARGV[4])
+      
+      local cursor, iteration 
+      if pattern and #pattern > 0 then
+        cursor, iteration = unpack(redis.call("SCAN", prev_cursor, "MATCH", pattern, "COUNT", scan_batch_size))
+      else
+        cursor, iteration = unpack(redis.call("SCAN", prev_cursor, "COUNT", scan_batch_size))
+      end
+      
+      local matched = {}
+      for _, chankey in pairs(iteration) do
+        local match = true
+        if min_subscribers then
+          match = match and (tonumber(redis.call('HGET', chankey, 'fake_subscribers') or 0) >= min_subscribers)
+        end
+        if match then
+          table.insert(matched, chankey)
+        end
+      end
+      
+      return {cursor, matched}
+    EOF
+    
+    results = []
+    batch_size=500
+    masters.each do |m|
+      hash = m.script "load", script
+      cursor, pattern = "0", "{channel:*}"
+      loop do
+        cursor, batch_results = m.evalsha hash, keys: [], argv: [cursor, pattern, batch_size, filters[:min_subscribers]]
+        results += batch_results
+        pattern = ""
+        break if cursor.to_i == 0
+      end
+    end
+    results
+    
+    results.map! do |key|
+      m = key.match(/^\{channel\:(.*)\}$/)
+      m[1] || key
+    end
+  end
+end

data/lib/nchan_tools/version.rb

@@ -1,3 +1,3 @@
 module NchanTools
-  VERSION = "0.1.4"
+  VERSION = "0.1.9"
 end

data/nchan_tools.gemspec

@@ -29,15 +29,16 @@ Gem::Specification.new do |spec|
   spec.add_dependency "celluloid"
   spec.add_dependency "celluloid-io"
   spec.add_dependency "HDRHistogram"
+  spec.add_dependency "redis"
   
   spec.add_dependency "websocket-driver"
   spec.add_dependency 'websocket-extensions'
   spec.add_dependency "permessage_deflate"
   spec.add_dependency 'http_parser.rb'
-if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.2.2')
-  spec.add_dependency 'http-2'
-end  
+  if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.2.2')
+    spec.add_dependency 'http-2'
+  end  
   spec.add_development_dependency "pry"
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nchan_tools
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.9
 platform: ruby
 authors:
 - Leo Ponomarev
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-10-06 00:00:00.000000000 Z
+date: 2021-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: websocket-driver
   requirement: !ruby/object:Gem::Requirement
@@ -182,36 +196,37 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 description: publishing, subscribing, testing, and benchmarking utilities for Nchan.
 email:
 - leo@nchan.io
 executables:
 - nchan-benchmark
 - nchan-pub
+- nchan-redis-debug
 - nchan-sub
 extensions: []
 extra_rdoc_files: []
@@ -226,16 +241,19 @@ files:
 - bin/setup
 - exe/nchan-benchmark
 - exe/nchan-pub
+- exe/nchan-redis-debug
 - exe/nchan-sub
 - lib/nchan_tools.rb
+- lib/nchan_tools/benchmark.rb
 - lib/nchan_tools/pubsub.rb
+- lib/nchan_tools/rdsck.rb
 - lib/nchan_tools/version.rb
 - nchan_tools.gemspec
 homepage: https://nchan.io
 licenses:
 - WTFPL
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -250,9 +268,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Development and testing utilities for Nchan
 test_files: []