message_bus 3.3.7 → 4.1.0

data/lib/message_bus.rb

@@ -7,9 +7,7 @@ require_relative "message_bus/version"
 require_relative "message_bus/message"
 require_relative "message_bus/client"
 require_relative "message_bus/connection_manager"
-require_relative "message_bus/diagnostics"
 require_relative "message_bus/rack/middleware"
-require_relative "message_bus/rack/diagnostics"
 require_relative "message_bus/timer_thread"
 require_relative "message_bus/codec/base"
 require_relative "message_bus/backends"
@@ -41,21 +39,10 @@ module MessageBus::Implementation
   def initialize
     @config = {}
     @mutex = Synchronizer.new
-  end
-
-  # @param [Boolean] val whether or not to cache static assets for the diagnostics pages
-  # @return [void]
-  def cache_assets=(val)
-    configure(cache_assets: val)
-  end
-
-  # @return [Boolean] whether or not to cache static assets for the diagnostics pages
-  def cache_assets
-    if defined? @config[:cache_assets]
-      @config[:cache_assets]
-    else
-      true
-    end
+    @off = false
+    @destroyed = false
+    @timer_thread = nil
+    @subscriber_thread = nil
   end
 
   # @param [Logger] logger a logger object to be used by the bus
@@ -293,15 +280,20 @@ module MessageBus::Implementation
     @config[:transport_codec] ||= MessageBus::Codec::Json.new
   end
 
-  # @param [MessageBus::Backend::Base] pub_sub a configured backend
+  # @param [MessageBus::Backend::Base] backend_instance A configured backend
   # @return [void]
+  def backend_instance=(backend_instance)
+    configure(backend_instance: backend_instance)
+  end
+
   def reliable_pub_sub=(pub_sub)
-    configure(reliable_pub_sub: pub_sub)
+    logger.warn "MessageBus.reliable_pub_sub= is deprecated, use MessageBus.backend_instance= instead."
+    self.backend_instance = pub_sub
   end
 
   # @return [MessageBus::Backend::Base] the configured backend. If not
   #   explicitly set, will be loaded based on the configuration provided.
-  def reliable_pub_sub
+  def backend_instance
     @mutex.synchronize do
       return nil if @destroyed
 
@@ -309,7 +301,7 @@ module MessageBus::Implementation
       # passed to backend.
       logger
 
-      @config[:reliable_pub_sub] ||= begin
+      @config[:backend_instance] ||= begin
         @config[:backend_options] ||= {}
         require "message_bus/backends/#{backend}"
         MessageBus::BACKENDS[backend].new @config
@@ -317,17 +309,16 @@ module MessageBus::Implementation
     end
   end
 
+  def reliable_pub_sub
+    logger.warn "MessageBus.reliable_pub_sub is deprecated, use MessageBus.backend_instance instead."
+    backend_instance
+  end
+
   # @return [Symbol] the name of the backend implementation configured
   def backend
     @config[:backend] || :redis
   end
 
-  # Enables diagnostics tracking
-  # @return [void]
-  def enable_diagnostics
-    MessageBus::Diagnostics.enable(self)
-  end
-
   # Publishes a message to a channel
   #
   # @param [String] channel the name of the channel to which the message should be published
@@ -394,7 +385,7 @@ module MessageBus::Implementation
     end
 
     encoded_channel_name = encode_channel_name(channel, site_id)
-    reliable_pub_sub.publish(encoded_channel_name, encoded_data, channel_opts)
+    backend_instance.publish(encoded_channel_name, encoded_data, channel_opts)
   end
 
   # Subscribe to messages. Each message will be delivered by yielding to the
@@ -410,9 +401,9 @@ module MessageBus::Implementation
   # @return [void]
   def blocking_subscribe(channel = nil, &blk)
     if channel
-      reliable_pub_sub.subscribe(encode_channel_name(channel), &blk)
+      backend_instance.subscribe(encode_channel_name(channel), &blk)
     else
-      reliable_pub_sub.global_subscribe(&blk)
+      backend_instance.global_subscribe(&blk)
     end
   end
 
@@ -491,9 +482,9 @@ module MessageBus::Implementation
   def backlog(channel = nil, last_id = nil, site_id = nil)
     old =
       if channel
-        reliable_pub_sub.backlog(encode_channel_name(channel, site_id), last_id)
+        backend_instance.backlog(encode_channel_name(channel, site_id), last_id)
       else
-        reliable_pub_sub.global_backlog(last_id)
+        backend_instance.global_backlog(last_id)
       end
 
     old.each do |m|
@@ -509,7 +500,19 @@ module MessageBus::Implementation
   #
   # @return [Integer] the channel-specific ID of the last message published to the given channel
   def last_id(channel, site_id = nil)
-    reliable_pub_sub.last_id(encode_channel_name(channel, site_id))
+    backend_instance.last_id(encode_channel_name(channel, site_id))
+  end
+
+  # Get the ID of the last message published on multiple channels
+  #
+  # @param [Array<String>] channels - array of channels to fetch
+  # @param [String] site_id - the ID of the site by which to filter
+  #
+  # @return [Hash] the channel-specific IDs of the last message published to each requested channel
+  def last_ids(*channels, site_id: nil)
+    encoded_channel_names = channels.map { |c| encode_channel_name(c, site_id) }
+    ids = backend_instance.last_ids(*encoded_channel_names)
+    channels.zip(ids).to_h
   end
 
   # Get the last message published on a channel
@@ -532,7 +535,8 @@ module MessageBus::Implementation
   def destroy
     return if @destroyed
 
-    reliable_pub_sub.global_unsubscribe
+    backend_instance.global_unsubscribe
+    backend_instance.destroy
 
     @mutex.synchronize do
       return if @destroyed
@@ -550,7 +554,7 @@ module MessageBus::Implementation
   #   scheduled tasks.
   # @return [void]
   def after_fork
-    reliable_pub_sub.after_fork
+    backend_instance.after_fork
     ensure_subscriber_thread
     # will ensure timer is running
     timer.queue {}
@@ -559,12 +563,12 @@ module MessageBus::Implementation
   # @return [Boolean] whether or not the server is actively listening for
   #   publications on the bus
   def listening?
-    @subscriber_thread && @subscriber_thread.alive?
+    @subscriber_thread&.alive?
   end
 
   # (see MessageBus::Backend::Base#reset!)
   def reset!
-    reliable_pub_sub.reset! if reliable_pub_sub
+    backend_instance.reset! if backend_instance
   end
 
   # @return [MessageBus::TimerThread] the timer thread used for triggering
@@ -692,12 +696,6 @@ module MessageBus::Implementation
     @subscriptions[site_id][channel] << blk
     ensure_subscriber_thread
 
-    attempts = 100
-    while attempts > 0 && !reliable_pub_sub.subscribed
-      sleep 0.001
-      attempts -= 1
-    end
-
     raise MessageBus::BusDestroyed if @destroyed
 
     blk
@@ -715,10 +713,17 @@ module MessageBus::Implementation
 
   def ensure_subscriber_thread
     @mutex.synchronize do
-      return if (@subscriber_thread && @subscriber_thread.alive?) || @destroyed
+      return if @destroyed
+      next if @subscriber_thread&.alive?
 
       @subscriber_thread = new_subscriber_thread
     end
+
+    attempts = 100
+    while attempts > 0 && !backend_instance.subscribed
+      sleep 0.001
+      attempts -= 1
+    end
   end
 
   MIN_KEEPALIVE = 20
@@ -739,34 +744,11 @@ module MessageBus::Implementation
       if !@destroyed && thread.alive? && keepalive_interval > MIN_KEEPALIVE
 
         publish("/__mb_keepalive__/", Process.pid, user_ids: [-1])
-        # going for x3 keepalives missed for a restart, need to ensure this only very rarely happens
-        # note: after_fork will sort out a bad @last_message date, but thread will be dead anyway
         if (Time.now - (@last_message || Time.now)) > keepalive_interval * 3
-          logger.warn "Global messages on #{Process.pid} timed out, restarting process"
-          # No other clean way to remove this thread, its listening on a socket
-          #   no data is arriving
-          #
-          # In production we see this kind of situation ... sometimes ... when there is
-          # a VRRP failover, or weird networking condition
-          pid = Process.pid
-
-          # do the best we can to terminate self cleanly
-          fork do
-            Process.kill('TERM', pid)
-            sleep 10
-            begin
-              Process.kill('KILL', pid)
-            rescue Errno::ESRCH
-              logger.warn "#{Process.pid} successfully terminated by `TERM` signal."
-            end
-          end
-
-          sleep 10
-          Process.kill('KILL', pid)
-
-        else
-          timer.queue(keepalive_interval, &blk) if keepalive_interval > MIN_KEEPALIVE
+          logger.warn "Global messages on #{Process.pid} timed out, message bus is no longer functioning correctly"
         end
+
+        timer.queue(keepalive_interval, &blk) if keepalive_interval > MIN_KEEPALIVE
       end
     end
 
@@ -778,7 +760,7 @@ module MessageBus::Implementation
   def global_subscribe_thread
     # pretend we just got a message
     @last_message = Time.now
-    reliable_pub_sub.global_subscribe do |msg|
+    backend_instance.global_subscribe do |msg|
       begin
         @last_message = Time.now
         decode_message!(msg)

data/message_bus.gemspec

@@ -9,9 +9,8 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{}
   gem.homepage      = "https://github.com/discourse/message_bus"
   gem.license       = "MIT"
-  gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.files         = `git ls-files`.split($\) +
+                      ["vendor/assets/javascripts/message-bus.js", "vendor/assets/javascripts/message-bus-ajax.js"]
   gem.name          = "message_bus"
   gem.require_paths = ["lib"]
   gem.version       = MessageBus::VERSION
@@ -19,9 +18,11 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency 'rack', '>= 1.1.3'
 
+  # Optional runtime dependencies
   gem.add_development_dependency 'redis'
   gem.add_development_dependency 'pg'
   gem.add_development_dependency 'concurrent-ruby' # for distributed-cache
+
   gem.add_development_dependency 'minitest'
   gem.add_development_dependency 'minitest-hooks'
   gem.add_development_dependency 'minitest-global_expectations'

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "message-bus-client",
-  "version": "3.3.5",
+  "version": "0.0.0-version-placeholder",
   "description": "A message bus client in Javascript",
   "main": "assets/message-bus.js",
   "keywords": [
@@ -12,10 +12,7 @@
   ],
   "jsnext:main": "assets/message-bus.js",
   "module": "assets/message-bus.js",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/discourse/message_bus.git"
-  },
+  "repository": "https://github.com/discourse/message_bus",
   "author": "Sam Saffron, Robin Ward",
   "license": "MIT",
   "bugs": {

data/spec/helpers.rb

@@ -25,7 +25,12 @@ def test_config_for_backend(backend)
   when :redis
     config[:url] = ENV['REDISURL']
   when :postgres
-    config[:backend_options] = { host: ENV['PGHOST'], user: ENV['PGUSER'] || ENV['USER'], password: ENV['PGPASSWORD'], dbname: ENV['PGDATABASE'] || 'message_bus_test' }
+    config[:backend_options] = {
+      host: ENV['PGHOST'],
+      user: ENV['PGUSER'] || ENV['USER'],
+      password: ENV['PGPASSWORD'],
+      dbname: ENV['PGDATABASE'] || 'message_bus_test'
+    }
   end
   config
 end

data/spec/lib/fake_async_middleware.rb

@@ -7,6 +7,7 @@ class FakeAsyncMiddleware
     @simulate_thin_async = false
     @simulate_hijack = false
     @in_async = false
+    @allow_chunked = false
   end
 
   def app

data/spec/lib/message_bus/backend_spec.rb

@@ -3,13 +3,14 @@
 require_relative '../../spec_helper'
 require 'message_bus'
 
-describe PUB_SUB_CLASS do
+describe BACKEND_CLASS do
   before do
-    @bus = PUB_SUB_CLASS.new(test_config_for_backend(CURRENT_BACKEND))
+    @bus = BACKEND_CLASS.new(test_config_for_backend(CURRENT_BACKEND))
   end
 
   after do
     @bus.reset!
+    @bus.destroy
   end
 
   describe "API parity" do
@@ -29,7 +30,7 @@ describe PUB_SUB_CLASS do
   end
 
   it "should initialize with max_backlog_size" do
-    PUB_SUB_CLASS.new({}, 2000).max_backlog_size.must_equal 2000
+    BACKEND_CLASS.new({}, 2000).max_backlog_size.must_equal 2000
   end
 
   it "should truncate channels correctly" do
@@ -90,6 +91,22 @@ describe PUB_SUB_CLASS do
     @bus.last_id("/foo").must_equal 1
   end
 
+  it "should allow us to get multiple last_ids" do
+    @bus.last_ids("/foo", "/bar", "/foobar").must_equal [0, 0, 0]
+
+    @bus.publish("/foo", "one")
+    @bus.publish("/foo", "two")
+    @bus.publish("/foobar", "three")
+
+    @bus.last_ids("/foo", "/bar", "/foobar").must_equal(
+      [
+        @bus.last_id("/foo"),
+        @bus.last_id("/bar"),
+        @bus.last_id("/foobar")
+      ]
+    )
+  end
+
   it "can set backlog age" do
     @bus.max_backlog_age = 1
 

data/spec/lib/message_bus/client_spec.rb

@@ -16,6 +16,7 @@ describe MessageBus::Client do
     end
 
     after do
+      @client.close
       @bus.reset!
       @bus.destroy
     end

data/spec/lib/message_bus/connection_manager_spec.rb

@@ -6,6 +6,10 @@ require 'message_bus'
 class FakeAsync
   attr_accessor :cleanup_timer
 
+  def initialize
+    @sent = nil
+  end
+
   def <<(val)
     sleep 0.01 # simulate IO
     @sent ||= +""

data/spec/lib/message_bus/multi_process_spec.rb

@@ -3,7 +3,7 @@
 require_relative '../../spec_helper'
 require 'message_bus'
 
-describe PUB_SUB_CLASS do
+describe BACKEND_CLASS do
   def self.error!
     @error = true
   end
@@ -13,18 +13,20 @@ describe PUB_SUB_CLASS do
   end
 
   def new_bus
-    PUB_SUB_CLASS.new(test_config_for_backend(CURRENT_BACKEND).merge(db: 10))
+    BACKEND_CLASS.new(test_config_for_backend(CURRENT_BACKEND).merge(db: 10))
   end
 
   def work_it
     bus = new_bus
-    $stdout.reopen("/dev/null", "w")
-    $stderr.reopen("/dev/null", "w")
-    # subscribe blocks, so we need a new bus to transmit
-    new_bus.subscribe("/echo", 0) do |msg|
-      bus.publish("/response", "#{msg.data}-#{Process.pid.to_s}")
+    bus.subscribe("/echo", 0) do |msg|
+      if msg.data == "done"
+        bus.global_unsubscribe
+      else
+        bus.publish("/response", "#{msg.data}-#{Process.pid.to_s}")
+      end
     end
   ensure
+    bus.destroy
     exit!(0)
   end
 
@@ -44,12 +46,14 @@ describe PUB_SUB_CLASS do
       test_never :memory
       skip("previous error") if self.class.error?
       GC.start
-      new_bus.reset!
+      bus = new_bus
+      bus.reset!
+
       begin
         pids = (1..10).map { spawn_child }
         expected_responses = pids.map { |x| (0...10).map { |i| "0#{i}-#{x}" } }.flatten
         unexpected_responses = []
-        bus = new_bus
+
         t = Thread.new do
           bus.subscribe("/response", 0) do |msg|
             if expected_responses.include?(msg.data)
@@ -59,10 +63,14 @@ describe PUB_SUB_CLASS do
             end
           end
         end
+
         10.times { |i| bus.publish("/echo", "0#{i}") }
-        wait_for 4000 do
+
+        wait_for(2000) do
           expected_responses.empty?
         end
+
+        bus.publish("/echo", "done")
         bus.global_unsubscribe
         t.join
 
@@ -81,7 +89,10 @@ describe PUB_SUB_CLASS do
             Process.wait(pid)
           end
         end
+
         bus.global_unsubscribe
+        bus.reset!
+        bus.destroy
       end
     end
   end

data/spec/lib/message_bus/rack/middleware_spec.rb

@@ -83,7 +83,7 @@ describe MessageBus::Rack::Middleware do
         { "FOO" => "BAR" }
       end
 
-      Thread.new do
+      t = Thread.new do
         wait_for(2000) { middleware.in_async? }
         bus.publish "/foo", "םוֹלשָׁ"
       end
@@ -96,6 +96,7 @@ describe MessageBus::Rack::Middleware do
       parsed[0]["data"].must_equal "םוֹלשָׁ"
 
       last_response.headers["FOO"].must_equal "BAR"
+      t.join
     end
 
     it "should timeout within its alloted slot" do
@@ -141,54 +142,6 @@ describe MessageBus::Rack::Middleware do
     end
   end
 
-  describe "diagnostics" do
-    it "should return a 403 if an unauthorized user attempts to get at the _diagnostics path" do
-      get "/message-bus/_diagnostics"
-      last_response.status.must_equal 403
-    end
-
-    it "should get a 200 with html for an authorized user" do
-      def @bus.is_admin_lookup
-        proc { |_| true }
-      end
-
-      get "/message-bus/_diagnostics"
-      last_response.status.must_equal 200
-    end
-
-    describe "with an altered base_route" do
-      let(:base_route) { "/base/route/" }
-
-      it "should get a 200 with html for an authorized user" do
-        def @bus.is_admin_lookup
-          proc { |_| true }
-        end
-
-        get "/base/route/message-bus/_diagnostics"
-        last_response.status.must_equal 200
-      end
-    end
-
-    it "should get the script it asks for" do
-      def @bus.is_admin_lookup
-        proc { |_| true }
-      end
-
-      get "/message-bus/_diagnostics/assets/message-bus.js"
-      last_response.status.must_equal 200
-      last_response.content_type.must_equal "application/javascript;charset=UTF-8"
-    end
-
-    it "should return 404 for invalid assets path" do
-      def @bus.is_admin_lookup
-        proc { |_| true }
-      end
-
-      get "/message-bus/_diagnostics/assets/../Gemfile"
-      last_response.status.must_equal 404
-    end
-  end
-
   describe "polling" do
     before do
       @bus.long_polling_enabled = false

data/spec/lib/message_bus/timer_thread_spec.rb

@@ -52,10 +52,6 @@ describe MessageBus::TimerThread do
   it "should call the error callback if something goes wrong" do
     error = nil
 
-    @timer.queue do
-      boom
-    end
-
     @timer.on_error do |e|
       error = e
     end
@@ -64,7 +60,7 @@ describe MessageBus::TimerThread do
       boom
     end
 
-    wait_for(10) do
+    wait_for(100) do
       error
     end
 

data/spec/lib/message_bus_spec.rb

@@ -23,7 +23,8 @@ describe MessageBus do
     @bus.off?.must_equal true
   end
 
-  it "can call destroy twice" do
+  it "can call destroy multiple times" do
+    @bus.destroy
     @bus.destroy
     @bus.destroy
   end
@@ -36,6 +37,14 @@ describe MessageBus do
     @bus.after_fork
   end
 
+  it "destroying immediately after `after_fork` does not lock" do
+    10.times do
+      @bus.on
+      @bus.after_fork
+      @bus.destroy
+    end
+  end
+
   describe "#base_route=" do
     it "adds leading and trailing slashes" do
       @bus.base_route = "my/base/route"
@@ -106,7 +115,7 @@ describe MessageBus do
     @bus.publish("/chuck", norris: true)
     @bus.publish("/chuck", norris: true)
 
-    @bus.reliable_pub_sub.reset!
+    @bus.backend_instance.reset!
 
     @bus.publish("/chuck", yeager: true)
 
@@ -124,7 +133,7 @@ describe MessageBus do
     @bus.publish("/chuck", norris: true)
     @bus.publish("/chuck", norris: true)
 
-    @bus.reliable_pub_sub.expire_all_backlogs!
+    @bus.backend_instance.expire_all_backlogs!
 
     @bus.publish("/chuck", yeager: true)
 

data/spec/performance/backlog.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', '..', 'lib')
+require 'logger'
+require 'benchmark'
+require 'message_bus'
+
+require_relative "../helpers"
+
+backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym)
+channel = "/foo"
+iterations = 10_000
+results = []
+
+puts "Running backlog benchmark with #{iterations} iterations on backends: #{backends.inspect}"
+
+run_benchmark = lambda do |bm, backend|
+  bus = MessageBus::Instance.new
+  bus.configure(test_config_for_backend(backend))
+
+  bus.backend_instance.max_backlog_size = 100
+  bus.backend_instance.max_global_backlog_size = 1000
+
+  channel_names = 10.times.map { |i| "channel#{i}" }
+
+  100.times do |i|
+    channel_names.each do |ch|
+      bus.publish(ch, { message_number_is: i })
+    end
+  end
+
+  last_ids = channel_names.map { |ch| [ch, bus.last_id(ch)] }.to_h
+
+  1000.times do
+    # Warmup
+    client = MessageBus::Client.new(message_bus: bus)
+    channel_names.each { |ch| client.subscribe(ch, -1) }
+    client.backlog
+  end
+
+  bm.report("#{backend} - #backlog with no backlogs requested") do
+    iterations.times do
+      client = MessageBus::Client.new(message_bus: bus)
+      channel_names.each { |ch| client.subscribe(ch, -1) }
+      client.backlog
+    end
+  end
+
+  (0..5).each do |ch_i|
+    channels_with_messages = (ch_i) * 2
+
+    bm.report("#{backend} - #backlog when #{channels_with_messages}/10 channels have new messages") do
+      iterations.times do
+        client = MessageBus::Client.new(message_bus: bus)
+        channel_names.each_with_index do |ch, i|
+          client.subscribe(ch, last_ids[ch] + ((i < channels_with_messages) ? -1 : 0))
+        end
+        result = client.backlog
+        if result.length != channels_with_messages
+          raise "Result has #{result.length} messages. Expected #{channels_with_messages}"
+        end
+      end
+    end
+  end
+
+  bus.reset!
+  bus.destroy
+end
+
+puts
+
+Benchmark.benchmark("     duration\n", 60, "%10.2rs\n", "") do |bm|
+  backends.each do |backend|
+    run_benchmark.call(bm, backend)
+  end
+end
+puts
+results.each do |result|
+  puts result
+end