message_bus 3.3.7 → 4.1.0

data/Rakefile

@@ -5,18 +5,16 @@ require 'bundler'
 require 'bundler/gem_tasks'
 require 'bundler/setup'
 require 'rubocop/rake_task'
+require 'yard'
 
-RuboCop::RakeTask.new
+Bundler.require(:default, :test)
 
-require 'yard'
+RuboCop::RakeTask.new
 YARD::Rake::YardocTask.new
 
-desc "Generate documentation for Yard, and fail if there are any warnings"
-task :test_doc do
-  sh "yard --fail-on-warning #{'--no-progress' if ENV['CI']}"
-end
-
-Bundler.require(:default, :test)
+BACKENDS = Dir["lib/message_bus/backends/*.rb"].map { |file| file.match(%r{backends/(?<backend>.*).rb})[:backend] } - ["base"]
+SPEC_FILES = Dir['spec/**/*_spec.rb']
+INTEGRATION_FILES = Dir['spec/integration/**/*_spec.rb']
 
 module CustomBuild
   def build_gem
@@ -31,6 +29,11 @@ module Bundler
   end
 end
 
+desc "Generate documentation for Yard, and fail if there are any warnings"
+task :test_doc do
+  sh "yard --fail-on-warning #{'--no-progress' if ENV['CI']}"
+end
+
 namespace :jasmine do
   desc "Run Jasmine tests in headless mode"
   task 'ci' do
@@ -40,18 +43,16 @@ namespace :jasmine do
   end
 end
 
-backends = Dir["lib/message_bus/backends/*.rb"].map { |file| file.match(%r{backends/(?<backend>.*).rb})[:backend] } - ["base"]
-
 namespace :spec do
-  spec_files = Dir['spec/**/*_spec.rb']
-  integration_files = Dir['spec/integration/**/*_spec.rb']
-
-  backends.each do |backend|
+  BACKENDS.each do |backend|
     desc "Run tests on the #{backend} backend"
     task backend do
       begin
         ENV['MESSAGE_BUS_BACKEND'] = backend
-        sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{(spec_files - integration_files).to_a.join(' ')}"
+        Rake::TestTask.new(backend) do |t|
+          t.test_files = SPEC_FILES - INTEGRATION_FILES
+        end
+        Rake::Task[backend].invoke
       ensure
         ENV.delete('MESSAGE_BUS_BACKEND')
       end
@@ -74,7 +75,10 @@ namespace :spec do
       ENV['MESSAGE_BUS_BACKEND'] = 'memory'
       pid = spawn("bundle exec puma -p 9292 spec/fixtures/test/config.ru")
       sleep 1 while port_available?(9292)
-      sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{integration_files.to_a.join(' ')}"
+      Rake::TestTask.new(:integration) do |t|
+        t.test_files = INTEGRATION_FILES
+      end
+      Rake::Task[:integration].invoke
     ensure
       ENV.delete('MESSAGE_BUS_BACKEND')
       Process.kill('TERM', pid) if pid
@@ -83,12 +87,12 @@ namespace :spec do
 end
 
 desc "Run tests on all backends, plus client JS tests"
-task spec: backends.map { |backend| "spec:#{backend}" } + ["jasmine:ci", "spec:integration"]
+task spec: BACKENDS.map { |backend| "spec:#{backend}" } + ["jasmine:ci", "spec:integration"]
 
 desc "Run performance benchmarks on all backends"
 task :performance do
   begin
-    ENV['MESSAGE_BUS_BACKENDS'] = backends.join(",")
+    ENV['MESSAGE_BUS_BACKENDS'] = BACKENDS.join(",")
     sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/performance/*.rb'].to_a.join(' ')}"
   ensure
     ENV.delete('MESSAGE_BUS_BACKENDS')
@@ -97,7 +101,3 @@ end
 
 desc "Run all tests, link checks and confirms documentation compiles without error"
 task default: [:spec, :rubocop, :test_doc]
-
-Rake::Task['release'].enhance do
-  sh "yarn publish"
-end

data/docker-compose.yml

@@ -36,7 +36,7 @@ services:
   example:
     build:
       context: .
-    command: bash -c "cd examples/diagnostics && bundle install && bundle exec rackup --server puma --host 0.0.0.0"
+    command: bash -c "cd examples/chat && bundle install && bundle exec rackup --server puma --host 0.0.0.0"
     environment:
       BUNDLE_TO: /usr/local/bundle
       REDISURL: redis://redis:6379

data/lib/message_bus/backends/base.rb

@@ -84,6 +84,11 @@ module MessageBus
         raise ConcreteClassMustImplementError
       end
 
+      # Closes all open connections to the storage.
+      def destroy
+        raise ConcreteClassMustImplementError
+      end
+
       # Deletes all backlogs and their data. Does not delete non-backlog data that message_bus may persist, depending on the concrete backend implementation. Use with extreme caution.
       # @abstract
       def expire_all_backlogs!
@@ -113,6 +118,15 @@ module MessageBus
         raise ConcreteClassMustImplementError
       end
 
+      # Get the ID of the last message published on multiple channels
+      #
+      # @param [Array<String>] channels - array of channels to fetch
+      #
+      # @return [Array<Integer>] the channel-specific IDs of the last message published to each requested channel
+      def last_ids(*channels)
+        raise ConcreteClassMustImplementError
+      end
+
       # Get messages from a channel backlog
       #
       # @param [String] channel the name of the channel in question

data/lib/message_bus/backends/memory.rb

@@ -212,6 +212,12 @@ module MessageBus
         client.reset!
       end
 
+      # No-op; this backend doesn't maintain any storage connections.
+      # (see Base#destroy)
+      def destroy
+        nil
+      end
+
       # (see Base#expire_all_backlogs!)
       def expire_all_backlogs!
         client.expire_all_backlogs!
@@ -238,6 +244,13 @@ module MessageBus
         client.max_id(channel)
       end
 
+      # (see Base#last_ids)
+      def last_ids(*channels)
+        channels.map do |c|
+          last_id(c)
+        end
+      end
+
       # (see Base#backlog)
       def backlog(channel, last_id = 0)
         items = client.backlog channel, last_id.to_i

data/lib/message_bus/backends/postgres.rb

@@ -45,6 +45,7 @@ module MessageBus
           @available = []
           @allocated = {}
           @subscribe_connection = nil
+          @subscribed = false
           @mutex = Mutex.new
           @pid = Process.pid
         end
@@ -86,10 +87,12 @@ module MessageBus
           hold { |conn| exec_prepared(conn, 'get_message', [channel, id]) { |r| r.getvalue(0, 0) } }
         end
 
-        def reconnect
+        def after_fork
           sync do
-            @listening_on.clear
+            @pid = Process.pid
+            INHERITED_CONNECTIONS.concat(@available)
             @available.clear
+            @listening_on.clear
           end
         end
 
@@ -101,6 +104,13 @@ module MessageBus
           end
         end
 
+        def destroy
+          sync do
+            @available.each(&:close)
+            @available.clear
+          end
+        end
+
         # use with extreme care, will nuke all of the data
         def expire_all_backlogs!
           reset!
@@ -122,6 +132,21 @@ module MessageBus
           end
         end
 
+        def max_ids(*channels)
+          block = proc do |pg_result|
+            ids = Array.new(channels.size, 0)
+            pg_result.ntuples.times do |i|
+              channel = pg_result.getvalue(i, 0)
+              max_id = pg_result.getvalue(i, 1)
+              channel_index = channels.index(channel)
+              ids[channel_index] = max_id.to_i
+            end
+            ids
+          end
+
+          hold { |conn| exec_prepared(conn, 'max_channel_ids', [PG::TextEncoder::Array.new.encode(channels)], &block) }
+        end
+
         def publish(channel, data)
           hold { |conn| exec_prepared(conn, 'publish', [channel, data]) }
         end
@@ -165,20 +190,23 @@ module MessageBus
         end
 
         def create_table(conn)
-          conn.exec 'CREATE TABLE message_bus (id bigserial PRIMARY KEY, channel text NOT NULL, value text NOT NULL CHECK (octet_length(value) >= 2), added_at timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL)'
-          conn.exec 'CREATE INDEX table_channel_id_index ON message_bus (channel, id)'
-          conn.exec 'CREATE INDEX table_added_at_index ON message_bus (added_at)'
+          sync do
+            begin
+              conn.exec("SELECT 'message_bus'::regclass")
+            rescue PG::UndefinedTable
+              conn.exec 'CREATE TABLE message_bus (id bigserial PRIMARY KEY, channel text NOT NULL, value text NOT NULL CHECK (octet_length(value) >= 2), added_at timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL)'
+              conn.exec 'CREATE INDEX table_channel_id_index ON message_bus (channel, id)'
+              conn.exec 'CREATE INDEX table_added_at_index ON message_bus (added_at)'
+            end
+          end
+
           nil
         end
 
         def hold
           current_pid = Process.pid
           if current_pid != @pid
-            @pid = current_pid
-            sync do
-              INHERITED_CONNECTIONS.concat(@available)
-              @available.clear
-            end
+            after_fork
           end
 
           if conn = sync { @allocated[Thread.current] }
@@ -208,11 +236,7 @@ module MessageBus
         def new_pg_connection
           conn = raw_pg_connection
 
-          begin
-            conn.exec("SELECT 'message_bus'::regclass")
-          rescue PG::UndefinedTable
-            create_table(conn)
-          end
+          create_table(conn)
 
           conn.exec 'PREPARE insert_message AS INSERT INTO message_bus (channel, value) VALUES ($1, $2) RETURNING id'
           conn.exec 'PREPARE clear_global_backlog AS DELETE FROM message_bus WHERE (id <= $1)'
@@ -222,6 +246,7 @@ module MessageBus
           conn.exec "PREPARE expire AS DELETE FROM message_bus WHERE added_at < CURRENT_TIMESTAMP - ($1::text || ' seconds')::interval"
           conn.exec 'PREPARE get_message AS SELECT value FROM message_bus WHERE ((channel = $1) AND (id = $2))'
           conn.exec 'PREPARE max_channel_id AS SELECT max(id) FROM message_bus WHERE (channel = $1)'
+          conn.exec 'PREPARE max_channel_ids AS SELECT channel, max(id) FROM message_bus WHERE (channel = ANY($1)) GROUP BY channel'
           conn.exec 'PREPARE max_id AS SELECT max(id) FROM message_bus'
           conn.exec 'PREPARE publish AS SELECT pg_notify($1, $2)'
 
@@ -253,12 +278,14 @@ module MessageBus
         # after 7 days inactive backlogs will be removed
         @max_backlog_age = 604800
         @clear_every = config[:clear_every] || 1
+        @mutex = Mutex.new
+        @client = nil
       end
 
       # Reconnects to Postgres; used after a process fork, typically triggered by a forking webserver
       # @see Base#after_fork
       def after_fork
-        client.reconnect
+        client.after_fork
       end
 
       # (see Base#reset!)
@@ -266,6 +293,11 @@ module MessageBus
         client.reset!
       end
 
+      # (see Base#destroy)
+      def destroy
+        client.destroy
+      end
+
       # (see Base#expire_all_backlogs!)
       def expire_all_backlogs!
         client.expire_all_backlogs!
@@ -297,7 +329,12 @@ module MessageBus
         client.max_id(channel)
       end
 
-      # (see Base#last_id)
+      # (see Base#last_ids)
+      def last_ids(*channels)
+        client.max_ids(*channels)
+      end
+
+      # (see Base#backlog)
       def backlog(channel, last_id = 0)
         items = client.backlog channel, last_id.to_i
 
@@ -401,11 +438,7 @@ module MessageBus
       private
 
       def client
-        @client ||= new_connection
-      end
-
-      def new_connection
-        Client.new(@config)
+        @client || @mutex.synchronize { @client ||= Client.new(@config) }
       end
 
       def postgresql_channel_name

data/lib/message_bus/backends/redis.rb

@@ -58,6 +58,8 @@ module MessageBus
         @in_memory_backlog = []
         @lock = Mutex.new
         @flush_backlog_thread = nil
+        @pub_redis = nil
+        @subscribed = false
         # after 7 days inactive backlogs will be removed
         @max_backlog_age = 604800
       end
@@ -65,7 +67,7 @@ module MessageBus
       # Reconnects to Redis; used after a process fork, typically triggered by a forking webserver
       # @see Base#after_fork
       def after_fork
-        pub_redis.disconnect!
+        @pub_redis&.disconnect!
       end
 
       # (see Base#reset!)
@@ -75,6 +77,11 @@ module MessageBus
         end
       end
 
+      # (see Base#destroy)
+      def destroy
+        @pub_redis&.disconnect!
+      end
+
       # Deletes all backlogs and their data. Does not delete ID pointers, so new publications will get IDs that continue from the last publication before the expiry. Use with extreme caution.
       # @see Base#expire_all_backlogs!
       def expire_all_backlogs!
@@ -187,6 +194,13 @@ LUA
         pub_redis.get(backlog_id_key).to_i
       end
 
+      # (see Base#last_ids)
+      def last_ids(*channels)
+        return [] if channels.size == 0
+        backlog_id_keys = channels.map { |c| backlog_id_key(c) }
+        pub_redis.mget(*backlog_id_keys).map(&:to_i)
+      end
+
       # (see Base#backlog)
       def backlog(channel, last_id = 0)
         redis = pub_redis
@@ -254,6 +268,7 @@ LUA
           new_redis.publish(redis_channel_name, UNSUB_MESSAGE)
         ensure
           new_redis&.disconnect!
+          @subscribed = false
         end
       end
 
@@ -296,6 +311,7 @@ LUA
 
             on.message do |_c, m|
               if m == UNSUB_MESSAGE
+                @subscribed = false
                 global_redis.unsubscribe
                 return
               end

data/lib/message_bus/client.rb

@@ -46,6 +46,9 @@ class MessageBus::Client
     @bus = opts[:message_bus] || MessageBus
     @subscriptions = {}
     @chunks_sent = 0
+    @async_response = nil
+    @io = nil
+    @wrote_headers = false
   end
 
   # @yield executed with a lock on the Client instance
@@ -175,31 +178,32 @@ class MessageBus::Client
     r = []
     new_message_ids = nil
 
-    @subscriptions.each do |k, v|
-      id = v.to_i
+    last_bus_ids = @bus.last_ids(*@subscriptions.keys, site_id: site_id)
 
-      if id < -1
-        last_id = @bus.last_id(k, site_id)
-        id = last_id + id + 1
-        id = 0 if id < 0
+    @subscriptions.each do |k, v|
+      last_client_id = v.to_i
+      last_bus_id = last_bus_ids[k]
+
+      if last_client_id < -1 # Client requesting backlog relative to bus position
+        last_client_id = last_bus_id + last_client_id + 1
+        last_client_id = 0 if last_client_id < 0
+      elsif last_client_id == -1 # Client not requesting backlog
+        next
+      elsif last_client_id == last_bus_id # Client already up-to-date
+        next
+      elsif last_client_id > last_bus_id # Client ahead of the bus
+        @subscriptions[k] = -1
+        next
       end
 
-      next if id < 0
+      messages = @bus.backlog(k, last_client_id, site_id)
 
-      messages = @bus.backlog(k, id, site_id)
-
-      if messages.length == 0
-        if id > @bus.last_id(k, site_id)
-          @subscriptions[k] = -1
-        end
-      else
-        messages.each do |msg|
-          if allowed?(msg)
-            r << msg
-          else
-            new_message_ids ||= {}
-            new_message_ids[k] = msg.message_id
-          end
+      messages.each do |msg|
+        if allowed?(msg)
+          r << msg
+        else
+          new_message_ids ||= {}
+          new_message_ids[k] = msg.message_id
         end
       end
     end
@@ -209,7 +213,7 @@ class MessageBus::Client
     @subscriptions.each do |k, v|
       if v.to_i == -1 || (new_message_ids && new_message_ids[k])
         status_message ||= {}
-        @subscriptions[k] = status_message[k] = @bus.last_id(k, site_id)
+        @subscriptions[k] = status_message[k] = last_bus_ids[k]
       end
     end
 

data/lib/message_bus/distributed_cache.rb

@@ -22,6 +22,7 @@ module MessageBus
         @lock = Mutex.new
         @message_bus = message_bus || MessageBus
         @publish_queue_in_memory = publish_queue_in_memory
+        @app_version = nil
       end
 
       def subscribers

data/lib/message_bus/rack/middleware.rb

@@ -41,7 +41,6 @@ class MessageBus::Rack::Middleware
     @started_listener = false
     @base_route = "#{@bus.base_route}message-bus/"
     @base_route_length = @base_route.length
-    @diagnostics_route = "#{@base_route}_diagnostics"
     @broadcast_route = "#{@base_route}broadcast"
     start_listener unless @bus.off?
   end
@@ -79,11 +78,6 @@ class MessageBus::Rack::Middleware
       return [200, { "Content-Type" => "text/html" }, ["sent"]]
     end
 
-    if env['PATH_INFO'].start_with? @diagnostics_route
-      diags = MessageBus::Rack::Diagnostics.new(@app, message_bus: @bus)
-      return diags.call(env)
-    end
-
     client_id = env['PATH_INFO'][@base_route_length..-1].split("/")[0]
     return [404, {}, ["not found"]] unless client_id
 

data/lib/message_bus/rack/thin_ext.rb

@@ -9,6 +9,7 @@ module Thin
 
       def initialize
         @queue = []
+        @body_callback = nil
       end
 
       def call(body)

data/lib/message_bus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MessageBus
-  VERSION = "3.3.7"
+  VERSION = "4.1.0"
 end