message_bus 3.3.8 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b45c0d65f5a82c9465f620059d2125ebca09ee5619a9094fcbc309a0e179a983
-  data.tar.gz: b691c40ae2360ae661aa83f83e9647cb0223a70a6ca536f285db21ce791a3ae3
+  metadata.gz: edad2fa285f65718a34893791166991c2cdf893b6e7064c5163882cd92f5fc95
+  data.tar.gz: 686c4fcc4c02add5adf75fa08586491af36173d67bfac917093b291d8d3343f2
 SHA512:
-  metadata.gz: 2bb5db140d0fc8bbee5df6e00f57560b1e399c7cac86d825f9841c4c9d8bc805ec8f9ab2ea8ba0ad066c53039a44b5044ecef7299a74f942b3a52cd426ecbef6
-  data.tar.gz: 66fdc2a3c483d1b775b3963b71bdc422edcdf16233cb47aa2ec99d3a6946e6e0f6f6ca917db12e5d528de6dfb39cc494923e2abe380ca0c75d64d92f6f167d7e
+  metadata.gz: fb23eadc6112f7d28128b46e1e00d7bbdd196cf574398a3746927210d9affd4f127b83e6a9f104a65d8909d03729af16221ef7ec159126cdc5208d6eacf3e2d5
+  data.tar.gz: bdbf6a4d6d3c129f58a8c0fb02d5275cda59becb59120a602423340597e7afcb84410b39f58b537a2962a0157d0a1ca10d2efbc28f6bfbea4e3fe160a7cf46f7

data/.eslintrc.js

@@ -10,12 +10,5 @@ module.exports = {
     sourceType: 'module',
   },
   rules: {},
-  ignorePatterns: [
-    '/vendor',
-    '/doc',
-    '/assets/babel.min.js',
-    '/assets/jquery-1.8.2.js',
-    '/assets/react-dom.js',
-    '/assets/react.js',
-  ],
+  ignorePatterns: ['/vendor', '/doc', '/assets/jquery-1.8.2.js'],
 };

data/.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.6, 2.7, 3.0]
+        ruby: [2.6, 2.7, '3.0', 3.1]
         redis: [5, 6]
 
     services:
@@ -60,6 +60,8 @@ jobs:
         run: npm install
 
       - name: Tests
+        env:
+          TESTOPTS: --verbose
         run: bundle exec rake
         timeout-minutes: 3
 

data/CHANGELOG

@@ -1,3 +1,42 @@
+22-02-2022
+
+- Version 4.2.0
+
+  - FEATURE: Add support for `clear_every` parameter in Redis backend
+
+    This allows the clearing of the backlog to take place once every N messages. In high-load scenarios, this can provide significant performance benefit.
+
+16-02-2022
+
+- Version 4.1.0
+
+  - PERF: Optimize Client#backlog for up-to-date clients
+
+    Also introduces a new MessageBus#last_ids(*channels) api for fetching the last_ids of
+    multiple channels simultaneously
+
+11-01-2022
+
+- Version 4.0.0
+
+  - DEV: Remove backend diagnostics
+  - DEV: Rename reliable_pub_sub to backend_instance
+  - FIX: `destroy` following `after_fork` could thread lock
+
+31-12-2021
+
+- Version 3.4.0
+
+  - FEATURE: Remove process auto-termination on missed keepalives
+
+20-12-2021
+
+- Version 3.3.8
+
+  - FIX: Restore dist js files to fix a regression in 3.3.7
+  - FIX: Stop various thread/pg connection leaks
+  - DEV: Fix ruby warnings
+
 15-12-2021
 
 - Version 3.3.7
@@ -63,14 +102,14 @@
 
 - Version 3.3.0
 
-   - FEATURE: `MessageBus.base_route=` to alter the route that message bus will listen on.
+  - FEATURE: `MessageBus.base_route=` to alter the route that message bus will listen on.
 
 07-05-2020
 
 - Version 3.2.0
 
-   - FIX: compatibility with Rails 6.0.3, note: apps without ActionDispatch::Flash may stop working after this upgrade
-   to correct this disable middleware injection with `config.skip_message_bus_middleware = true` and configure middleware by hand with `app.middleware.use(MessageBus::Rack::Middleware)`
+  - FIX: compatibility with Rails 6.0.3, note: apps without ActionDispatch::Flash may stop working after this upgrade
+    to correct this disable middleware injection with `config.skip_message_bus_middleware = true` and configure middleware by hand with `app.middleware.use(MessageBus::Rack::Middleware)`
 
 28-04-2020
 
@@ -105,7 +144,6 @@
 
   - FIX: In the redis backend make the `is_readonly?` method compatible with the redis gem both pre and post v4.0 when the `client` attribute was removed
 
-
 30-04-2019
 
 - Version 2.2.1
@@ -136,7 +174,7 @@
 - Version 2.2.0.pre
 
 - FIX: In redis backend we now expire the key used to track channel id this can cause a redis key leak
-    with large amounts of subscriptions that go away
+  with large amounts of subscriptions that go away
 - FEATURE: Much extra implementation documentation, and some improvements to usage documentation.
 - FEATURE: Improvements to development workflow:
   - Fully docker-based development and testing, with no other dependencies.
@@ -153,44 +191,47 @@
   - Supports setting backlog size on publication for memory/postgres
 - FEATURE: `MessageBus.off` now prevents the server subscription from starting up.
 - FEATURE: Trims unused parts of the public API:
+
   - Methods removed:
-    * ConnectionManager#stats (never used and the ConnectionManager is not exposed to application code)
-    * Client#cancel (effectively duplicate of Client#close and the Client is only available via the ConnectionManager, thus not available to application code)
+
+    - ConnectionManager#stats (never used and the ConnectionManager is not exposed to application code)
+    - Client#cancel (effectively duplicate of Client#close and the Client is only available via the ConnectionManager, thus not available to application code)
 
   - Methods made private:
-    * MessageBus::Implementation#encode_channel_name
-    * MessageBus::Implementation#decode_channel_name
-    * Client#in_async?
-    * Client#ensure_closed!
-    * ConnectionManager#subscribe_client
-    * Diagnostics.full_process_path
-    * Diagnostics.hostname
-    * MessageBus::Rack::Diagnostics#js_asset
-    * MessageBus::Rack::Diagnostics#generate_script_tag
-    * MessageBus::Rack::Diagnostics#file_hash
-    * MessageBus::Rack::Diagnostics#asset_contents
-    * MessageBus::Rack::Diagnostics#asset_path
-    * MessageBus::Rack::Diagnostics#index
-    * MessageBus::Rack::Diagnostics#translate_handlebars
-    * MessageBus::Rack::Diagnostics#indent
-    * MessageBus::Rack::Middleware#start_listener
-    * MessageBus::Rack::Middleware#close_db_connection!
-    * MessageBus::Rack::Middleware#add_client_with_timeout
+
+    - MessageBus::Implementation#encode_channel_name
+    - MessageBus::Implementation#decode_channel_name
+    - Client#in_async?
+    - Client#ensure_closed!
+    - ConnectionManager#subscribe_client
+    - Diagnostics.full_process_path
+    - Diagnostics.hostname
+    - MessageBus::Rack::Diagnostics#js_asset
+    - MessageBus::Rack::Diagnostics#generate_script_tag
+    - MessageBus::Rack::Diagnostics#file_hash
+    - MessageBus::Rack::Diagnostics#asset_contents
+    - MessageBus::Rack::Diagnostics#asset_path
+    - MessageBus::Rack::Diagnostics#index
+    - MessageBus::Rack::Diagnostics#translate_handlebars
+    - MessageBus::Rack::Diagnostics#indent
+    - MessageBus::Rack::Middleware#start_listener
+    - MessageBus::Rack::Middleware#close_db_connection!
+    - MessageBus::Rack::Middleware#add_client_with_timeout
 
   - Methods switched from protected to private:
-    * MessageBus::Implementation#global?
-    * MessageBus::Implementation#decode_message!
-    * MessageBus::Implementation#replay_backlog
-    * MessageBus::Implementation#subscribe_impl
-    * MessageBus::Implementation#unsubscribe_impl
-    * MessageBus::Implementation#ensure_subscriber_thread
-    * MessageBus::Implementation#new_subscriber_thread
-    * MessageBus::Implementation#global_subscribe_thread
-    * MessageBus::Implementation#multi_each
-    * Client#write_headers
-    * Client#write_chunk
-    * Client#write_and_close
-    * Client#messages_to_json
+    - MessageBus::Implementation#global?
+    - MessageBus::Implementation#decode_message!
+    - MessageBus::Implementation#replay_backlog
+    - MessageBus::Implementation#subscribe_impl
+    - MessageBus::Implementation#unsubscribe_impl
+    - MessageBus::Implementation#ensure_subscriber_thread
+    - MessageBus::Implementation#new_subscriber_thread
+    - MessageBus::Implementation#global_subscribe_thread
+    - MessageBus::Implementation#multi_each
+    - Client#write_headers
+    - Client#write_chunk
+    - Client#write_and_close
+    - Client#messages_to_json
 
 15-10-2018
 
@@ -219,7 +260,7 @@
 - Version 2.1.2
 
 - FEATURE: minHiddenPollInterval set to 1500ms out of the box, ensures we never do hidden tab
-   polls at a high rate if tons of tabs are open
+  polls at a high rate if tons of tabs are open
 - FEATURE: added random 500ms to delayed polls to increase tab entropy
 
 18-12-2017
@@ -233,7 +274,7 @@
 - Version 2.1.0
 
 - FEATURE: you can now lookup last N messages on channel on subscribe from JavaScript
-   Subscribe at position (-1 - numberOfMessages) from the client
+  Subscribe at position (-1 - numberOfMessages) from the client
 
 24-11-2017
 
@@ -253,8 +294,7 @@
 - Version 2.0.7
 
 - Fix/Feature: use LUA script for publishing to bus, this eliminates a race condition
-   and ensures that we are never stuck in a multi transaction by mistake
-
+  and ensures that we are never stuck in a multi transaction by mistake
 
 29-09-2017
 
@@ -282,12 +322,10 @@
 - Version 2.0.2
 - Feature: Add on_middleware_error callback for remapping middleware errors to HTTP results
 
-
 25-07-2016
 
 - Feature: Add JavaScript MessageBus.status() function
 
-
 21-06-2016
 
 - Version 2.0.1
@@ -368,8 +406,8 @@
 
 - Version 1.1.1
 - Fix: In multisite config there was no way to specify site for last_id or backlog
-   to resolve overrides were added to #last_id and #backlog, MessageBus::Client now
-   uses the new overrides
+  to resolve overrides were added to #last_id and #backlog, MessageBus::Client now
+  uses the new overrides
 
 07-12-2015
 
@@ -379,86 +417,105 @@
 - Feature: remove most jQuery dependency from message-bus.js
 
 09-07-2015
+
 - Version 1.0.16
 - Fix: correct edge cases around keepalive checks on bus
 
 09-07-2015
+
 - Version 1.0.15
 - Feature: MessageBus.reliable_pub_sub.max_backlog_age (in secs) configurable (default to 7 days)
 - Fix: API for MessageBus.backlog("/bla") was returning global backlog by mistake
 - Change: Max global backlog size reduced to 2000 elements
 
 08-06-2015
+
 - Version 1.0.14
 - Fix: we can not use Thread#kill best keepalive can do is terminate process cleanly
 - Feature: you can opt-out of keepalive with MessageBus.keepalive_timeout = 0
 
 08-06-2015
+
 - Version 1.0.13
 - Fix: on global subscribe reconnect replay missed messages
 - Feature: keepalive tests for global subscribe, catches hung redis connections
 
 28-05-2015
+
 - Version 1.0.12
 - Feature: Support client_id targeted message
 
 06-05-2015
+
 - Version 1.0.11
 - Fix: race condition in TimerThread
 
 01-05-2015
+
 - Version: 1.0.10
 - Feature: no longer depends on EventMachine (only used for Thin backend)
 - Feature: reliable pub sub will queue messages in memory if redis is readonly, configurable
 - Fix: if redis is flushed we will continue to deliver messages
 
 23-03-2015
+
 - Version 1.0.9
 - Fix: inherit off StandardError not Exception for all exceptions raised
 
 20-03-2015
+
 - Version 1.0.8
 - Fix: aggressive short polling in background
 
 16-03-2015
+
 - Version 1.0.7
 - Feature: added pause and resume methods
 
 03-02-2015
+
 - Version 1.0.6
 - Fix: global backlog not truncating correctly
 
 23-09-2014
+
 - Version 1.0.5
 - Fix: missing custom headers from long polls
 
 23-09-2014
+
 - Version 1.0.4
 - Change: MessageBus.access_control_allow_origin_lookup to extra_response_headers_lookup
 
 23-09-2014
+
 - Version 1.0.3
 - Change: MessageBus.access_control_allow_origin to MessageBus.access_control_allow_origin_lookup
 
 23-09-2014
+
 - Version 1.0.2
 - Feature: MessageBus.access_control_allow_origin to control origin header
 
 23-09-2014
+
 - Version 1.0.1
 - Feature: $.ajax dependency can be passed in.
 - Feature: unsubscribe accepts a second param for the function to unsubscribe.
 
 22-09-2014
+
 - Version 1.0.0
 - Feature: add backgroundCallbackInterval - interval to send polls when page is in the background
 - Feature: issue a long poll as soon as page moves into the foreground
 
 11-08-2014
+
 - Version 0.9.5
 - Fix: release db connection a lot earlier for long polling (rails defer closes)
 
 13-01-2014
+
 - Version 0.9.4
 - Added support for /global/ channel to publish messages across a multisite
 - Cleaned up test harness so it uses local bus as opposed to global
@@ -467,19 +524,23 @@
 - ensure_reactor could say the reactor is running, but it was not, on first call
 
 06-12-2013
+
 - Version 0.9.3.2
 - Fix permissions in gem
 
 05-12-2013
+
 - Version 0.9.3.1
 - Add MessageBus.diagnostics() for diagnosing bus issues client side
 - Add more robustness to JavaScript, if callbacks used to fail they would halt the chain
 
 03-12-2013
+
 - Version 0.9.3
 - Remove thin dependency
 - Improve robustness under failure conditions
 
 30-09-2013
+
 - Fix failures in Ruby 1.9
 - Set up rack hijack by default in light of passengers new setting

data/README.md

@@ -26,15 +26,21 @@ MessageBus only support officially supported versions of Ruby; as of [2021-03-31
 
 Add this line to your application's Gemfile:
 
-    gem 'message_bus'
+```ruby
+gem 'message_bus'
+```
 
 And then execute:
 
-    $ bundle
+```shell
+$ bundle
+```
 
 Or install it yourself as:
 
-    $ gem install message_bus
+```shell
+$ gem install message_bus
+```
 
 ## Usage
 
@@ -109,7 +115,7 @@ MessageBus.user_id_lookup do |env|
 end
 ```
 
-If both `user_ids` and `group_ids` options are supplied when publishing a message, the message will be targeted at clients with lookup return values that  matches on either the `user_ids` **or** the `group_ids` options.
+If both `user_ids` and `group_ids` options are supplied when publishing a message, the message will be targeted at clients with lookup return values that matches on either the `user_ids` **or** the `group_ids` options.
 
 ```ruby
 MessageBus.publish "/channel", "hello", user_ids: [1, 2, 3], group_ids: [1, 2, 3]
@@ -129,7 +135,7 @@ Custom client message filters can be registered via `MessageBus#register_client_
 
 For example, ensuring that only messages seen by the server in the last 20 seconds are published to the client:
 
-```
+```ruby
 MessageBus.register_client_message_filter('/test') do |message|
   (Time.now.to_i - message.data[:published_at]) <= 20
 end
@@ -163,32 +169,6 @@ curl -H "Content-Type: application/x-www-form-urlencoded" -X POST --data "/messa
 
 You should see a reply with the messages of that channel you requested (in this case `/message`) starting at the message ID you requested (`0`). The URL parameter `dlp=t` disables long-polling: we do not want this request to stay open.
 
-### Diagnostics
-
-MessageBus comes with a diagnostics interface, which you can access at `/message-bus/_diagnostics`. This interface allows you visibility into the runtime behaviour of message_bus.
-
-In order to use the diagnostics UI in your application, it is necessary to:
-
-* Enable it
-* Define a user ID for requests
-* Define a check for admin role
-
-as an example, you can do something like this:
-
-```ruby
-MessageBus.enable_diagnostics # Must be called after `MessageBus.after_fork` if using a forking webserver
-
-MessageBus.user_id_lookup do |_env|
-  1
-end
-
-MessageBus.is_admin_lookup do |_env|
-  true
-end
-```
-
-Of course, in your real-world application, you would define these values according to your authentication/authorization logic.
-
 ### Transport
 
 MessageBus ships with 3 transport mechanisms.
@@ -280,7 +260,7 @@ MessageBus.start(); // call once at startup
 MessageBus.callbackInterval = 500;
 
 // you will get all new messages sent to channel
-MessageBus.subscribe("/channel", function(data){
+MessageBus.subscribe("/channel", function (data) {
   // data shipped from server
 });
 
@@ -386,6 +366,16 @@ headers|{}|Extra headers to be include with requests. Properties and values of o
 
 message_bus can be configured to use one of several available storage backends, and each has its own configuration options.
 
+### Keepalive
+
+To ensure correct operation of message_bus, every 60 seconds a message is broadcast to itself. If for any reason the message is not consumed by the same process within 3 keepalive intervals a warning log message is raised.
+
+To control keepalive interval use
+
+```ruby
+MessageBus.configure(keepalive_interval: 60)
+```
+
 ### Redis
 
 message_bus supports using Redis as a storage backend, and you can configure message_bus to use redis in `config/initializers/message_bus.rb`, like so:
@@ -398,19 +388,24 @@ The redis client message_bus uses is [redis-rb](https://github.com/redis/redis-r
 
 #### Data Retention
 
-Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get cleared automatically after 7 days of inactivity.
+Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get
+cleared automatically after 7 days of inactivity. By default, the backlog will be pruned on every message publication. If exact backlog
+length limiting is not required, the `clear_every` parameter can be set higher to improve performance.
 
-This is configurable via accessors on the ReliablePubSub instance.
+This is configurable via accessors on the Backend instance.
 
 ```ruby
 # only store 100 messages per channel
-MessageBus.reliable_pub_sub.max_backlog_size = 100
+MessageBus.backend_instance.max_backlog_size = 100
 
 # only store 100 global messages
-MessageBus.reliable_pub_sub.max_global_backlog_size = 100
+MessageBus.backend_instance.max_global_backlog_size = 100
 
 # flush per-channel backlog after 100 seconds of inactivity
-MessageBus.reliable_pub_sub.max_backlog_age = 100
+MessageBus.backend_instance.max_backlog_age = 100
+
+# clear the backlog every 50 messages
+MessageBus.backend_instance.clear_every = 50
 ```
 
 ### PostgreSQL
@@ -433,8 +428,7 @@ message_bus also supports an in-memory backend. This can be used for testing or
 MessageBus.configure(backend: :memory)
 ```
 
-The `:clear_every` option supported by the PostgreSQL backend is also supported by the in-memory backend.
-
+The `:clear_every` option is supported in the same way as the PostgreSQL backend.
 
 ### Transport codecs
 
@@ -456,7 +450,7 @@ Keep in mind, much of MessageBus internals and supporting tools expect data to b
 
 Another example may be very large and complicated messages where Oj in compatibility mode outperforms JSON. To opt for the Oj codec use:
 
-```
+```ruby
 MessageBus.configure(transport_codec: MessageBus::Codec::Oj.new)
 ```
 
@@ -503,7 +497,6 @@ For more information see the [Passenger documentation](https://www.phusionpassen
 
 ```ruby
 # path/to/your/config/puma.rb
-require 'message_bus' # omit this line for Rails 5
 on_worker_boot do
   MessageBus.after_fork
 end
@@ -513,7 +506,6 @@ end
 
 ```ruby
 # path/to/your/config/unicorn.rb
-require 'message_bus'
 after_fork do |server, worker|
   MessageBus.after_fork
 end
@@ -554,26 +546,21 @@ MessageBus ships with an optional `DistributedCache` API which provides a simple
 require 'message_bus/distributed_cache'
 
 # process 1
-
 cache = MessageBus::DistributedCache.new("animals")
 
 # process 2
-
 cache = MessageBus::DistributedCache.new("animals")
 
 # process 1
-
 cache["frogs"] = 5
 
 # process 2
-
 puts cache["frogs"]
 # => 5
 
 cache["frogs"] = nil
 
 # process 1
-
 puts cache["frogs"]
 # => nil
 ```
@@ -631,7 +618,7 @@ In e.g. `config/initializers/message_bus.rb`:
 ```ruby
 MessageBus.extra_response_headers_lookup do |env|
   [
-      ["Access-Control-Allow-Origin", "http://example.com:3000"],
+    ["Access-Control-Allow-Origin", "http://example.com:3000"],
   ]
 end
 ```
@@ -692,8 +679,8 @@ In certain conditions, a status message will be delivered and look like this:
   "message_id": -1,
   "channel": "/__status",
   "data": {
-    "/some/channel":5,
-    "/other/channel":9
+    "/some/channel": 5,
+    "/other/channel": 9
   }
 }
 ```
@@ -727,7 +714,7 @@ When submitting a PR, please be sure to include notes on it in the `Unreleased`
 
 To run tests you need both Postgres and Redis installed. By default on Redis the tests connect to `localhost:6379` and on Postgres connect the database `localhost:5432/message_bus_test` with the system username; if you wish to override this, you can set alternative values:
 
-```
+```shell
 PGUSER=some_user PGDATABASE=some_db bundle exec rake
 ```
 
@@ -742,7 +729,3 @@ While working on documentation, it is useful to automatically re-build it as you
 ### Benchmarks
 
 Some simple benchmarks are implemented in `spec/performance` and can be executed using `rake performance` (or `docker-compose run tests rake performance`). You should run these before and after your changes to avoid introducing performance regressions.
-
-### Diagnostics Interface
-
-It is possible to manually test the diagnostics interface by executing `docker-compose up example` and then `open http://localhost:9292`.