message_bus 3.3.7 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16601ac02268a477a94bf0347922162f66ce17dfdba1c4c655d576b46e78a79c
-  data.tar.gz: 746e2295f873d9cfe75c28b079b406bb42329a992146d414c63292c2148a8f48
+  metadata.gz: 11a14ba0ec9ad142d3335fb7eb764be8e9b975a8dc8367fd4bf359e128a56095
+  data.tar.gz: bf41cede85c8e75e56829447cd3cf9ed6cd531ba27fcddfbccb6043f3a82da34
 SHA512:
-  metadata.gz: 3d048dde14c63d5abdda283571cb9c2be1b7fc809a94c7f0b4ec2eaa4785da03bc47535264741f65cfa1d66c75e1b0169ea0de39449456b9c791ba91518d62e9
-  data.tar.gz: 0cfbed1cab954e33805c94c7662c9094276615d5f338168da48d26f033da0f42685024b87aaee64d221e192d0592cd655fba7473da8d973616b218ce2858b045
+  metadata.gz: 23c1d181028194c16e98c5a3df42e445458aef4779c97ae2d6d5100adf5053680f62ea4c914bea7c84c124bd453127e141c68ffeb1c742cc208e1d040bc8558f
+  data.tar.gz: 7efffe711fac2520356b115820acba2d4206d53f35017ba64c86e3c571a85fb423e943cb9301a03d3b47d6f442f1626dea0495c0242220283685eaa944c890b6

data/.eslintrc.js

@@ -10,12 +10,5 @@ module.exports = {
     sourceType: 'module',
   },
   rules: {},
-  ignorePatterns: [
-    '/vendor',
-    '/doc',
-    '/assets/babel.min.js',
-    '/assets/jquery-1.8.2.js',
-    '/assets/react-dom.js',
-    '/assets/react.js',
-  ],
+  ignorePatterns: ['/vendor', '/doc', '/assets/jquery-1.8.2.js'],
 };

data/.github/workflows/ci.yml

@@ -1,16 +1,10 @@
-name: Message Bus Tests
+name: CI
 
 on:
-  pull_request:
   push:
     branches:
       - main
-
-env:
-  PGHOST: localhost
-  PGPORT: 5432
-  PGPASSWORD: postgres
-  PGUSER: postgres
+  pull_request:
 
 jobs:
   build:
@@ -18,10 +12,22 @@ jobs:
     name: Ruby ${{ matrix.ruby }} (redis ${{ matrix.redis }})
     timeout-minutes: 10
 
+    env:
+      PGHOST: localhost
+      PGPASSWORD: postgres
+      PGUSER: postgres
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.6, 2.7, '3.0', 3.1]
+        redis: [5, 6]
+
     services:
       postgres:
-        image: postgres:14.0
+        image: postgres:14
         env:
+          POSTGRES_DB: message_bus_test
           POSTGRES_PASSWORD: postgres
         ports:
           - 5432:5432
@@ -36,19 +42,13 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: ["3.0", "2.7", "2.6"]
-        redis: ["5", "6"]
-
     steps:
       - uses: actions/checkout@v2
 
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true # 'bundle install' and cache
+          bundler-cache: true
 
       - name: Set up Node.js
         uses: actions/setup-node@v2
@@ -59,13 +59,43 @@ jobs:
       - name: Setup npm
         run: npm install
 
-      - name: Create Database
-        run: |
-          createdb message_bus_test
-
       - name: Tests
+        env:
+          TESTOPTS: --verbose
         run: bundle exec rake
         timeout-minutes: 3
 
       - name: Linting
         run: npx eslint .
+
+  publish:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Release gem
+        uses: discourse/publish-rubygems-action@v2
+        id: publish-gem
+        env:
+          RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }}
+          GIT_EMAIL: team@discourse.org
+          GIT_NAME: discoursebot
+
+      - name: Update package version
+        if: steps.publish-gem.outputs.new_version == 'true'
+        run: |
+          VERSION=$(ruby -r './lib/message_bus/version' -e 'puts MessageBus::VERSION')
+          sed -i "s/0.0.0-version-placeholder/$VERSION/" package.json
+          git config --global user.email "ci@ci.invalid"
+          git config --global user.name "Discourse CI"
+          git add package.json
+          git commit -m 'bump'
+
+      - name: Publish package
+        uses: JS-DevTools/npm-publish@v1
+        if: steps.publish-gem.outputs.new_version == 'true'
+        with:
+          token: ${{ secrets.NPM_TOKEN }}

data/.prettierrc

@@ -0,0 +1 @@
+{}

data/CHANGELOG

@@ -1,3 +1,34 @@
+16-02-2022
+
+- Version 4.1.0
+
+  - PERF: Optimize Client#backlog for up-to-date clients
+
+    Also introduces a new MessageBus#last_ids(*channels) api for fetching the last_ids of
+    multiple channels simultaneously
+
+11-01-2022
+
+- Version 4.0.0
+
+  - DEV: Remove backend diagnostics
+  - DEV: Rename reliable_pub_sub to backend_instance
+  - FIX: `destroy` following `after_fork` could thread lock
+
+31-12-2021
+
+- Version 3.4.0
+
+  - FEATURE: Remove process auto-termination on missed keepalives
+
+20-12-2021
+
+- Version 3.3.8
+
+  - FIX: Restore dist js files to fix a regression in 3.3.7
+  - FIX: Stop various thread/pg connection leaks
+  - DEV: Fix ruby warnings
+
 15-12-2021
 
 - Version 3.3.7
@@ -63,14 +94,14 @@
 
 - Version 3.3.0
 
-   - FEATURE: `MessageBus.base_route=` to alter the route that message bus will listen on.
+  - FEATURE: `MessageBus.base_route=` to alter the route that message bus will listen on.
 
 07-05-2020
 
 - Version 3.2.0
 
-   - FIX: compatibility with Rails 6.0.3, note: apps without ActionDispatch::Flash may stop working after this upgrade
-   to correct this disable middleware injection with `config.skip_message_bus_middleware = true` and configure middleware by hand with `app.middleware.use(MessageBus::Rack::Middleware)`
+  - FIX: compatibility with Rails 6.0.3, note: apps without ActionDispatch::Flash may stop working after this upgrade
+    to correct this disable middleware injection with `config.skip_message_bus_middleware = true` and configure middleware by hand with `app.middleware.use(MessageBus::Rack::Middleware)`
 
 28-04-2020
 
@@ -105,7 +136,6 @@
 
   - FIX: In the redis backend make the `is_readonly?` method compatible with the redis gem both pre and post v4.0 when the `client` attribute was removed
 
-
 30-04-2019
 
 - Version 2.2.1
@@ -136,7 +166,7 @@
 - Version 2.2.0.pre
 
 - FIX: In redis backend we now expire the key used to track channel id this can cause a redis key leak
-    with large amounts of subscriptions that go away
+  with large amounts of subscriptions that go away
 - FEATURE: Much extra implementation documentation, and some improvements to usage documentation.
 - FEATURE: Improvements to development workflow:
   - Fully docker-based development and testing, with no other dependencies.
@@ -153,44 +183,47 @@
   - Supports setting backlog size on publication for memory/postgres
 - FEATURE: `MessageBus.off` now prevents the server subscription from starting up.
 - FEATURE: Trims unused parts of the public API:
+
   - Methods removed:
-    * ConnectionManager#stats (never used and the ConnectionManager is not exposed to application code)
-    * Client#cancel (effectively duplicate of Client#close and the Client is only available via the ConnectionManager, thus not available to application code)
+
+    - ConnectionManager#stats (never used and the ConnectionManager is not exposed to application code)
+    - Client#cancel (effectively duplicate of Client#close and the Client is only available via the ConnectionManager, thus not available to application code)
 
   - Methods made private:
-    * MessageBus::Implementation#encode_channel_name
-    * MessageBus::Implementation#decode_channel_name
-    * Client#in_async?
-    * Client#ensure_closed!
-    * ConnectionManager#subscribe_client
-    * Diagnostics.full_process_path
-    * Diagnostics.hostname
-    * MessageBus::Rack::Diagnostics#js_asset
-    * MessageBus::Rack::Diagnostics#generate_script_tag
-    * MessageBus::Rack::Diagnostics#file_hash
-    * MessageBus::Rack::Diagnostics#asset_contents
-    * MessageBus::Rack::Diagnostics#asset_path
-    * MessageBus::Rack::Diagnostics#index
-    * MessageBus::Rack::Diagnostics#translate_handlebars
-    * MessageBus::Rack::Diagnostics#indent
-    * MessageBus::Rack::Middleware#start_listener
-    * MessageBus::Rack::Middleware#close_db_connection!
-    * MessageBus::Rack::Middleware#add_client_with_timeout
+
+    - MessageBus::Implementation#encode_channel_name
+    - MessageBus::Implementation#decode_channel_name
+    - Client#in_async?
+    - Client#ensure_closed!
+    - ConnectionManager#subscribe_client
+    - Diagnostics.full_process_path
+    - Diagnostics.hostname
+    - MessageBus::Rack::Diagnostics#js_asset
+    - MessageBus::Rack::Diagnostics#generate_script_tag
+    - MessageBus::Rack::Diagnostics#file_hash
+    - MessageBus::Rack::Diagnostics#asset_contents
+    - MessageBus::Rack::Diagnostics#asset_path
+    - MessageBus::Rack::Diagnostics#index
+    - MessageBus::Rack::Diagnostics#translate_handlebars
+    - MessageBus::Rack::Diagnostics#indent
+    - MessageBus::Rack::Middleware#start_listener
+    - MessageBus::Rack::Middleware#close_db_connection!
+    - MessageBus::Rack::Middleware#add_client_with_timeout
 
   - Methods switched from protected to private:
-    * MessageBus::Implementation#global?
-    * MessageBus::Implementation#decode_message!
-    * MessageBus::Implementation#replay_backlog
-    * MessageBus::Implementation#subscribe_impl
-    * MessageBus::Implementation#unsubscribe_impl
-    * MessageBus::Implementation#ensure_subscriber_thread
-    * MessageBus::Implementation#new_subscriber_thread
-    * MessageBus::Implementation#global_subscribe_thread
-    * MessageBus::Implementation#multi_each
-    * Client#write_headers
-    * Client#write_chunk
-    * Client#write_and_close
-    * Client#messages_to_json
+    - MessageBus::Implementation#global?
+    - MessageBus::Implementation#decode_message!
+    - MessageBus::Implementation#replay_backlog
+    - MessageBus::Implementation#subscribe_impl
+    - MessageBus::Implementation#unsubscribe_impl
+    - MessageBus::Implementation#ensure_subscriber_thread
+    - MessageBus::Implementation#new_subscriber_thread
+    - MessageBus::Implementation#global_subscribe_thread
+    - MessageBus::Implementation#multi_each
+    - Client#write_headers
+    - Client#write_chunk
+    - Client#write_and_close
+    - Client#messages_to_json
 
 15-10-2018
 
@@ -219,7 +252,7 @@
 - Version 2.1.2
 
 - FEATURE: minHiddenPollInterval set to 1500ms out of the box, ensures we never do hidden tab
-   polls at a high rate if tons of tabs are open
+  polls at a high rate if tons of tabs are open
 - FEATURE: added random 500ms to delayed polls to increase tab entropy
 
 18-12-2017
@@ -233,7 +266,7 @@
 - Version 2.1.0
 
 - FEATURE: you can now lookup last N messages on channel on subscribe from JavaScript
-   Subscribe at position (-1 - numberOfMessages) from the client
+  Subscribe at position (-1 - numberOfMessages) from the client
 
 24-11-2017
 
@@ -253,8 +286,7 @@
 - Version 2.0.7
 
 - Fix/Feature: use LUA script for publishing to bus, this eliminates a race condition
-   and ensures that we are never stuck in a multi transaction by mistake
-
+  and ensures that we are never stuck in a multi transaction by mistake
 
 29-09-2017
 
@@ -282,12 +314,10 @@
 - Version 2.0.2
 - Feature: Add on_middleware_error callback for remapping middleware errors to HTTP results
 
-
 25-07-2016
 
 - Feature: Add JavaScript MessageBus.status() function
 
-
 21-06-2016
 
 - Version 2.0.1
@@ -368,8 +398,8 @@
 
 - Version 1.1.1
 - Fix: In multisite config there was no way to specify site for last_id or backlog
-   to resolve overrides were added to #last_id and #backlog, MessageBus::Client now
-   uses the new overrides
+  to resolve overrides were added to #last_id and #backlog, MessageBus::Client now
+  uses the new overrides
 
 07-12-2015
 
@@ -379,86 +409,105 @@
 - Feature: remove most jQuery dependency from message-bus.js
 
 09-07-2015
+
 - Version 1.0.16
 - Fix: correct edge cases around keepalive checks on bus
 
 09-07-2015
+
 - Version 1.0.15
 - Feature: MessageBus.reliable_pub_sub.max_backlog_age (in secs) configurable (default to 7 days)
 - Fix: API for MessageBus.backlog("/bla") was returning global backlog by mistake
 - Change: Max global backlog size reduced to 2000 elements
 
 08-06-2015
+
 - Version 1.0.14
 - Fix: we can not use Thread#kill best keepalive can do is terminate process cleanly
 - Feature: you can opt-out of keepalive with MessageBus.keepalive_timeout = 0
 
 08-06-2015
+
 - Version 1.0.13
 - Fix: on global subscribe reconnect replay missed messages
 - Feature: keepalive tests for global subscribe, catches hung redis connections
 
 28-05-2015
+
 - Version 1.0.12
 - Feature: Support client_id targeted message
 
 06-05-2015
+
 - Version 1.0.11
 - Fix: race condition in TimerThread
 
 01-05-2015
+
 - Version: 1.0.10
 - Feature: no longer depends on EventMachine (only used for Thin backend)
 - Feature: reliable pub sub will queue messages in memory if redis is readonly, configurable
 - Fix: if redis is flushed we will continue to deliver messages
 
 23-03-2015
+
 - Version 1.0.9
 - Fix: inherit off StandardError not Exception for all exceptions raised
 
 20-03-2015
+
 - Version 1.0.8
 - Fix: aggressive short polling in background
 
 16-03-2015
+
 - Version 1.0.7
 - Feature: added pause and resume methods
 
 03-02-2015
+
 - Version 1.0.6
 - Fix: global backlog not truncating correctly
 
 23-09-2014
+
 - Version 1.0.5
 - Fix: missing custom headers from long polls
 
 23-09-2014
+
 - Version 1.0.4
 - Change: MessageBus.access_control_allow_origin_lookup to extra_response_headers_lookup
 
 23-09-2014
+
 - Version 1.0.3
 - Change: MessageBus.access_control_allow_origin to MessageBus.access_control_allow_origin_lookup
 
 23-09-2014
+
 - Version 1.0.2
 - Feature: MessageBus.access_control_allow_origin to control origin header
 
 23-09-2014
+
 - Version 1.0.1
 - Feature: $.ajax dependency can be passed in.
 - Feature: unsubscribe accepts a second param for the function to unsubscribe.
 
 22-09-2014
+
 - Version 1.0.0
 - Feature: add backgroundCallbackInterval - interval to send polls when page is in the background
 - Feature: issue a long poll as soon as page moves into the foreground
 
 11-08-2014
+
 - Version 0.9.5
 - Fix: release db connection a lot earlier for long polling (rails defer closes)
 
 13-01-2014
+
 - Version 0.9.4
 - Added support for /global/ channel to publish messages across a multisite
 - Cleaned up test harness so it uses local bus as opposed to global
@@ -467,19 +516,23 @@
 - ensure_reactor could say the reactor is running, but it was not, on first call
 
 06-12-2013
+
 - Version 0.9.3.2
 - Fix permissions in gem
 
 05-12-2013
+
 - Version 0.9.3.1
 - Add MessageBus.diagnostics() for diagnosing bus issues client side
 - Add more robustness to JavaScript, if callbacks used to fail they would halt the chain
 
 03-12-2013
+
 - Version 0.9.3
 - Remove thin dependency
 - Improve robustness under failure conditions
 
 30-09-2013
+
 - Fix failures in Ruby 1.9
 - Set up rack hijack by default in light of passengers new setting

data/README.md

@@ -26,15 +26,21 @@ MessageBus only support officially supported versions of Ruby; as of [2021-03-31
 
 Add this line to your application's Gemfile:
 
-    gem 'message_bus'
+```ruby
+gem 'message_bus'
+```
 
 And then execute:
 
-    $ bundle
+```shell
+$ bundle
+```
 
 Or install it yourself as:
 
-    $ gem install message_bus
+```shell
+$ gem install message_bus
+```
 
 ## Usage
 
@@ -109,7 +115,7 @@ MessageBus.user_id_lookup do |env|
 end
 ```
 
-If both `user_ids` and `group_ids` options are supplied when publishing a message, the message will be targeted at clients with lookup return values that  matches on either the `user_ids` **or** the `group_ids` options.
+If both `user_ids` and `group_ids` options are supplied when publishing a message, the message will be targeted at clients with lookup return values that matches on either the `user_ids` **or** the `group_ids` options.
 
 ```ruby
 MessageBus.publish "/channel", "hello", user_ids: [1, 2, 3], group_ids: [1, 2, 3]
@@ -129,7 +135,7 @@ Custom client message filters can be registered via `MessageBus#register_client_
 
 For example, ensuring that only messages seen by the server in the last 20 seconds are published to the client:
 
-```
+```ruby
 MessageBus.register_client_message_filter('/test') do |message|
   (Time.now.to_i - message.data[:published_at]) <= 20
 end
@@ -163,32 +169,6 @@ curl -H "Content-Type: application/x-www-form-urlencoded" -X POST --data "/messa
 
 You should see a reply with the messages of that channel you requested (in this case `/message`) starting at the message ID you requested (`0`). The URL parameter `dlp=t` disables long-polling: we do not want this request to stay open.
 
-### Diagnostics
-
-MessageBus comes with a diagnostics interface, which you can access at `/message-bus/_diagnostics`. This interface allows you visibility into the runtime behaviour of message_bus.
-
-In order to use the diagnostics UI in your application, it is necessary to:
-
-* Enable it
-* Define a user ID for requests
-* Define a check for admin role
-
-as an example, you can do something like this:
-
-```ruby
-MessageBus.enable_diagnostics # Must be called after `MessageBus.after_fork` if using a forking webserver
-
-MessageBus.user_id_lookup do |_env|
-  1
-end
-
-MessageBus.is_admin_lookup do |_env|
-  true
-end
-```
-
-Of course, in your real-world application, you would define these values according to your authentication/authorization logic.
-
 ### Transport
 
 MessageBus ships with 3 transport mechanisms.
@@ -280,7 +260,7 @@ MessageBus.start(); // call once at startup
 MessageBus.callbackInterval = 500;
 
 // you will get all new messages sent to channel
-MessageBus.subscribe("/channel", function(data){
+MessageBus.subscribe("/channel", function (data) {
   // data shipped from server
 });
 
@@ -386,6 +366,16 @@ headers|{}|Extra headers to be include with requests. Properties and values of o
 
 message_bus can be configured to use one of several available storage backends, and each has its own configuration options.
 
+### Keepalive
+
+To ensure correct operation of message_bus, every 60 seconds a message is broadcast to itself. If for any reason the message is not consumed by the same process within 3 keepalive intervals a warning log message is raised.
+
+To control keepalive interval use
+
+```ruby
+MessageBus.configure(keepalive_interval: 60)
+```
+
 ### Redis
 
 message_bus supports using Redis as a storage backend, and you can configure message_bus to use redis in `config/initializers/message_bus.rb`, like so:
@@ -400,17 +390,17 @@ The redis client message_bus uses is [redis-rb](https://github.com/redis/redis-r
 
 Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get cleared automatically after 7 days of inactivity.
 
-This is configurable via accessors on the ReliablePubSub instance.
+This is configurable via accessors on the Backend instance.
 
 ```ruby
 # only store 100 messages per channel
-MessageBus.reliable_pub_sub.max_backlog_size = 100
+MessageBus.backend_instance.max_backlog_size = 100
 
 # only store 100 global messages
-MessageBus.reliable_pub_sub.max_global_backlog_size = 100
+MessageBus.backend_instance.max_global_backlog_size = 100
 
 # flush per-channel backlog after 100 seconds of inactivity
-MessageBus.reliable_pub_sub.max_backlog_age = 100
+MessageBus.backend_instance.max_backlog_age = 100
 ```
 
 ### PostgreSQL
@@ -435,7 +425,6 @@ MessageBus.configure(backend: :memory)
 
 The `:clear_every` option supported by the PostgreSQL backend is also supported by the in-memory backend.
 
-
 ### Transport codecs
 
 By default MessageBus serializes messages to the backend using JSON. Under most situation this performs extremely well.
@@ -456,7 +445,7 @@ Keep in mind, much of MessageBus internals and supporting tools expect data to b
 
 Another example may be very large and complicated messages where Oj in compatibility mode outperforms JSON. To opt for the Oj codec use:
 
-```
+```ruby
 MessageBus.configure(transport_codec: MessageBus::Codec::Oj.new)
 ```
 
@@ -503,7 +492,6 @@ For more information see the [Passenger documentation](https://www.phusionpassen
 
 ```ruby
 # path/to/your/config/puma.rb
-require 'message_bus' # omit this line for Rails 5
 on_worker_boot do
   MessageBus.after_fork
 end
@@ -513,7 +501,6 @@ end
 
 ```ruby
 # path/to/your/config/unicorn.rb
-require 'message_bus'
 after_fork do |server, worker|
   MessageBus.after_fork
 end
@@ -554,26 +541,21 @@ MessageBus ships with an optional `DistributedCache` API which provides a simple
 require 'message_bus/distributed_cache'
 
 # process 1
-
 cache = MessageBus::DistributedCache.new("animals")
 
 # process 2
-
 cache = MessageBus::DistributedCache.new("animals")
 
 # process 1
-
 cache["frogs"] = 5
 
 # process 2
-
 puts cache["frogs"]
 # => 5
 
 cache["frogs"] = nil
 
 # process 1
-
 puts cache["frogs"]
 # => nil
 ```
@@ -631,7 +613,7 @@ In e.g. `config/initializers/message_bus.rb`:
 ```ruby
 MessageBus.extra_response_headers_lookup do |env|
   [
-      ["Access-Control-Allow-Origin", "http://example.com:3000"],
+    ["Access-Control-Allow-Origin", "http://example.com:3000"],
   ]
 end
 ```
@@ -692,8 +674,8 @@ In certain conditions, a status message will be delivered and look like this:
   "message_id": -1,
   "channel": "/__status",
   "data": {
-    "/some/channel":5,
-    "/other/channel":9
+    "/some/channel": 5,
+    "/other/channel": 9
   }
 }
 ```
@@ -727,7 +709,7 @@ When submitting a PR, please be sure to include notes on it in the `Unreleased`
 
 To run tests you need both Postgres and Redis installed. By default on Redis the tests connect to `localhost:6379` and on Postgres connect the database `localhost:5432/message_bus_test` with the system username; if you wish to override this, you can set alternative values:
 
-```
+```shell
 PGUSER=some_user PGDATABASE=some_db bundle exec rake
 ```
 
@@ -742,7 +724,3 @@ While working on documentation, it is useful to automatically re-build it as you
 ### Benchmarks
 
 Some simple benchmarks are implemented in `spec/performance` and can be executed using `rake performance` (or `docker-compose run tests rake performance`). You should run these before and after your changes to avoid introducing performance regressions.
-
-### Diagnostics Interface
-
-It is possible to manually test the diagnostics interface by executing `docker-compose up example` and then `open http://localhost:9292`.