message_bus 4.6.0 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +9 -1
- data/CHANGELOG +23 -0
- data/Dockerfile +1 -1
- data/README.md +2 -2
- data/lib/message_bus/backends/base.rb +14 -1
- data/lib/message_bus/backends/memory.rb +9 -4
- data/lib/message_bus/backends/postgres.rb +75 -37
- data/lib/message_bus/backends/redis.rb +30 -7
- data/lib/message_bus/version.rb +1 -1
- data/lib/message_bus.rb +21 -6
- data/message_bus.gemspec +3 -2
- data/spec/helpers.rb +61 -1
- data/spec/lib/message_bus/backend_spec.rb +316 -97
- data/spec/lib/message_bus/rack/middleware_spec.rb +5 -4
- data/spec/lib/message_bus_spec.rb +106 -0
- data/spec/spec_helper.rb +3 -1
- metadata +18 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 52492917d068eb8e797c93a776b0a7496f430bad3bf0fe79c5b1e192091cb18d
|
|
4
|
+
data.tar.gz: da51e98e5f18e67a4de7769b43cf039eada80a5314af9552635e9e4934167c2d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b59e2edf77537c7c8d5d55903517a37378f041dfbf4cb0f63e4dc6f2ab51c744395527c34b8152ac9f82b6435390edb84fc91816980698bc58210bc5d0a138b4
|
|
7
|
+
data.tar.gz: e750096d991a99db95d4e447e6c5f22f023df1155ef38b5517ab5aa23488fd2f5c87dce04153b8246ff439690385e819cfd3e052ef9181030815b78b95054e23
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -50,9 +50,15 @@ jobs:
|
|
|
50
50
|
strategy:
|
|
51
51
|
fail-fast: false
|
|
52
52
|
matrix:
|
|
53
|
-
ruby: ["3.
|
|
53
|
+
ruby: ["3.3", "3.4", "4.0"]
|
|
54
54
|
rack: ["rack2", "rack3"]
|
|
55
55
|
redis: ["redis:7.4", "valkey/valkey"]
|
|
56
|
+
experimental: [false]
|
|
57
|
+
include:
|
|
58
|
+
- ruby: ruby-head
|
|
59
|
+
rack: rack3
|
|
60
|
+
redis: redis:7.4
|
|
61
|
+
experimental: true
|
|
56
62
|
|
|
57
63
|
services:
|
|
58
64
|
postgres:
|
|
@@ -77,6 +83,7 @@ jobs:
|
|
|
77
83
|
- uses: actions/checkout@v4
|
|
78
84
|
|
|
79
85
|
- uses: ruby/setup-ruby@v1
|
|
86
|
+
continue-on-error: ${{ matrix.experimental }}
|
|
80
87
|
with:
|
|
81
88
|
ruby-version: ${{ matrix.ruby }}
|
|
82
89
|
bundler-cache: true
|
|
@@ -92,6 +99,7 @@ jobs:
|
|
|
92
99
|
run: pnpm install
|
|
93
100
|
|
|
94
101
|
- name: Tests
|
|
102
|
+
continue-on-error: ${{ matrix.experimental }}
|
|
95
103
|
env:
|
|
96
104
|
TESTOPTS: --verbose
|
|
97
105
|
run: bundle exec rake
|
data/CHANGELOG
CHANGED
|
@@ -1,8 +1,31 @@
|
|
|
1
|
+
19-11-2026
|
|
2
|
+
|
|
3
|
+
- Version 5.1.0
|
|
4
|
+
|
|
5
|
+
- FEATURE: `expire_all_backlogs!` accepts an optional `site_id:` to expire the backlogs of a single site
|
|
6
|
+
- FIX: `expire_all_backlogs!` on the Redis backend now expires per-channel backlogs, not only the global backlog
|
|
7
|
+
|
|
8
|
+
04-09-2026
|
|
9
|
+
|
|
10
|
+
- Version 5.0.0
|
|
11
|
+
|
|
12
|
+
- BREAKING: Drop support for EOL Ruby 3.2; Ruby 3.3 or newer is now required
|
|
13
|
+
- DEV: Test against all maintained Ruby versions: 3.3, 3.4, and 4.0, plus Ruby head as a non-blocking compatibility check
|
|
14
|
+
- DEV: Keep the Rack 2 test suite compatible with Ruby 4.0
|
|
15
|
+
- DEV: Stabilize time-sensitive backlog expiry and long-polling tests on busy CI runners
|
|
16
|
+
- FIX: Recover Redis and PostgreSQL backend subscribers from half-open connections without restarting the process
|
|
17
|
+
- FIX: Keep the keepalive watchdog active if a subscriber reconnect request fails
|
|
18
|
+
- FIX: Replay missed global messages after Redis and PostgreSQL subscriber reconnects without delivering duplicates
|
|
19
|
+
- FIX: Request PostgreSQL reconnects on the subscriber thread to respect libpq thread ownership
|
|
20
|
+
|
|
1
21
|
03-08-2026
|
|
2
22
|
|
|
3
23
|
- Version 4.6.0
|
|
4
24
|
|
|
5
25
|
- DEV: Export the JavaScript client as a CommonJS/ES module so it can be imported directly by bundlers and ESM projects, while still assigning the `MessageBus` global for existing consumers
|
|
26
|
+
- DEV: Add `logger` as a runtime dependency
|
|
27
|
+
- DEV: Make `MessageBus::Message` a regular class instead of a `Struct`
|
|
28
|
+
- PERF: Use `Set#intersect?` when checking allowed groups
|
|
6
29
|
|
|
7
30
|
13-02-2026
|
|
8
31
|
|
data/Dockerfile
CHANGED
data/README.md
CHANGED
|
@@ -12,7 +12,7 @@ Read the generated docs: <https://www.rubydoc.info/gems/message_bus>
|
|
|
12
12
|
|
|
13
13
|
## Ruby version support
|
|
14
14
|
|
|
15
|
-
MessageBus only
|
|
15
|
+
MessageBus only supports Ruby versions currently maintained by the Ruby core team. As of [2026-09-04](https://www.ruby-lang.org/en/downloads/branches/), the supported versions are Ruby 3.3, 3.4, and 4.0; the minimum required version is Ruby 3.3.
|
|
16
16
|
|
|
17
17
|
## Can you handle concurrent requests?
|
|
18
18
|
|
|
@@ -368,7 +368,7 @@ message_bus can be configured to use one of several available storage backends,
|
|
|
368
368
|
|
|
369
369
|
### Keepalive
|
|
370
370
|
|
|
371
|
-
To ensure correct operation of message_bus, every 60 seconds a message is broadcast to itself. If for any reason the message is not consumed by the same process within 3 keepalive intervals a warning log message is raised.
|
|
371
|
+
To ensure correct operation of message_bus, every 60 seconds a message is broadcast to itself. If for any reason the message is not consumed by the same process within 3 keepalive intervals, a warning log message is raised and the backend subscriber is asked to reconnect, so a wedged connection (for example, a half-open socket) can self-heal without a process restart.
|
|
372
372
|
|
|
373
373
|
To control keepalive interval use
|
|
374
374
|
|
|
@@ -89,8 +89,9 @@ module MessageBus
|
|
|
89
89
|
end
|
|
90
90
|
|
|
91
91
|
# Deletes all backlogs and their data. Does not delete non-backlog data that message_bus may persist, depending on the concrete backend implementation. Use with extreme caution.
|
|
92
|
+
# @param [String] site_id if given, only the backlogs of that site's channels are deleted
|
|
92
93
|
# @abstract
|
|
93
|
-
def expire_all_backlogs!
|
|
94
|
+
def expire_all_backlogs!(site_id: nil)
|
|
94
95
|
raise ConcreteClassMustImplementError
|
|
95
96
|
end
|
|
96
97
|
|
|
@@ -175,6 +176,18 @@ module MessageBus
|
|
|
175
176
|
raise ConcreteClassMustImplementError
|
|
176
177
|
end
|
|
177
178
|
|
|
179
|
+
# Asks the subscriber to drop its connection and re-establish it, so a
|
|
180
|
+
# wedged connection (a half-open socket, for example) recovers without a
|
|
181
|
+
# process restart. The backend's rescue/retry around {#global_subscribe}
|
|
182
|
+
# performs the actual reconnection.
|
|
183
|
+
#
|
|
184
|
+
# Called from the keepalive watchdog thread, never from the subscriber
|
|
185
|
+
# thread; implementations must not touch anything requiring
|
|
186
|
+
# single-thread ownership (see the Postgres backend). Backends with no
|
|
187
|
+
# way to detach a stuck connection may leave this as the inherited no-op.
|
|
188
|
+
def request_reconnect
|
|
189
|
+
end
|
|
190
|
+
|
|
178
191
|
# Subscribe to messages on all channels. Each message since the last ID
|
|
179
192
|
# specified will be delivered by yielding to the passed block as soon as
|
|
180
193
|
# it is available. This will block until subscription is terminated.
|
|
@@ -130,9 +130,14 @@ module MessageBus
|
|
|
130
130
|
end
|
|
131
131
|
|
|
132
132
|
# use with extreme care, will nuke all of the data
|
|
133
|
-
def expire_all_backlogs!
|
|
133
|
+
def expire_all_backlogs!(site_id: nil)
|
|
134
134
|
sync do
|
|
135
|
-
|
|
135
|
+
if site_id
|
|
136
|
+
site_suffix = "#{MessageBus::Implementation::ENCODE_SITE_TOKEN}#{site_id}"
|
|
137
|
+
@channels.delete_if { |name, _| name.end_with?(site_suffix) }
|
|
138
|
+
else
|
|
139
|
+
@channels = {}
|
|
140
|
+
end
|
|
136
141
|
end
|
|
137
142
|
end
|
|
138
143
|
|
|
@@ -219,8 +224,8 @@ module MessageBus
|
|
|
219
224
|
end
|
|
220
225
|
|
|
221
226
|
# (see Base#expire_all_backlogs!)
|
|
222
|
-
def expire_all_backlogs!
|
|
223
|
-
client.expire_all_backlogs!
|
|
227
|
+
def expire_all_backlogs!(site_id: nil)
|
|
228
|
+
client.expire_all_backlogs!(site_id: site_id)
|
|
224
229
|
end
|
|
225
230
|
|
|
226
231
|
# (see Base#publish)
|
|
@@ -20,6 +20,10 @@ module MessageBus
|
|
|
20
20
|
#
|
|
21
21
|
# @see Base general information about message_bus backends
|
|
22
22
|
class Postgres < Base
|
|
23
|
+
# Raised inside the subscriber thread to make it drop its LISTEN
|
|
24
|
+
# connection and reconnect. See {Postgres#request_reconnect}.
|
|
25
|
+
ReconnectRequested = Class.new(StandardError)
|
|
26
|
+
|
|
23
27
|
class Client
|
|
24
28
|
INHERITED_CONNECTIONS = []
|
|
25
29
|
|
|
@@ -45,6 +49,7 @@ module MessageBus
|
|
|
45
49
|
@available = []
|
|
46
50
|
@allocated = {}
|
|
47
51
|
@subscribe_connection = nil
|
|
52
|
+
@reconnect_requested = false
|
|
48
53
|
@subscribed = false
|
|
49
54
|
@mutex = Mutex.new
|
|
50
55
|
@pid = Process.pid
|
|
@@ -112,8 +117,13 @@ module MessageBus
|
|
|
112
117
|
end
|
|
113
118
|
|
|
114
119
|
# use with extreme care, will nuke all of the data
|
|
115
|
-
def expire_all_backlogs!
|
|
116
|
-
|
|
120
|
+
def expire_all_backlogs!(site_id: nil)
|
|
121
|
+
if site_id
|
|
122
|
+
hold { |conn| exec_prepared(conn, 'expire_site_backlogs', ["#{MessageBus::Implementation::ENCODE_SITE_TOKEN}#{site_id}"]) }
|
|
123
|
+
nil
|
|
124
|
+
else
|
|
125
|
+
reset!
|
|
126
|
+
end
|
|
117
127
|
end
|
|
118
128
|
|
|
119
129
|
def max_id(channel = nil)
|
|
@@ -153,35 +163,55 @@ module MessageBus
|
|
|
153
163
|
|
|
154
164
|
def subscribe(channel)
|
|
155
165
|
obj = Object.new
|
|
156
|
-
sync
|
|
166
|
+
sync do
|
|
167
|
+
@listening_on[channel] = obj
|
|
168
|
+
@reconnect_requested = false
|
|
169
|
+
end
|
|
157
170
|
listener = Listener.new
|
|
158
171
|
yield listener
|
|
159
172
|
|
|
160
173
|
conn = @subscribe_connection = raw_pg_connection
|
|
161
|
-
conn.exec "LISTEN #{channel}"
|
|
162
|
-
listener.do_sub.call
|
|
163
|
-
while listening_on?(channel, obj)
|
|
164
|
-
conn.wait_for_notify(10) do |_, _, payload|
|
|
165
|
-
break unless listening_on?(channel, obj)
|
|
166
174
|
|
|
167
|
-
|
|
175
|
+
begin
|
|
176
|
+
conn.exec "LISTEN #{channel}"
|
|
177
|
+
listener.do_sub.call
|
|
178
|
+
while listening_on?(channel, obj)
|
|
179
|
+
raise ReconnectRequested if reconnect_requested?
|
|
180
|
+
|
|
181
|
+
conn.wait_for_notify(10) do |_, _, payload|
|
|
182
|
+
break unless listening_on?(channel, obj)
|
|
183
|
+
|
|
184
|
+
listener.do_message.call(nil, payload)
|
|
185
|
+
end
|
|
168
186
|
end
|
|
169
|
-
|
|
170
|
-
listener.do_unsub.call
|
|
187
|
+
listener.do_unsub.call
|
|
171
188
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
189
|
+
conn.exec "UNLISTEN #{channel}"
|
|
190
|
+
nil
|
|
191
|
+
ensure
|
|
192
|
+
@subscribe_connection&.close
|
|
193
|
+
@subscribe_connection = nil
|
|
194
|
+
end
|
|
177
195
|
end
|
|
178
196
|
|
|
179
197
|
def unsubscribe
|
|
180
198
|
sync { @listening_on.clear }
|
|
181
199
|
end
|
|
182
200
|
|
|
201
|
+
# Sets a flag rather than closing the connection directly, since only
|
|
202
|
+
# the owning thread may touch a PGconn. Picked up by {#subscribe}'s
|
|
203
|
+
# wait_for_notify loop, which raises to trigger the retry in
|
|
204
|
+
# {Postgres#global_subscribe}.
|
|
205
|
+
def request_reconnect
|
|
206
|
+
sync { @reconnect_requested = true }
|
|
207
|
+
end
|
|
208
|
+
|
|
183
209
|
private
|
|
184
210
|
|
|
211
|
+
def reconnect_requested?
|
|
212
|
+
sync { @reconnect_requested }
|
|
213
|
+
end
|
|
214
|
+
|
|
185
215
|
def exec_prepared(conn, *a)
|
|
186
216
|
r = conn.exec_prepared(*a)
|
|
187
217
|
yield r if block_given?
|
|
@@ -244,6 +274,7 @@ module MessageBus
|
|
|
244
274
|
conn.exec 'PREPARE channel_backlog AS SELECT id, value FROM message_bus WHERE ((channel = $1) AND (id > $2)) ORDER BY id'
|
|
245
275
|
conn.exec 'PREPARE global_backlog AS SELECT id, channel, value FROM message_bus WHERE (id > $1) ORDER BY id'
|
|
246
276
|
conn.exec "PREPARE expire AS DELETE FROM message_bus WHERE added_at < CURRENT_TIMESTAMP - ($1::text || ' seconds')::interval"
|
|
277
|
+
conn.exec 'PREPARE expire_site_backlogs AS DELETE FROM message_bus WHERE right(channel, length($1)) = $1'
|
|
247
278
|
conn.exec 'PREPARE get_message AS SELECT value FROM message_bus WHERE ((channel = $1) AND (id = $2))'
|
|
248
279
|
conn.exec 'PREPARE max_channel_id AS SELECT max(id) FROM message_bus WHERE (channel = $1)'
|
|
249
280
|
conn.exec 'PREPARE max_channel_ids AS SELECT channel, max(id) FROM message_bus WHERE (channel = ANY($1)) GROUP BY channel'
|
|
@@ -299,8 +330,8 @@ module MessageBus
|
|
|
299
330
|
end
|
|
300
331
|
|
|
301
332
|
# (see Base#expire_all_backlogs!)
|
|
302
|
-
def expire_all_backlogs!
|
|
303
|
-
client.expire_all_backlogs!
|
|
333
|
+
def expire_all_backlogs!(site_id: nil)
|
|
334
|
+
client.expire_all_backlogs!(site_id: site_id)
|
|
304
335
|
end
|
|
305
336
|
|
|
306
337
|
# (see Base#publish)
|
|
@@ -378,6 +409,16 @@ module MessageBus
|
|
|
378
409
|
@subscribed = false
|
|
379
410
|
end
|
|
380
411
|
|
|
412
|
+
# (see Base#request_reconnect)
|
|
413
|
+
#
|
|
414
|
+
# Flags the subscriber thread instead of closing its connection: a
|
|
415
|
+
# PG::Connection may only be used by its owning thread. The reconnect
|
|
416
|
+
# therefore takes effect when the subscriber's `wait_for_notify` call
|
|
417
|
+
# next returns, so within 10 seconds.
|
|
418
|
+
def request_reconnect
|
|
419
|
+
client.request_reconnect
|
|
420
|
+
end
|
|
421
|
+
|
|
381
422
|
# (see Base#global_subscribe)
|
|
382
423
|
def global_subscribe(last_id = nil)
|
|
383
424
|
raise ArgumentError unless block_given?
|
|
@@ -385,15 +426,18 @@ module MessageBus
|
|
|
385
426
|
highest_id = last_id
|
|
386
427
|
|
|
387
428
|
begin
|
|
429
|
+
# Seed a cursor if we don't already have one, so that a reconnect
|
|
430
|
+
# (triggered by ReconnectRequested below) always has a replay
|
|
431
|
+
# point and never treats "no explicit last_id" as "never replay".
|
|
432
|
+
highest_id ||= client.max_id
|
|
433
|
+
|
|
388
434
|
client.subscribe(postgresql_channel_name) do |on|
|
|
389
435
|
h = {}
|
|
390
436
|
|
|
391
437
|
on.subscribe do
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
yield m
|
|
396
|
-
end
|
|
438
|
+
highest_id = process_global_backlog(highest_id) do |m|
|
|
439
|
+
h[m.global_id] = true
|
|
440
|
+
yield m
|
|
397
441
|
end
|
|
398
442
|
h = nil if h.empty?
|
|
399
443
|
@subscribed = true
|
|
@@ -410,25 +454,19 @@ module MessageBus
|
|
|
410
454
|
end
|
|
411
455
|
m = MessageBus::Message.decode m
|
|
412
456
|
|
|
413
|
-
#
|
|
414
|
-
#
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
# If already yielded during the clear backlog when subscribing,
|
|
421
|
-
# don't yield a duplicate copy.
|
|
422
|
-
unless h.delete(m.global_id)
|
|
423
|
-
h = nil if h.empty?
|
|
424
|
-
yield m
|
|
425
|
-
end
|
|
426
|
-
else
|
|
457
|
+
# If already yielded during the clear backlog when subscribing,
|
|
458
|
+
# don't yield a duplicate copy.
|
|
459
|
+
duplicate = h && h.delete(m.global_id)
|
|
460
|
+
h = nil if h&.empty?
|
|
461
|
+
|
|
462
|
+
unless duplicate
|
|
463
|
+
highest_id = m.global_id if m.global_id > highest_id
|
|
427
464
|
yield m
|
|
428
465
|
end
|
|
429
466
|
end
|
|
430
467
|
end
|
|
431
468
|
rescue => error
|
|
469
|
+
@subscribed = false
|
|
432
470
|
@config[:logger].warn "#{error} subscribe failed, reconnecting in 1 second. Call stack\n#{error.backtrace.join("\n")}"
|
|
433
471
|
sleep 1
|
|
434
472
|
retry
|
|
@@ -60,6 +60,7 @@ module MessageBus
|
|
|
60
60
|
@flush_backlog_thread = nil
|
|
61
61
|
@pub_redis = nil
|
|
62
62
|
@subscribed = false
|
|
63
|
+
@subscriber_connection = nil
|
|
63
64
|
# after 7 days inactive backlogs will be removed
|
|
64
65
|
@max_backlog_age = 604_800
|
|
65
66
|
end
|
|
@@ -82,8 +83,10 @@ module MessageBus
|
|
|
82
83
|
|
|
83
84
|
# Deletes all backlogs and their data. Does not delete ID pointers, so new publications will get IDs that continue from the last publication before the expiry. Use with extreme caution.
|
|
84
85
|
# @see Base#expire_all_backlogs!
|
|
85
|
-
def expire_all_backlogs!
|
|
86
|
-
|
|
86
|
+
def expire_all_backlogs!(site_id: nil)
|
|
87
|
+
site_suffix = escape_glob("#{MessageBus::Implementation::ENCODE_SITE_TOKEN}#{site_id}") if site_id
|
|
88
|
+
pub_redis.scan_each(match: "#{backlog_key("*")}#{site_suffix}") { |k| pub_redis.del k }
|
|
89
|
+
pub_redis.del(global_backlog_key) if !site_id
|
|
87
90
|
end
|
|
88
91
|
|
|
89
92
|
# Note, the script takes care of all expiry of keys, however
|
|
@@ -286,13 +289,18 @@ LUA
|
|
|
286
289
|
end
|
|
287
290
|
|
|
288
291
|
begin
|
|
289
|
-
global_redis = new_redis_connection
|
|
292
|
+
global_redis = @subscriber_connection = new_redis_connection
|
|
290
293
|
|
|
291
|
-
|
|
294
|
+
# Seed a cursor if we don't already have one, so that a reconnect
|
|
295
|
+
# (triggered by request_reconnect below) always has a replay point
|
|
296
|
+
# and never treats "no explicit last_id" as "never replay".
|
|
297
|
+
highest_id ||= current_global_id
|
|
298
|
+
|
|
299
|
+
clear_backlog.call(&blk)
|
|
292
300
|
|
|
293
301
|
global_redis.subscribe(redis_channel_name) do |on|
|
|
294
302
|
on.subscribe do
|
|
295
|
-
clear_backlog.call(&blk)
|
|
303
|
+
clear_backlog.call(&blk)
|
|
296
304
|
@subscribed = true
|
|
297
305
|
end
|
|
298
306
|
|
|
@@ -312,7 +320,7 @@ LUA
|
|
|
312
320
|
# 2. message came in the incorrect order COMPLICATED, wait a tiny bit and clear backlog
|
|
313
321
|
# 3. message came in the incorrect order and is lowest than current highest id, reset
|
|
314
322
|
|
|
315
|
-
if
|
|
323
|
+
if m.global_id == highest_id + 1
|
|
316
324
|
highest_id = m.global_id
|
|
317
325
|
yield m
|
|
318
326
|
else
|
|
@@ -321,15 +329,22 @@ LUA
|
|
|
321
329
|
end
|
|
322
330
|
end
|
|
323
331
|
rescue => error
|
|
332
|
+
@subscribed = false
|
|
324
333
|
@logger.warn "#{error} subscribe failed, reconnecting in 1 second. Call stack #{error.backtrace.join("\n")}"
|
|
325
334
|
sleep 1
|
|
326
335
|
global_redis&.disconnect!
|
|
327
336
|
retry
|
|
328
337
|
ensure
|
|
329
338
|
global_redis&.disconnect!
|
|
339
|
+
@subscriber_connection = nil
|
|
330
340
|
end
|
|
331
341
|
end
|
|
332
342
|
|
|
343
|
+
# (see Base#request_reconnect)
|
|
344
|
+
def request_reconnect
|
|
345
|
+
@subscriber_connection&.disconnect!
|
|
346
|
+
end
|
|
347
|
+
|
|
333
348
|
private
|
|
334
349
|
|
|
335
350
|
def new_redis_connection
|
|
@@ -346,6 +361,10 @@ LUA
|
|
|
346
361
|
"_message_bus_#{db}"
|
|
347
362
|
end
|
|
348
363
|
|
|
364
|
+
def escape_glob(str)
|
|
365
|
+
str.gsub(/[*?\[\]\\]/) { |c| "\\#{c}" }
|
|
366
|
+
end
|
|
367
|
+
|
|
349
368
|
def backlog_key(channel)
|
|
350
369
|
"__mb_backlog_n_#{channel}"
|
|
351
370
|
end
|
|
@@ -362,8 +381,12 @@ LUA
|
|
|
362
381
|
"__mb_global_backlog_n"
|
|
363
382
|
end
|
|
364
383
|
|
|
384
|
+
def current_global_id
|
|
385
|
+
pub_redis.get(global_id_key).to_i
|
|
386
|
+
end
|
|
387
|
+
|
|
365
388
|
def process_global_backlog(highest_id, raise_error)
|
|
366
|
-
highest_id = 0 if highest_id >
|
|
389
|
+
highest_id = 0 if highest_id > current_global_id
|
|
367
390
|
|
|
368
391
|
global_backlog(highest_id).each do |old|
|
|
369
392
|
if highest_id + 1 == old.global_id
|
data/lib/message_bus/version.rb
CHANGED
data/lib/message_bus.rb
CHANGED
|
@@ -591,7 +591,9 @@ module MessageBus::Implementation
|
|
|
591
591
|
|
|
592
592
|
# @param [Integer] interval the keepalive interval in seconds.
|
|
593
593
|
# Set to 0 to disable; anything higher and a keepalive will run every N
|
|
594
|
-
# seconds. If
|
|
594
|
+
# seconds. If a keepalive message is not received within 3 intervals, a
|
|
595
|
+
# warning is logged and the backend subscriber is asked to reconnect
|
|
596
|
+
# (see {Backends::Base#request_reconnect}).
|
|
595
597
|
def keepalive_interval=(interval)
|
|
596
598
|
configure(keepalive_interval: interval)
|
|
597
599
|
end
|
|
@@ -748,13 +750,26 @@ module MessageBus::Implementation
|
|
|
748
750
|
|
|
749
751
|
blk = proc do
|
|
750
752
|
if !@destroyed && thread.alive? && keepalive_interval > MIN_KEEPALIVE
|
|
753
|
+
begin
|
|
754
|
+
publish("/__mb_keepalive__/", Process.pid, user_ids: [-1])
|
|
751
755
|
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
+
if (Time.now - (@last_message || Time.now)) > keepalive_interval * 3
|
|
757
|
+
logger.warn "Global messages on #{Process.pid} timed out, message bus is no longer functioning correctly"
|
|
758
|
+
|
|
759
|
+
# Reset first so a slow reconnect isn't re-triggered on the next tick.
|
|
760
|
+
@last_message = Time.now
|
|
756
761
|
|
|
757
|
-
|
|
762
|
+
begin
|
|
763
|
+
backend_instance.request_reconnect
|
|
764
|
+
rescue => e
|
|
765
|
+
logger.warn "Failed to request subscriber reconnect on #{Process.pid}: #{e}"
|
|
766
|
+
end
|
|
767
|
+
end
|
|
768
|
+
ensure
|
|
769
|
+
# This proc is the only thing that re-schedules itself; an exception
|
|
770
|
+
# escaping here would disable the watchdog permanently.
|
|
771
|
+
timer.queue(keepalive_interval, &blk) if keepalive_interval > MIN_KEEPALIVE
|
|
772
|
+
end
|
|
758
773
|
end
|
|
759
774
|
end
|
|
760
775
|
|
data/message_bus.gemspec
CHANGED
|
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
|
|
|
14
14
|
gem.name = "message_bus"
|
|
15
15
|
gem.require_paths = ["lib"]
|
|
16
16
|
gem.version = MessageBus::VERSION
|
|
17
|
-
gem.required_ruby_version = ">= 3.
|
|
17
|
+
gem.required_ruby_version = ">= 3.3.0"
|
|
18
18
|
|
|
19
19
|
gem.add_runtime_dependency 'rack', '> 2', '< 4'
|
|
20
20
|
gem.add_runtime_dependency 'logger'
|
|
@@ -24,9 +24,10 @@ Gem::Specification.new do |gem|
|
|
|
24
24
|
gem.add_development_dependency 'pg'
|
|
25
25
|
gem.add_development_dependency 'concurrent-ruby' # for distributed-cache
|
|
26
26
|
|
|
27
|
-
gem.add_development_dependency 'minitest'
|
|
27
|
+
gem.add_development_dependency 'minitest', '< 6'
|
|
28
28
|
gem.add_development_dependency 'minitest-hooks'
|
|
29
29
|
gem.add_development_dependency 'minitest-global_expectations'
|
|
30
|
+
gem.add_development_dependency 'cgi' # Rack 2 tests on Ruby 4+
|
|
30
31
|
gem.add_development_dependency 'rake'
|
|
31
32
|
gem.add_development_dependency 'http_parser.rb'
|
|
32
33
|
gem.add_development_dependency 'thin'
|
data/spec/helpers.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'logger'
|
|
4
4
|
require 'method_source'
|
|
5
|
+
require 'stringio'
|
|
5
6
|
|
|
6
7
|
def wait_for(timeout_milliseconds = 2000, &blk)
|
|
7
8
|
timeout = timeout_milliseconds / 1000.0
|
|
@@ -23,7 +24,7 @@ def test_config_for_backend(backend)
|
|
|
23
24
|
|
|
24
25
|
case backend
|
|
25
26
|
when :redis
|
|
26
|
-
config[:
|
|
27
|
+
config[:redis_config] = { url: ENV['REDISURL'] }
|
|
27
28
|
when :postgres
|
|
28
29
|
config[:backend_options] = {
|
|
29
30
|
host: ENV['PGHOST'],
|
|
@@ -34,3 +35,62 @@ def test_config_for_backend(backend)
|
|
|
34
35
|
end
|
|
35
36
|
config
|
|
36
37
|
end
|
|
38
|
+
|
|
39
|
+
# Captures logger output regardless of where each backend stashes its
|
|
40
|
+
# logger instance, so shared reconnect-continuity tests can detect the
|
|
41
|
+
# "subscribe failed, reconnecting in 1 second" line that both network
|
|
42
|
+
# backends emit right before their retry sleep.
|
|
43
|
+
def capture_backend_log(bus)
|
|
44
|
+
log_output = StringIO.new
|
|
45
|
+
logger = Logger.new(log_output)
|
|
46
|
+
|
|
47
|
+
case CURRENT_BACKEND
|
|
48
|
+
when :redis
|
|
49
|
+
bus.instance_variable_set(:@logger, logger)
|
|
50
|
+
when :postgres
|
|
51
|
+
bus.instance_variable_get(:@config)[:logger] = logger
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
log_output
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Postgres polls wait_for_notify(10) by default; shorten that only for
|
|
58
|
+
# Postgres so reconnect-continuity assertions run quickly without
|
|
59
|
+
# affecting Redis, which reacts to disconnect immediately.
|
|
60
|
+
def speed_up_reconnect_detection(bus)
|
|
61
|
+
instrument_postgres_client(bus) if CURRENT_BACKEND == :postgres
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Records LISTEN attempts and which thread closed each subscriber
|
|
65
|
+
# connection (pool connections are excluded), and shortens the notify poll
|
|
66
|
+
# so a reconnect request is picked up quickly.
|
|
67
|
+
def instrument_postgres_client(backend)
|
|
68
|
+
recorded = { subscribes: [], close_threads: [] }
|
|
69
|
+
client = backend.send(:client)
|
|
70
|
+
real_subscribe = client.method(:subscribe)
|
|
71
|
+
real_raw_pg_connection = client.method(:raw_pg_connection)
|
|
72
|
+
|
|
73
|
+
client.define_singleton_method(:subscribe) do |channel, &blk|
|
|
74
|
+
recorded[:subscribes] << channel
|
|
75
|
+
real_subscribe.call(channel, &blk)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
client.define_singleton_method(:raw_pg_connection) do
|
|
79
|
+
conn = real_raw_pg_connection.call
|
|
80
|
+
subscriber = false
|
|
81
|
+
|
|
82
|
+
conn.define_singleton_method(:wait_for_notify) do |_timeout = nil, &blk|
|
|
83
|
+
subscriber = true
|
|
84
|
+
super(0.05, &blk)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
conn.define_singleton_method(:close) do
|
|
88
|
+
recorded[:close_threads] << Thread.current if subscriber
|
|
89
|
+
super()
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
conn
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
recorded
|
|
96
|
+
end
|
|
@@ -17,6 +17,10 @@ describe BACKEND_CLASS do
|
|
|
17
17
|
it "has the same public methods as the base class" do
|
|
18
18
|
@bus.public_methods.sort.must_equal MessageBus::Backends::Base.new(test_config_for_backend(CURRENT_BACKEND)).public_methods.sort
|
|
19
19
|
end
|
|
20
|
+
|
|
21
|
+
it "treats request_reconnect as an optional no-op rather than an error" do
|
|
22
|
+
MessageBus::Backends::Base.new.request_reconnect.must_be_nil
|
|
23
|
+
end
|
|
20
24
|
end
|
|
21
25
|
|
|
22
26
|
it "should be able to access the backlog" do
|
|
@@ -29,6 +33,32 @@ describe BACKEND_CLASS do
|
|
|
29
33
|
]
|
|
30
34
|
end
|
|
31
35
|
|
|
36
|
+
it "can expire all backlogs" do
|
|
37
|
+
@bus.publish "/foo", "aa"
|
|
38
|
+
@bus.publish "/foo$|$one", "bb"
|
|
39
|
+
|
|
40
|
+
@bus.expire_all_backlogs!
|
|
41
|
+
|
|
42
|
+
@bus.backlog("/foo", 0).to_a.must_equal []
|
|
43
|
+
@bus.backlog("/foo$|$one", 0).to_a.must_equal []
|
|
44
|
+
@bus.global_backlog(0).to_a.must_equal []
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
it "can expire the backlogs of a single site" do
|
|
48
|
+
@bus.publish "/foo$|$one", "aa"
|
|
49
|
+
@bus.publish "/foo$|$two", "bb"
|
|
50
|
+
@bus.publish "/foo$|$*", "cc"
|
|
51
|
+
@bus.publish "/foo", "dd"
|
|
52
|
+
|
|
53
|
+
@bus.expire_all_backlogs!(site_id: "one")
|
|
54
|
+
@bus.expire_all_backlogs!(site_id: "t*")
|
|
55
|
+
|
|
56
|
+
@bus.backlog("/foo$|$one", 0).to_a.must_equal []
|
|
57
|
+
@bus.backlog("/foo$|$two", 0).map(&:data).must_equal ["bb"]
|
|
58
|
+
@bus.backlog("/foo$|$*", 0).map(&:data).must_equal ["cc"]
|
|
59
|
+
@bus.backlog("/foo", 0).map(&:data).must_equal ["dd"]
|
|
60
|
+
end
|
|
61
|
+
|
|
32
62
|
it "should initialize with max_backlog_size" do
|
|
33
63
|
BACKEND_CLASS.new({}, 2000).max_backlog_size.must_equal 2000
|
|
34
64
|
end
|
|
@@ -108,114 +138,30 @@ describe BACKEND_CLASS do
|
|
|
108
138
|
end
|
|
109
139
|
|
|
110
140
|
it "can set backlog age" do
|
|
111
|
-
@bus.max_backlog_age =
|
|
112
|
-
|
|
113
|
-
expected_backlog_size = 0
|
|
114
|
-
|
|
115
|
-
# Start at time = 0s
|
|
141
|
+
@bus.max_backlog_age = 0
|
|
116
142
|
@bus.publish "/foo", "bar"
|
|
117
|
-
expected_backlog_size += 1
|
|
118
|
-
|
|
119
|
-
@bus.global_backlog.length.must_equal expected_backlog_size
|
|
120
|
-
@bus.backlog("/foo", 0).length.must_equal expected_backlog_size
|
|
121
|
-
|
|
122
|
-
sleep 1.25 # Should now be at time =~ 1.25s. Our backlog should have expired by now.
|
|
123
|
-
expected_backlog_size = 0
|
|
124
|
-
|
|
125
|
-
case CURRENT_BACKEND
|
|
126
|
-
when :postgres
|
|
127
|
-
# Force triggering backlog expiry: postgres backend doesn't expire backlogs on a timer, but at publication time.
|
|
128
|
-
@bus.global_backlog.length.wont_equal expected_backlog_size
|
|
129
|
-
@bus.backlog("/foo", 0).length.wont_equal expected_backlog_size
|
|
130
|
-
@bus.publish "/foo", "baz"
|
|
131
|
-
expected_backlog_size += 1
|
|
132
|
-
end
|
|
133
143
|
|
|
134
|
-
#
|
|
135
|
-
|
|
136
|
-
|
|
144
|
+
# Redis and PostgreSQL expire synchronously during publication. The memory
|
|
145
|
+
# backend normally expires on a timer, so invoke the same expiry operation
|
|
146
|
+
# directly rather than making the test wait for that timer to run.
|
|
147
|
+
@bus.send(:client).expire if CURRENT_BACKEND == :memory
|
|
137
148
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
@bus.publish "/foo", "baz" # Publish something else before another expiry
|
|
141
|
-
expected_backlog_size += 1
|
|
142
|
-
|
|
143
|
-
sleep 0.75 # Should now be at time =~ 2.75s
|
|
144
|
-
# Our oldest message is now 1.5s old, but we didn't cease publishing for a period of 1s at a time, so we should not have expired the backlog.
|
|
145
|
-
|
|
146
|
-
@bus.publish "/foo", "baz" # Publish something else to ward off another expiry
|
|
147
|
-
expected_backlog_size += 1
|
|
148
|
-
|
|
149
|
-
case CURRENT_BACKEND
|
|
150
|
-
when :postgres
|
|
151
|
-
# Postgres expires individual messages that have lived longer than the TTL, not whole backlogs
|
|
152
|
-
expected_backlog_size -= 1
|
|
153
|
-
else
|
|
154
|
-
# Assert that the backlog did not expire, and has all of our publications since the last expiry.
|
|
155
|
-
end
|
|
156
|
-
@bus.global_backlog.length.must_equal expected_backlog_size
|
|
157
|
-
@bus.backlog("/foo", 0).length.must_equal expected_backlog_size
|
|
149
|
+
@bus.global_backlog.must_be_empty
|
|
150
|
+
@bus.backlog("/foo", 0).must_be_empty
|
|
158
151
|
end
|
|
159
152
|
|
|
160
153
|
it "can set backlog age on publish" do
|
|
161
154
|
@bus.max_backlog_age = 100
|
|
162
|
-
|
|
163
|
-
expected_backlog_size = 0
|
|
164
|
-
|
|
165
155
|
initial_id = @bus.last_id("/foo")
|
|
166
156
|
|
|
167
|
-
|
|
168
|
-
@bus.
|
|
169
|
-
expected_backlog_size += 1
|
|
170
|
-
|
|
171
|
-
@bus.global_backlog.length.must_equal expected_backlog_size
|
|
172
|
-
@bus.backlog("/foo", 0).length.must_equal expected_backlog_size
|
|
173
|
-
|
|
174
|
-
sleep 1.25 # Should now be at time =~ 1.25s. Our backlog should have expired by now.
|
|
175
|
-
expected_backlog_size = 0
|
|
176
|
-
|
|
177
|
-
case CURRENT_BACKEND
|
|
178
|
-
when :postgres
|
|
179
|
-
# Force triggering backlog expiry: postgres backend doesn't expire backlogs on a timer, but at publication time.
|
|
180
|
-
@bus.global_backlog.length.wont_equal expected_backlog_size
|
|
181
|
-
@bus.backlog("/foo", 0).length.wont_equal expected_backlog_size
|
|
182
|
-
@bus.publish "/foo", "baz", max_backlog_age: 1
|
|
183
|
-
expected_backlog_size += 1
|
|
184
|
-
end
|
|
185
|
-
|
|
186
|
-
# Assert that the backlog did expire, and now has only the new publication in it.
|
|
187
|
-
@bus.global_backlog.length.must_equal expected_backlog_size
|
|
188
|
-
@bus.backlog("/foo", 0).length.must_equal expected_backlog_size
|
|
189
|
-
|
|
190
|
-
# for the time being we can give pg a pass here
|
|
191
|
-
# TODO: make the implementation here consistent
|
|
192
|
-
if CURRENT_BACKEND != :postgres
|
|
193
|
-
# ids are not opaque we expect them to be reset on our channel if it
|
|
194
|
-
# got cleared due to an expire, the reason for this is cause we will leak entries due to tracking
|
|
195
|
-
# this in turn can bloat storage for the backend
|
|
196
|
-
@bus.last_id("/foo").must_equal initial_id
|
|
197
|
-
end
|
|
198
|
-
|
|
199
|
-
sleep 0.75 # Should now be at time =~ 2s
|
|
200
|
-
|
|
201
|
-
@bus.publish "/foo", "baz", max_backlog_age: 1 # Publish something else before another expiry
|
|
202
|
-
expected_backlog_size += 1
|
|
157
|
+
@bus.publish "/foo", "bar", max_backlog_age: 0
|
|
158
|
+
@bus.send(:client).expire if CURRENT_BACKEND == :memory
|
|
203
159
|
|
|
204
|
-
|
|
205
|
-
|
|
160
|
+
@bus.global_backlog.must_be_empty
|
|
161
|
+
@bus.backlog("/foo", 0).must_be_empty
|
|
206
162
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
case CURRENT_BACKEND
|
|
211
|
-
when :postgres
|
|
212
|
-
# Postgres expires individual messages that have lived longer than the TTL, not whole backlogs
|
|
213
|
-
expected_backlog_size -= 1
|
|
214
|
-
else
|
|
215
|
-
# Assert that the backlog did not expire, and has all of our publications since the last expiry.
|
|
216
|
-
end
|
|
217
|
-
@bus.global_backlog.length.must_equal expected_backlog_size
|
|
218
|
-
@bus.backlog("/foo", 0).length.must_equal expected_backlog_size
|
|
163
|
+
# The memory and Redis backends discard channel ID state on expiry.
|
|
164
|
+
@bus.last_id("/foo").must_equal initial_id if CURRENT_BACKEND != :postgres
|
|
219
165
|
end
|
|
220
166
|
|
|
221
167
|
it "can set backlog size on publish" do
|
|
@@ -392,4 +338,277 @@ describe BACKEND_CLASS do
|
|
|
392
338
|
got.map { |m| m.data }.must_equal ["12"]
|
|
393
339
|
end
|
|
394
340
|
|
|
341
|
+
it "tolerates request_reconnect when nothing is subscribed" do
|
|
342
|
+
@bus.request_reconnect
|
|
343
|
+
|
|
344
|
+
@bus.publish("/reconnect-noop", "bar")
|
|
345
|
+
|
|
346
|
+
@bus.backlog("/reconnect-noop", 0).map(&:data).must_equal ["bar"]
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
it "retries global_subscribe after a redis subscribe failure" do
|
|
350
|
+
test_only(:redis)
|
|
351
|
+
|
|
352
|
+
log_output = StringIO.new
|
|
353
|
+
@bus.instance_variable_set(:@logger, Logger.new(log_output))
|
|
354
|
+
|
|
355
|
+
fail_first_subscribe = true
|
|
356
|
+
real_new_redis_connection = @bus.method(:new_redis_connection)
|
|
357
|
+
|
|
358
|
+
@bus.define_singleton_method(:new_redis_connection) do
|
|
359
|
+
redis = real_new_redis_connection.call
|
|
360
|
+
|
|
361
|
+
if fail_first_subscribe
|
|
362
|
+
fail_first_subscribe = false
|
|
363
|
+
redis.define_singleton_method(:subscribe) do |_channel, &_blk|
|
|
364
|
+
raise IOError, "forced subscribe failure"
|
|
365
|
+
end
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
redis
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
got = []
|
|
372
|
+
|
|
373
|
+
t = Thread.new do
|
|
374
|
+
@bus.global_subscribe(0) do |msg|
|
|
375
|
+
got << msg
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
wait_for(4000) { log_output.string.include?("forced subscribe failure") }
|
|
380
|
+
wait_for(5000) { @bus.subscribed }
|
|
381
|
+
|
|
382
|
+
@bus.publish("/redis-retry", "delivered-after-retry")
|
|
383
|
+
wait_for(3000) { got.any? { |m| m.data == "delivered-after-retry" } }
|
|
384
|
+
|
|
385
|
+
@bus.global_unsubscribe
|
|
386
|
+
t.join(2)
|
|
387
|
+
t.kill if t.alive?
|
|
388
|
+
|
|
389
|
+
log_output.string.must_include "subscribe failed, reconnecting in 1 second"
|
|
390
|
+
got.map(&:data).must_include "delivered-after-retry"
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
it "retries global_subscribe after request_reconnect disconnects redis" do
|
|
394
|
+
test_only(:redis)
|
|
395
|
+
|
|
396
|
+
log_output = StringIO.new
|
|
397
|
+
@bus.instance_variable_set(:@logger, Logger.new(log_output))
|
|
398
|
+
subscribe_attempts = 0
|
|
399
|
+
real_new_redis_connection = @bus.method(:new_redis_connection)
|
|
400
|
+
|
|
401
|
+
@bus.define_singleton_method(:new_redis_connection) do
|
|
402
|
+
redis = real_new_redis_connection.call
|
|
403
|
+
real_subscribe = redis.method(:subscribe)
|
|
404
|
+
redis.define_singleton_method(:subscribe) do |*args, &blk|
|
|
405
|
+
subscribe_attempts += 1
|
|
406
|
+
real_subscribe.call(*args, &blk)
|
|
407
|
+
end
|
|
408
|
+
redis
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
got = []
|
|
412
|
+
|
|
413
|
+
t = Thread.new do
|
|
414
|
+
@bus.global_subscribe(0) do |msg|
|
|
415
|
+
got << msg
|
|
416
|
+
end
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
wait_for(5000) { @bus.subscribed }
|
|
420
|
+
wait_for(5000) { subscribe_attempts >= 1 }
|
|
421
|
+
|
|
422
|
+
@bus.publish("/redis-reconnect", "before-reconnect")
|
|
423
|
+
wait_for(3000) { got.any? { |m| m.data == "before-reconnect" } }
|
|
424
|
+
|
|
425
|
+
@bus.request_reconnect
|
|
426
|
+
|
|
427
|
+
wait_for(7000) { log_output.string.include?("subscribe failed, reconnecting in 1 second") }
|
|
428
|
+
wait_for(7000) { subscribe_attempts >= 2 }
|
|
429
|
+
|
|
430
|
+
@bus.publish("/redis-reconnect", "after-reconnect")
|
|
431
|
+
wait_for(5000) { got.any? { |m| m.data == "after-reconnect" } }
|
|
432
|
+
|
|
433
|
+
@bus.global_unsubscribe
|
|
434
|
+
t.join(2)
|
|
435
|
+
t.kill if t.alive?
|
|
436
|
+
|
|
437
|
+
got.map(&:data).must_include "after-reconnect"
|
|
438
|
+
subscribe_attempts.must_be :>=, 2
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
it "replays messages published during a reconnect window without loss or duplication" do
|
|
442
|
+
test_never(:memory)
|
|
443
|
+
|
|
444
|
+
log_output = capture_backend_log(@bus)
|
|
445
|
+
speed_up_reconnect_detection(@bus)
|
|
446
|
+
|
|
447
|
+
got = []
|
|
448
|
+
|
|
449
|
+
t = Thread.new do
|
|
450
|
+
@bus.global_subscribe do |msg|
|
|
451
|
+
got << msg
|
|
452
|
+
end
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
wait_for(5000) { @bus.subscribed }
|
|
456
|
+
|
|
457
|
+
@bus.publish("/reconnect-continuity", "before")
|
|
458
|
+
wait_for(3000) { got.any? { |m| m.data == "before" } }
|
|
459
|
+
|
|
460
|
+
@bus.request_reconnect
|
|
461
|
+
wait_for(7000) { log_output.string.include?("subscribe failed, reconnecting in 1 second") }
|
|
462
|
+
|
|
463
|
+
@bus.publish("/reconnect-continuity", "during")
|
|
464
|
+
|
|
465
|
+
wait_for(7000) { @bus.subscribed }
|
|
466
|
+
|
|
467
|
+
@bus.publish("/reconnect-continuity", "after")
|
|
468
|
+
wait_for(5000) { got.any? { |m| m.data == "after" } }
|
|
469
|
+
|
|
470
|
+
@bus.global_unsubscribe
|
|
471
|
+
t.join(3)
|
|
472
|
+
t.kill if t.alive?
|
|
473
|
+
|
|
474
|
+
got.map(&:data).must_equal ["before", "during", "after"]
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
it "replays a message published during a reconnect window that opened before any message was received" do
|
|
478
|
+
test_never(:memory)
|
|
479
|
+
|
|
480
|
+
log_output = capture_backend_log(@bus)
|
|
481
|
+
speed_up_reconnect_detection(@bus)
|
|
482
|
+
|
|
483
|
+
got = []
|
|
484
|
+
|
|
485
|
+
t = Thread.new do
|
|
486
|
+
@bus.global_subscribe do |msg|
|
|
487
|
+
got << msg
|
|
488
|
+
end
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
wait_for(5000) { @bus.subscribed }
|
|
492
|
+
|
|
493
|
+
# Request reconnect before any live message has been received, so the
|
|
494
|
+
# backend has never had a chance to derive a cursor from a message: it
|
|
495
|
+
# must fall back to the cursor seeded at subscription start.
|
|
496
|
+
@bus.request_reconnect
|
|
497
|
+
wait_for(7000) { log_output.string.include?("subscribe failed, reconnecting in 1 second") }
|
|
498
|
+
|
|
499
|
+
@bus.publish("/reconnect-continuity-cold", "during")
|
|
500
|
+
|
|
501
|
+
wait_for(7000) { @bus.subscribed }
|
|
502
|
+
|
|
503
|
+
@bus.publish("/reconnect-continuity-cold", "after")
|
|
504
|
+
wait_for(5000) { got.any? { |m| m.data == "after" } }
|
|
505
|
+
|
|
506
|
+
@bus.global_unsubscribe
|
|
507
|
+
t.join(3)
|
|
508
|
+
t.kill if t.alive?
|
|
509
|
+
|
|
510
|
+
got.map(&:data).must_equal ["during", "after"]
|
|
511
|
+
end
|
|
512
|
+
|
|
513
|
+
it "reports subscribed as false during a reconnect window and true once replaced" do
|
|
514
|
+
test_never(:memory)
|
|
515
|
+
|
|
516
|
+
log_output = capture_backend_log(@bus)
|
|
517
|
+
speed_up_reconnect_detection(@bus)
|
|
518
|
+
|
|
519
|
+
got = []
|
|
520
|
+
|
|
521
|
+
t = Thread.new do
|
|
522
|
+
@bus.global_subscribe do |msg|
|
|
523
|
+
got << msg
|
|
524
|
+
end
|
|
525
|
+
end
|
|
526
|
+
|
|
527
|
+
wait_for(5000) { @bus.subscribed }
|
|
528
|
+
|
|
529
|
+
@bus.request_reconnect
|
|
530
|
+
wait_for(7000) { log_output.string.include?("subscribe failed, reconnecting in 1 second") }
|
|
531
|
+
|
|
532
|
+
@bus.subscribed.must_equal false
|
|
533
|
+
|
|
534
|
+
wait_for(7000) { @bus.subscribed }
|
|
535
|
+
@bus.subscribed.must_equal true
|
|
536
|
+
|
|
537
|
+
@bus.global_unsubscribe
|
|
538
|
+
t.join(3)
|
|
539
|
+
t.kill if t.alive?
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
it "reconnects the postgres subscriber from its own thread" do
|
|
543
|
+
test_only(:postgres)
|
|
544
|
+
|
|
545
|
+
log_output = StringIO.new
|
|
546
|
+
@bus.instance_variable_get(:@config)[:logger] = Logger.new(log_output)
|
|
547
|
+
recorded = instrument_postgres_client(@bus)
|
|
548
|
+
|
|
549
|
+
got = []
|
|
550
|
+
|
|
551
|
+
t = Thread.new do
|
|
552
|
+
@bus.global_subscribe(0) do |msg|
|
|
553
|
+
got << msg
|
|
554
|
+
end
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
wait_for(5000) { @bus.subscribed }
|
|
558
|
+
|
|
559
|
+
@bus.publish("/pg-reconnect", "before-reconnect")
|
|
560
|
+
wait_for(3000) { got.any? { |m| m.data == "before-reconnect" } }
|
|
561
|
+
|
|
562
|
+
@bus.request_reconnect
|
|
563
|
+
wait_for(7000) { recorded[:subscribes].length >= 2 }
|
|
564
|
+
|
|
565
|
+
# ReconnectRequested is raised before on.unsubscribe fires, so @subscribed
|
|
566
|
+
# must already have been reset explicitly by the rescue path.
|
|
567
|
+
@bus.subscribed.must_equal false
|
|
568
|
+
|
|
569
|
+
@bus.publish("/pg-reconnect", "during-reconnect")
|
|
570
|
+
|
|
571
|
+
wait_for(5000) { @bus.subscribed }
|
|
572
|
+
|
|
573
|
+
@bus.publish("/pg-reconnect", "after-reconnect")
|
|
574
|
+
wait_for(5000) { got.any? { |m| m.data == "after-reconnect" } }
|
|
575
|
+
|
|
576
|
+
@bus.global_unsubscribe
|
|
577
|
+
t.join(3)
|
|
578
|
+
t.kill if t.alive?
|
|
579
|
+
|
|
580
|
+
# Exact order, no duplicates, no loss: the cursor must be advanced by
|
|
581
|
+
# every accepted live message and carried across the reconnect.
|
|
582
|
+
got.map(&:data).must_equal ["before-reconnect", "during-reconnect", "after-reconnect"]
|
|
583
|
+
log_output.string.must_include "subscribe failed, reconnecting in 1 second"
|
|
584
|
+
|
|
585
|
+
# Regression check: only the subscriber thread may close its own PGconn.
|
|
586
|
+
recorded[:close_threads].uniq.must_equal [t]
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
it "ignores a postgres reconnect request made while nothing was subscribed" do
|
|
590
|
+
test_only(:postgres)
|
|
591
|
+
|
|
592
|
+
recorded = instrument_postgres_client(@bus)
|
|
593
|
+
@bus.request_reconnect
|
|
594
|
+
|
|
595
|
+
got = []
|
|
596
|
+
|
|
597
|
+
t = Thread.new do
|
|
598
|
+
@bus.global_subscribe(0) do |msg|
|
|
599
|
+
got << msg
|
|
600
|
+
end
|
|
601
|
+
end
|
|
602
|
+
|
|
603
|
+
wait_for(5000) { @bus.subscribed }
|
|
604
|
+
|
|
605
|
+
@bus.publish("/pg-stale-reconnect", "delivered")
|
|
606
|
+
wait_for(3000) { got.any? { |m| m.data == "delivered" } }
|
|
607
|
+
|
|
608
|
+
@bus.global_unsubscribe
|
|
609
|
+
t.join(3)
|
|
610
|
+
t.kill if t.alive?
|
|
611
|
+
|
|
612
|
+
recorded[:subscribes].length.must_equal 1
|
|
613
|
+
end
|
|
395
614
|
end
|
|
@@ -99,13 +99,14 @@ describe MessageBus::Rack::Middleware do
|
|
|
99
99
|
t.join
|
|
100
100
|
end
|
|
101
101
|
|
|
102
|
-
it "should timeout within its
|
|
102
|
+
it "should timeout within its allotted slot" do
|
|
103
103
|
begin
|
|
104
104
|
@bus.long_polling_interval = 10
|
|
105
|
-
|
|
105
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
106
106
|
post "/message-bus/ABC", '/foo' => nil
|
|
107
|
-
|
|
108
|
-
|
|
107
|
+
elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000
|
|
108
|
+
# Leave enough headroom for scheduler jitter on busy CI runners.
|
|
109
|
+
elapsed_ms.must_be :<, 250
|
|
109
110
|
ensure
|
|
110
111
|
@bus.long_polling_interval = 5000
|
|
111
112
|
end
|
|
@@ -398,4 +398,110 @@ describe MessageBus do
|
|
|
398
398
|
channel.must_equal('/test')
|
|
399
399
|
end
|
|
400
400
|
end
|
|
401
|
+
|
|
402
|
+
describe "keepalive reconnect watchdog" do
|
|
403
|
+
def fast_keepalive
|
|
404
|
+
0.5
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
before do
|
|
408
|
+
@original_min_keepalive = MessageBus::Implementation::MIN_KEEPALIVE
|
|
409
|
+
MessageBus::Implementation.send(:remove_const, :MIN_KEEPALIVE)
|
|
410
|
+
MessageBus::Implementation.const_set(:MIN_KEEPALIVE, 0)
|
|
411
|
+
|
|
412
|
+
@log_output = StringIO.new
|
|
413
|
+
@bus.configure(keepalive_interval: fast_keepalive, logger: Logger.new(@log_output))
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
after do
|
|
417
|
+
MessageBus::Implementation.send(:remove_const, :MIN_KEEPALIVE)
|
|
418
|
+
MessageBus::Implementation.const_set(:MIN_KEEPALIVE, @original_min_keepalive)
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
def stall_global_subscribe(backend)
|
|
422
|
+
backend.define_singleton_method(:global_subscribe) do |_last_id = nil, &_blk|
|
|
423
|
+
@subscribed = true
|
|
424
|
+
sleep(0.001) while @subscribed
|
|
425
|
+
end
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
it "invokes request_reconnect and logs a warning when subscriber reads stall" do
|
|
429
|
+
backend = @bus.backend_instance
|
|
430
|
+
reconnect_calls = 0
|
|
431
|
+
stall_global_subscribe(backend)
|
|
432
|
+
|
|
433
|
+
backend.define_singleton_method(:request_reconnect) do
|
|
434
|
+
reconnect_calls += 1
|
|
435
|
+
@subscribed = false
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
@bus.after_fork
|
|
439
|
+
wait_for(1000) { @bus.listening? }
|
|
440
|
+
|
|
441
|
+
wait_for(4000) { reconnect_calls == 1 }
|
|
442
|
+
wait_for(4000) { @log_output.string.include?("no longer functioning correctly") }
|
|
443
|
+
|
|
444
|
+
reconnect_calls.must_equal 1
|
|
445
|
+
@log_output.string.must_include "timed out"
|
|
446
|
+
@log_output.string.must_include "no longer functioning correctly"
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
it "resets @last_message so reconnect attempts are not immediate cascades" do
|
|
450
|
+
backend = @bus.backend_instance
|
|
451
|
+
reconnect_times = []
|
|
452
|
+
stall_global_subscribe(backend)
|
|
453
|
+
|
|
454
|
+
backend.define_singleton_method(:request_reconnect) do
|
|
455
|
+
reconnect_times << Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
456
|
+
end
|
|
457
|
+
|
|
458
|
+
@bus.after_fork
|
|
459
|
+
wait_for(1000) { @bus.listening? }
|
|
460
|
+
wait_for(7000) { reconnect_times.length >= 2 }
|
|
461
|
+
|
|
462
|
+
first_gap = reconnect_times[1] - reconnect_times[0]
|
|
463
|
+
first_gap.must_be :>=, fast_keepalive * 2.8
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
it "stays armed and tries again when request_reconnect raises" do
|
|
467
|
+
backend = @bus.backend_instance
|
|
468
|
+
reconnect_calls = 0
|
|
469
|
+
stall_global_subscribe(backend)
|
|
470
|
+
|
|
471
|
+
backend.define_singleton_method(:request_reconnect) do
|
|
472
|
+
reconnect_calls += 1
|
|
473
|
+
raise IOError, "simulated connection error"
|
|
474
|
+
end
|
|
475
|
+
|
|
476
|
+
@bus.after_fork
|
|
477
|
+
wait_for(1000) { @bus.listening? }
|
|
478
|
+
|
|
479
|
+
wait_for(8000) { reconnect_calls >= 2 }
|
|
480
|
+
|
|
481
|
+
reconnect_calls.must_be :>=, 2
|
|
482
|
+
@log_output.string.must_include "Failed to request subscriber reconnect"
|
|
483
|
+
@log_output.string.must_include "simulated connection error"
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
it "stays armed when the keepalive publish raises" do
|
|
487
|
+
backend = @bus.backend_instance
|
|
488
|
+
publish_calls = 0
|
|
489
|
+
real_publish = backend.method(:publish)
|
|
490
|
+
stall_global_subscribe(backend)
|
|
491
|
+
|
|
492
|
+
backend.define_singleton_method(:publish) do |*args, &blk|
|
|
493
|
+
publish_calls += 1
|
|
494
|
+
raise IOError, "simulated publish failure" if publish_calls == 1
|
|
495
|
+
|
|
496
|
+
real_publish.call(*args, &blk)
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
@bus.after_fork
|
|
500
|
+
wait_for(1000) { @bus.listening? }
|
|
501
|
+
wait_for(5000) { publish_calls >= 3 }
|
|
502
|
+
|
|
503
|
+
publish_calls.must_be :>=, 3
|
|
504
|
+
@log_output.string.must_include "simulated publish failure"
|
|
505
|
+
end
|
|
506
|
+
end
|
|
401
507
|
end
|
data/spec/spec_helper.rb
CHANGED
|
@@ -19,7 +19,9 @@ BACKEND_CLASS = MessageBus::BACKENDS.fetch(CURRENT_BACKEND)
|
|
|
19
19
|
puts "Running with backend: #{CURRENT_BACKEND}"
|
|
20
20
|
|
|
21
21
|
def test_only(*backends)
|
|
22
|
-
|
|
22
|
+
return if backends.include?(CURRENT_BACKEND)
|
|
23
|
+
|
|
24
|
+
skip "Test doesn't apply to #{CURRENT_BACKEND}"
|
|
23
25
|
end
|
|
24
26
|
|
|
25
27
|
def test_never(*backends)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: message_bus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 5.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sam Saffron
|
|
@@ -87,6 +87,20 @@ dependencies:
|
|
|
87
87
|
version: '0'
|
|
88
88
|
- !ruby/object:Gem::Dependency
|
|
89
89
|
name: minitest
|
|
90
|
+
requirement: !ruby/object:Gem::Requirement
|
|
91
|
+
requirements:
|
|
92
|
+
- - "<"
|
|
93
|
+
- !ruby/object:Gem::Version
|
|
94
|
+
version: '6'
|
|
95
|
+
type: :development
|
|
96
|
+
prerelease: false
|
|
97
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
98
|
+
requirements:
|
|
99
|
+
- - "<"
|
|
100
|
+
- !ruby/object:Gem::Version
|
|
101
|
+
version: '6'
|
|
102
|
+
- !ruby/object:Gem::Dependency
|
|
103
|
+
name: minitest-hooks
|
|
90
104
|
requirement: !ruby/object:Gem::Requirement
|
|
91
105
|
requirements:
|
|
92
106
|
- - ">="
|
|
@@ -100,7 +114,7 @@ dependencies:
|
|
|
100
114
|
- !ruby/object:Gem::Version
|
|
101
115
|
version: '0'
|
|
102
116
|
- !ruby/object:Gem::Dependency
|
|
103
|
-
name: minitest-
|
|
117
|
+
name: minitest-global_expectations
|
|
104
118
|
requirement: !ruby/object:Gem::Requirement
|
|
105
119
|
requirements:
|
|
106
120
|
- - ">="
|
|
@@ -114,7 +128,7 @@ dependencies:
|
|
|
114
128
|
- !ruby/object:Gem::Version
|
|
115
129
|
version: '0'
|
|
116
130
|
- !ruby/object:Gem::Dependency
|
|
117
|
-
name:
|
|
131
|
+
name: cgi
|
|
118
132
|
requirement: !ruby/object:Gem::Requirement
|
|
119
133
|
requirements:
|
|
120
134
|
- - ">="
|
|
@@ -382,7 +396,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
382
396
|
requirements:
|
|
383
397
|
- - ">="
|
|
384
398
|
- !ruby/object:Gem::Version
|
|
385
|
-
version: 3.
|
|
399
|
+
version: 3.3.0
|
|
386
400
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
387
401
|
requirements:
|
|
388
402
|
- - ">="
|