comq 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/codeql.yml +4 -4
- package/.github/workflows/release.yaml +2 -2
- package/examples/streams/consumer.js +3 -3
- package/examples/streams/producer.js +2 -0
- package/features/backpressure.feature +1 -1
- package/fixplan.md +507 -0
- package/package.json +4 -7
- package/types/io.d.ts +28 -24
|
@@ -42,11 +42,11 @@ jobs:
|
|
|
42
42
|
|
|
43
43
|
steps:
|
|
44
44
|
- name: Checkout repository
|
|
45
|
-
uses: actions/checkout@
|
|
45
|
+
uses: actions/checkout@v6
|
|
46
46
|
|
|
47
47
|
# Initializes the CodeQL tools for scanning.
|
|
48
48
|
- name: Initialize CodeQL
|
|
49
|
-
uses: github/codeql-action/init@
|
|
49
|
+
uses: github/codeql-action/init@v4
|
|
50
50
|
with:
|
|
51
51
|
languages: ${{ matrix.language }}
|
|
52
52
|
# If you wish to specify custom queries, you can do so here or in a config file.
|
|
@@ -60,7 +60,7 @@ jobs:
|
|
|
60
60
|
# Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java).
|
|
61
61
|
# If this step fails, then you should remove it and run the build manually (see below)
|
|
62
62
|
- name: Autobuild
|
|
63
|
-
uses: github/codeql-action/autobuild@
|
|
63
|
+
uses: github/codeql-action/autobuild@v4
|
|
64
64
|
|
|
65
65
|
# ℹ️ Command-line programs to run using the OS shell.
|
|
66
66
|
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
|
@@ -73,6 +73,6 @@ jobs:
|
|
|
73
73
|
# ./location_of_script_within_repo/buildscript.sh
|
|
74
74
|
|
|
75
75
|
- name: Perform CodeQL Analysis
|
|
76
|
-
uses: github/codeql-action/analyze@
|
|
76
|
+
uses: github/codeql-action/analyze@v4
|
|
77
77
|
with:
|
|
78
78
|
category: "/language:${{matrix.language}}"
|
|
@@ -21,13 +21,13 @@ jobs:
|
|
|
21
21
|
|
|
22
22
|
steps:
|
|
23
23
|
- name: Setup Node
|
|
24
|
-
uses: actions/setup-node@
|
|
24
|
+
uses: actions/setup-node@v6
|
|
25
25
|
with:
|
|
26
26
|
node-version: 20
|
|
27
27
|
registry-url: 'https://registry.npmjs.org'
|
|
28
28
|
|
|
29
29
|
- name: Checkout
|
|
30
|
-
uses: actions/checkout@
|
|
30
|
+
uses: actions/checkout@v6
|
|
31
31
|
with:
|
|
32
32
|
fetch-depth: 0
|
|
33
33
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { connect } from 'comq'
|
|
2
2
|
import { url } from '../const.js'
|
|
3
3
|
|
|
4
|
-
const limit = Number(process.argv[2])
|
|
4
|
+
const limit = Number(process.argv[2]) ?? Math.round(Math.random() * 8 + 2)
|
|
5
5
|
|
|
6
6
|
let io
|
|
7
7
|
let stream
|
|
@@ -19,9 +19,9 @@ async function run () {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
async function fetch () {
|
|
22
|
-
console.log(`Fetching stream with ${limit} numbers`)
|
|
22
|
+
console.log(`Fetching stream with ${limit === 0 ? 'infinite' : limit} numbers`)
|
|
23
23
|
|
|
24
|
-
stream = await io.
|
|
24
|
+
stream = await io.request('get_numbers', { limit })
|
|
25
25
|
|
|
26
26
|
for await (const number of stream) console.log('Received:', number)
|
|
27
27
|
|
|
@@ -8,5 +8,5 @@ Feature: Back pressure handling
|
|
|
8
8
|
"""
|
|
9
9
|
() => { return Buffer.from('ok') }
|
|
10
10
|
"""
|
|
11
|
-
When I'm sending
|
|
11
|
+
When I'm sending 500kB requests to the `flood` queue at 5kHz for 0.5 seconds
|
|
12
12
|
Then back pressure was applied
|
package/fixplan.md
ADDED
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
# comq — reconnection hang fix plan
|
|
2
|
+
|
|
3
|
+
This document describes the defects identified in the single-connection
|
|
4
|
+
reconnection path (`source/connection.js`, `source/channel.js`,
|
|
5
|
+
`source/attributes/failsafe.js`) and concrete, minimally-invasive fixes for
|
|
6
|
+
each one. Problems are ordered by their likelihood of producing the observed
|
|
7
|
+
symptom: *"after RabbitMQ connection is lost, the application is sometimes
|
|
8
|
+
never recovered."*
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Problem 1 — Deadlock in `Channel.recover()` via `recall()` + `failsafe`
|
|
13
|
+
|
|
14
|
+
### Symptom
|
|
15
|
+
|
|
16
|
+
During reconnect, `Channel.recover()` rebuilds the AMQP channel and then
|
|
17
|
+
replays previously recorded `consume` / `subscribe` calls via `recall(this)`.
|
|
18
|
+
`#recovery` is resolved only *after* `recall(this)` completes:
|
|
19
|
+
|
|
20
|
+
```142:160:source/channel.js
|
|
21
|
+
async recover (connection) {
|
|
22
|
+
this.#connection = connection
|
|
23
|
+
|
|
24
|
+
await this.create()
|
|
25
|
+
|
|
26
|
+
lazy.reset(this)
|
|
27
|
+
await recall(this)
|
|
28
|
+
|
|
29
|
+
this.#unpause(INTERRUPTION)
|
|
30
|
+
|
|
31
|
+
for (const confirmation of this.#confirmations) confirmation.reject(INTERRUPTION)
|
|
32
|
+
|
|
33
|
+
// handle interruptions
|
|
34
|
+
await new Promise(resolve => setTimeout(resolve, 0))
|
|
35
|
+
|
|
36
|
+
this.#recovery.resolve()
|
|
37
|
+
this.#recovery = new Promex()
|
|
38
|
+
this.#diagnostics.emit('recover')
|
|
39
|
+
}
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The replayed methods are wrapped in `failsafe(this, this.#recover, ...)`. If
|
|
43
|
+
any replayed operation throws a *non-permanent* exception (`Channel closed`,
|
|
44
|
+
`Channel ended, no reply will be forthcoming`, or the internal
|
|
45
|
+
`INTERRUPTION`), `failsafe` calls:
|
|
46
|
+
|
|
47
|
+
```333:336:source/channel.js
|
|
48
|
+
async #recover (exception) {
|
|
49
|
+
if (permanent(exception)) return false
|
|
50
|
+
else await this.#recovery
|
|
51
|
+
}
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`failsafe` now awaits `this.#recovery`, which is the same Promex the enclosing
|
|
55
|
+
`recover()` is supposed to resolve further down. `recall(this)` cannot
|
|
56
|
+
finish, so `#recovery` is never resolved — the channel is stuck, and every
|
|
57
|
+
`send`/`publish`/`consume` parked on `failsafe → #recovery` hangs **forever**.
|
|
58
|
+
|
|
59
|
+
This reliably happens when the freshly created channel dies mid-setup, which
|
|
60
|
+
is common after a broker restart while heartbeats are still flapping, or
|
|
61
|
+
when `#assertQueue`/`#assertExchange` hits a transient amqplib "Channel
|
|
62
|
+
closed" race.
|
|
63
|
+
|
|
64
|
+
### Fix
|
|
65
|
+
|
|
66
|
+
Add a reentrancy guard so that `failsafe.#recover` refuses to wait on
|
|
67
|
+
`#recovery` while `recover()` is itself running — it must propagate instead,
|
|
68
|
+
letting `recall()` surface the failure to `Connection.#open`, which (after
|
|
69
|
+
Problem 2 is fixed) will ask `retry()` for another reopen attempt.
|
|
70
|
+
|
|
71
|
+
```js
|
|
72
|
+
// source/channel.js
|
|
73
|
+
#recovering = false
|
|
74
|
+
|
|
75
|
+
async recover (connection) {
|
|
76
|
+
this.#recovering = true
|
|
77
|
+
try {
|
|
78
|
+
this.#connection = connection
|
|
79
|
+
await this.create()
|
|
80
|
+
lazy.reset(this)
|
|
81
|
+
await recall(this)
|
|
82
|
+
this.#unpause(INTERRUPTION)
|
|
83
|
+
for (const confirmation of this.#confirmations) confirmation.reject(INTERRUPTION)
|
|
84
|
+
await new Promise(resolve => setTimeout(resolve, 0))
|
|
85
|
+
this.#recovery.resolve()
|
|
86
|
+
this.#recovery = new Promex()
|
|
87
|
+
this.#diagnostics.emit('recover')
|
|
88
|
+
} finally {
|
|
89
|
+
this.#recovering = false
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async #recover (exception) {
|
|
94
|
+
if (permanent(exception)) return false
|
|
95
|
+
if (this.#recovering) return false // never wait on #recovery from inside recover()
|
|
96
|
+
await this.#recovery
|
|
97
|
+
}
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Alternative: wrap `recall(this)` in try/catch inside `recover()`, and on
|
|
101
|
+
failure, reject the *old* `#recovery` (with a sentinel), create a fresh one,
|
|
102
|
+
and rethrow. The reentrancy guard is simpler and strictly local.
|
|
103
|
+
|
|
104
|
+
### Test
|
|
105
|
+
|
|
106
|
+
Add a unit test in `test/channel.test.js` that:
|
|
107
|
+
|
|
108
|
+
1. Creates a channel, records a `consume` call.
|
|
109
|
+
2. Triggers `recover()` but makes the mocked `channel.consume` throw a
|
|
110
|
+
`new Error('Channel closed')` on the replay.
|
|
111
|
+
3. Asserts `recover()` rejects (rather than hanging) within a short timeout.
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
## Problem 2 — `channel.recover()` errors escape the `retry()` loop in `Connection.#open`
|
|
116
|
+
|
|
117
|
+
### Symptom
|
|
118
|
+
|
|
119
|
+
Only `amqp.connect` is guarded by try/catch in `#open`; channel recovery
|
|
120
|
+
runs *after* the catch block:
|
|
121
|
+
|
|
122
|
+
```73:92:source/connection.js
|
|
123
|
+
#open = async (retry) => {
|
|
124
|
+
try {
|
|
125
|
+
this.#connection = await amqp.connect(this.#url)
|
|
126
|
+
} catch (exception) {
|
|
127
|
+
if (this.#transient(exception)) return retry
|
|
128
|
+
else throw exception
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// This prevents the process from crashing; 'close' will be emitted next.
|
|
132
|
+
// https://amqp-node.github.io/amqplib/channel_api.html#model_events
|
|
133
|
+
this.#connection.on('error', noop)
|
|
134
|
+
|
|
135
|
+
this.#connection.on('close', this.#close)
|
|
136
|
+
this.#diagnostics.emit('open')
|
|
137
|
+
|
|
138
|
+
for (const channel of this.#channels) await channel.recover(this.#connection)
|
|
139
|
+
|
|
140
|
+
this.#recovery.resolve()
|
|
141
|
+
this.#recovery = new Promex()
|
|
142
|
+
}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
If any `channel.recover()` throws (permanent topology error such as
|
|
146
|
+
`PRECONDITION_FAILED`, or the freshly opened connection dies before channel
|
|
147
|
+
creation completes, or the deadlock from Problem 1 is broken by rejection),
|
|
148
|
+
`#open` rejects → `retry()` rethrows → `open()` rejects. And `open()` was
|
|
149
|
+
called from:
|
|
150
|
+
|
|
151
|
+
```97:103:source/connection.js
|
|
152
|
+
#close = async (error) => {
|
|
153
|
+
this.#diagnostics.emit('close', error)
|
|
154
|
+
this.#connection.removeAllListeners()
|
|
155
|
+
this.#connection = undefined
|
|
156
|
+
|
|
157
|
+
if (error !== undefined) await this.open()
|
|
158
|
+
}
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
…which has no error handler — rejection becomes an `unhandledRejection`, and
|
|
162
|
+
the Connection stops trying to reconnect. Whether the broken amqplib
|
|
163
|
+
`Connection` eventually emits another `close` to re-arm the flow is not
|
|
164
|
+
guaranteed (TCP half-open, missing heartbeat). The process ends up with a
|
|
165
|
+
live event loop but no broker connectivity.
|
|
166
|
+
|
|
167
|
+
### Fix
|
|
168
|
+
|
|
169
|
+
Keep channel recovery inside the `retry()` contract: on any failure, close
|
|
170
|
+
the half-baked connection and ask `retry` to try again.
|
|
171
|
+
|
|
172
|
+
```js
|
|
173
|
+
#open = async (retry) => {
|
|
174
|
+
try {
|
|
175
|
+
this.#connection = await amqp.connect(this.#url)
|
|
176
|
+
} catch (exception) {
|
|
177
|
+
if (this.#transient(exception)) return retry
|
|
178
|
+
else throw exception
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
this.#connection.on('error', noop)
|
|
182
|
+
this.#connection.on('close', this.#close)
|
|
183
|
+
this.#diagnostics.emit('open')
|
|
184
|
+
|
|
185
|
+
try {
|
|
186
|
+
for (const channel of this.#channels) {
|
|
187
|
+
await channel.recover(this.#connection)
|
|
188
|
+
}
|
|
189
|
+
} catch (exception) {
|
|
190
|
+
this.#diagnostics.emit('error', exception)
|
|
191
|
+
// prevent close re-entry from this broken connection
|
|
192
|
+
this.#connection.removeAllListeners()
|
|
193
|
+
try { await this.#connection.close() } catch {}
|
|
194
|
+
this.#connection = undefined
|
|
195
|
+
return retry
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
this.#recovery.resolve()
|
|
199
|
+
this.#recovery = new Promex()
|
|
200
|
+
}
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
This keeps `retry()` in charge, honours its backoff, and never surfaces an
|
|
204
|
+
unhandled rejection to `#close`.
|
|
205
|
+
|
|
206
|
+
### Test
|
|
207
|
+
|
|
208
|
+
In `test/connection.test.js`, mock `channel.recover` to throw once, then
|
|
209
|
+
succeed; assert `amqplib.connect` is called at least twice and `createChannel`
|
|
210
|
+
eventually resolves.
|
|
211
|
+
|
|
212
|
+
---
|
|
213
|
+
|
|
214
|
+
## Problem 3 — Unhandled rejection from `#close`
|
|
215
|
+
|
|
216
|
+
### Symptom
|
|
217
|
+
|
|
218
|
+
`#close` is an event handler attached to the amqplib `Connection`'s `close`
|
|
219
|
+
event. It `await this.open()` with no surrounding `try/catch`. If `open()`
|
|
220
|
+
rejects (see Problem 2), the rejection becomes an `unhandledRejection`. On
|
|
221
|
+
processes that exit on unhandled rejections, this kills the app; on others,
|
|
222
|
+
it silently leaves the library without a working connection.
|
|
223
|
+
|
|
224
|
+
### Fix
|
|
225
|
+
|
|
226
|
+
Detach lifecycle from the event emitter so that the rejection is either
|
|
227
|
+
swallowed on purpose (with diagnostics) or restarted:
|
|
228
|
+
|
|
229
|
+
```js
|
|
230
|
+
#close = (error) => {
|
|
231
|
+
this.#diagnostics.emit('close', error)
|
|
232
|
+
this.#connection.removeAllListeners()
|
|
233
|
+
this.#connection = undefined
|
|
234
|
+
|
|
235
|
+
if (error !== undefined) {
|
|
236
|
+
this.open().catch((exception) => this.#diagnostics.emit('error', exception))
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
Users that observe diagnostics get a signal (the existing `close` diagnostic
|
|
242
|
+
plus a new `error` diagnostic) without the process being destabilised.
|
|
243
|
+
|
|
244
|
+
### Test
|
|
245
|
+
|
|
246
|
+
Subscribe to the `error` diagnostic, force `open()` to reject, emit `close`
|
|
247
|
+
with an error on the mocked amqp connection, assert the diagnostic fires and
|
|
248
|
+
the process does not register an unhandled rejection.
|
|
249
|
+
|
|
250
|
+
---
|
|
251
|
+
|
|
252
|
+
## Problem 4 — `#transient` is too narrow on the initial connect
|
|
253
|
+
|
|
254
|
+
### Symptom
|
|
255
|
+
|
|
256
|
+
```109:114:source/connection.js
|
|
257
|
+
#transient (exception) {
|
|
258
|
+
const abruptly = exception.message === 'Socket closed abruptly during opening handshake'
|
|
259
|
+
const tls = exception.message === 'Client network socket disconnected before secure TLS connection was established'
|
|
260
|
+
|
|
261
|
+
return this.#running || abruptly || tls
|
|
262
|
+
}
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
Before `#running` flips to `true` (i.e. on the *first* `open()`), only two
|
|
266
|
+
exact error messages are considered transient. Everyday transient failures
|
|
267
|
+
(`ECONNREFUSED`, `EAI_AGAIN`, `ENOTFOUND`, `ETIMEDOUT`, `ECONNRESET`) cause
|
|
268
|
+
the boot to throw instead of retrying, forcing process supervisors to do the
|
|
269
|
+
retry. After first connect, `#running` is `true` forever, so *everything*
|
|
270
|
+
becomes transient — the asymmetry between "before first open" and "after
|
|
271
|
+
first open" is surprising.
|
|
272
|
+
|
|
273
|
+
### Fix
|
|
274
|
+
|
|
275
|
+
Introduce an explicit whitelist of transient failures that applies in both
|
|
276
|
+
phases:
|
|
277
|
+
|
|
278
|
+
```js
|
|
279
|
+
#transient (exception) {
|
|
280
|
+
const transientCodes = new Set([
|
|
281
|
+
'ECONNREFUSED', 'EAI_AGAIN', 'ENOTFOUND', 'ETIMEDOUT', 'ECONNRESET',
|
|
282
|
+
'EHOSTUNREACH', 'ENETUNREACH'
|
|
283
|
+
])
|
|
284
|
+
const transientMessages = new Set([
|
|
285
|
+
'Socket closed abruptly during opening handshake',
|
|
286
|
+
'Client network socket disconnected before secure TLS connection was established'
|
|
287
|
+
])
|
|
288
|
+
|
|
289
|
+
if (this.#running) return true
|
|
290
|
+
if (transientCodes.has(exception.code)) return true
|
|
291
|
+
if (transientMessages.has(exception.message)) return true
|
|
292
|
+
return false
|
|
293
|
+
}
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
Optionally gate the `#running` shortcut behind a maximum attempt count or a
|
|
297
|
+
"permanent" classifier for things like `ACCESS_REFUSED`, so a misconfigured
|
|
298
|
+
credential doesn't retry forever — but that is a separate behaviour change
|
|
299
|
+
and should be opt-in.
|
|
300
|
+
|
|
301
|
+
### Test
|
|
302
|
+
|
|
303
|
+
Parametrised test in `test/connection.test.js` for each listed error code,
|
|
304
|
+
asserting `open()` resolves after one failing `amqplib.connect`.
|
|
305
|
+
|
|
306
|
+
---
|
|
307
|
+
|
|
308
|
+
## Problem 5 — Reentrant `#close` can run two overlapping `#open` loops
|
|
309
|
+
|
|
310
|
+
### Symptom
|
|
311
|
+
|
|
312
|
+
If the new connection dies *while* `#open` is still iterating
|
|
313
|
+
`channel.recover()`, amqplib fires `close` on that connection. The handler
|
|
314
|
+
executes:
|
|
315
|
+
|
|
316
|
+
```js
|
|
317
|
+
this.#connection.removeAllListeners()
|
|
318
|
+
this.#connection = undefined
|
|
319
|
+
if (error !== undefined) await this.open()
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
While the outer `#open` is still running and will, on its next loop
|
|
323
|
+
iteration, pass the now-`undefined` `this.#connection` to `channel.recover`
|
|
324
|
+
(crash → see Problem 2). You now have two concurrent `open()` retry loops
|
|
325
|
+
racing; whichever one wins may leave the other with a live amqp connection
|
|
326
|
+
that's unreferenced.
|
|
327
|
+
|
|
328
|
+
### Fix
|
|
329
|
+
|
|
330
|
+
Serialise `open()` with a single in-flight promise, and stop reading
|
|
331
|
+
`this.#connection` as a shared variable mid-loop:
|
|
332
|
+
|
|
333
|
+
```js
|
|
334
|
+
/** @type {Promise<void> | null} */
|
|
335
|
+
#opening = null
|
|
336
|
+
|
|
337
|
+
async open () {
|
|
338
|
+
if (this.#opening !== null) return this.#opening
|
|
339
|
+
this.#opening = retry(this.#open).finally(() => { this.#opening = null })
|
|
340
|
+
await this.#opening
|
|
341
|
+
this.#running = true
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
#open = async (retry) => {
|
|
345
|
+
let connection
|
|
346
|
+
try {
|
|
347
|
+
connection = await amqp.connect(this.#url)
|
|
348
|
+
} catch (exception) {
|
|
349
|
+
if (this.#transient(exception)) return retry
|
|
350
|
+
else throw exception
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
connection.on('error', noop)
|
|
354
|
+
connection.on('close', this.#close)
|
|
355
|
+
this.#connection = connection
|
|
356
|
+
this.#diagnostics.emit('open')
|
|
357
|
+
|
|
358
|
+
try {
|
|
359
|
+
for (const channel of this.#channels) await channel.recover(connection)
|
|
360
|
+
} catch (exception) {
|
|
361
|
+
this.#diagnostics.emit('error', exception)
|
|
362
|
+
connection.removeAllListeners()
|
|
363
|
+
try { await connection.close() } catch {}
|
|
364
|
+
if (this.#connection === connection) this.#connection = undefined
|
|
365
|
+
return retry
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
this.#recovery.resolve()
|
|
369
|
+
this.#recovery = new Promex()
|
|
370
|
+
}
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Two wins:
|
|
374
|
+
|
|
375
|
+
1. Channel recovery uses a local `connection` reference, so a concurrent
|
|
376
|
+
`#close` can't swap it out mid-loop.
|
|
377
|
+
2. Concurrent `open()` calls converge on a single retry loop.
|
|
378
|
+
|
|
379
|
+
### Test
|
|
380
|
+
|
|
381
|
+
Simulate two `close` events in quick succession (before the first recovery
|
|
382
|
+
finishes) and assert only one `amqp.connect` is *in progress* at any time and
|
|
383
|
+
the final state has exactly one live connection.
|
|
384
|
+
|
|
385
|
+
---
|
|
386
|
+
|
|
387
|
+
## Problem 6 — Partial `Channel.recover()` leaves the channel wedged
|
|
388
|
+
|
|
389
|
+
### Symptom
|
|
390
|
+
|
|
391
|
+
If `recover()` throws between `await this.create()` and the final
|
|
392
|
+
`#recovery.resolve()`, the Channel has already:
|
|
393
|
+
|
|
394
|
+
- replaced `this.#channel` with a new one,
|
|
395
|
+
- reset lazy locks,
|
|
396
|
+
- possibly pushed new consumer tags,
|
|
397
|
+
|
|
398
|
+
but has **not**:
|
|
399
|
+
|
|
400
|
+
- rejected pending confirmations,
|
|
401
|
+
- unpaused `#paused`,
|
|
402
|
+
- resolved `#recovery`.
|
|
403
|
+
|
|
404
|
+
Every user operation currently parked on `#recovery` stays parked forever
|
|
405
|
+
unless a subsequent recover succeeds. Combined with Problem 1, this is the
|
|
406
|
+
mechanism by which single failures persist.
|
|
407
|
+
|
|
408
|
+
### Fix
|
|
409
|
+
|
|
410
|
+
Make `recover()` structurally transactional — always release waiters on
|
|
411
|
+
failure by rejecting `#recovery` (with a sentinel that `failsafe.#recover`
|
|
412
|
+
classifies as permanent, so callers see a real error instead of hanging):
|
|
413
|
+
|
|
414
|
+
```js
|
|
415
|
+
async recover (connection) {
|
|
416
|
+
this.#recovering = true
|
|
417
|
+
try {
|
|
418
|
+
this.#connection = connection
|
|
419
|
+
await this.create()
|
|
420
|
+
lazy.reset(this)
|
|
421
|
+
await recall(this)
|
|
422
|
+
this.#unpause(INTERRUPTION)
|
|
423
|
+
for (const confirmation of this.#confirmations) confirmation.reject(INTERRUPTION)
|
|
424
|
+
await new Promise(resolve => setTimeout(resolve, 0))
|
|
425
|
+
this.#recovery.resolve()
|
|
426
|
+
this.#recovery = new Promex()
|
|
427
|
+
this.#diagnostics.emit('recover')
|
|
428
|
+
} catch (exception) {
|
|
429
|
+
// release everyone parked on #recovery; they will propagate the error
|
|
430
|
+
this.#unpause(exception)
|
|
431
|
+
for (const confirmation of this.#confirmations) confirmation.reject(exception)
|
|
432
|
+
this.#recovery.reject(exception)
|
|
433
|
+
this.#recovery = new Promex()
|
|
434
|
+
throw exception
|
|
435
|
+
} finally {
|
|
436
|
+
this.#recovering = false
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
```
|
|
440
|
+
|
|
441
|
+
Combined with Problem 2's fix, the Connection retry loop will catch this,
|
|
442
|
+
close the broken connection, and retry — instead of wedging callers.
|
|
443
|
+
|
|
444
|
+
### Test
|
|
445
|
+
|
|
446
|
+
Force `create()` to throw, call `recover()`, assert that a previously
|
|
447
|
+
`await`-ing `send()` rejects (rather than hanging) with the propagated
|
|
448
|
+
exception.
|
|
449
|
+
|
|
450
|
+
---
|
|
451
|
+
|
|
452
|
+
## Problem 7 — No liveness/heartbeat watchdog
|
|
453
|
+
|
|
454
|
+
### Symptom (optional but recommended)
|
|
455
|
+
|
|
456
|
+
amqplib's `Connection` relies on AMQP heartbeats and the underlying TCP
|
|
457
|
+
socket. In the half-open TCP case (NAT drop, cloud NLB idle timeout,
|
|
458
|
+
broker OOM without RST), neither `close` nor `error` may be emitted for a
|
|
459
|
+
long time — enough to look "stuck" even though the retry machinery is
|
|
460
|
+
healthy.
|
|
461
|
+
|
|
462
|
+
### Fix
|
|
463
|
+
|
|
464
|
+
Add a watchdog that forcibly destroys the underlying amqp `Connection` if no
|
|
465
|
+
progress has been observed within a configurable window. The simplest
|
|
466
|
+
implementation:
|
|
467
|
+
|
|
468
|
+
```js
|
|
469
|
+
// source/connection.js
|
|
470
|
+
#heartbeatTimer = null
|
|
471
|
+
|
|
472
|
+
#armWatchdog (connection, timeoutMs = 60_000) {
|
|
473
|
+
const reset = () => {
|
|
474
|
+
clearTimeout(this.#heartbeatTimer)
|
|
475
|
+
this.#heartbeatTimer = setTimeout(() => {
|
|
476
|
+
try { connection.connection?.stream?.destroy() } catch {}
|
|
477
|
+
}, timeoutMs)
|
|
478
|
+
}
|
|
479
|
+
reset()
|
|
480
|
+
connection.on('heartbeat', reset) // amqplib emits on each heartbeat
|
|
481
|
+
connection.on('close', () => clearTimeout(this.#heartbeatTimer))
|
|
482
|
+
}
|
|
483
|
+
```
|
|
484
|
+
|
|
485
|
+
Call `#armWatchdog(this.#connection)` from `#open`. Any truly silent
|
|
486
|
+
connection will be destroyed, emitting `close`, and the normal retry loop
|
|
487
|
+
will take over.
|
|
488
|
+
|
|
489
|
+
### Test
|
|
490
|
+
|
|
491
|
+
Integration test (can be skipped by default) that blocks RabbitMQ traffic
|
|
492
|
+
with `iptables` / network namespaces in CI, asserting the library recovers
|
|
493
|
+
within watchdog + retry window.
|
|
494
|
+
|
|
495
|
+
---
|
|
496
|
+
|
|
497
|
+
## Rollout order
|
|
498
|
+
|
|
499
|
+
1. Problem 1 and Problem 2 together — they remove the deadlock and keep
|
|
500
|
+
recovery inside the retry loop. These are the minimum fix for the hang.
|
|
501
|
+
2. Problem 3 — guard `#close`'s rejection.
|
|
502
|
+
3. Problem 6 — transactional `Channel.recover()`.
|
|
503
|
+
4. Problem 5 — serialise `open()` / local connection reference.
|
|
504
|
+
5. Problem 4 — broaden transient classifier.
|
|
505
|
+
6. Problem 7 — optional watchdog.
|
|
506
|
+
|
|
507
|
+
Each step is independently deployable and testable.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "comq",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "ComQ",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Artem Gurtovoi",
|
|
@@ -18,9 +18,6 @@
|
|
|
18
18
|
"AMQP",
|
|
19
19
|
"RabbitMQ"
|
|
20
20
|
],
|
|
21
|
-
"engines": {
|
|
22
|
-
"node": ">= 12.0.0"
|
|
23
|
-
},
|
|
24
21
|
"publishConfig": {
|
|
25
22
|
"access": "public"
|
|
26
23
|
},
|
|
@@ -34,14 +31,14 @@
|
|
|
34
31
|
"@types/jest": "^29.5.8",
|
|
35
32
|
"husky": "^9.0.11",
|
|
36
33
|
"jest": "^29.5.0",
|
|
37
|
-
"js-yaml": "^4.1.
|
|
34
|
+
"js-yaml": "^4.1.1",
|
|
38
35
|
"randomstring": "^1.3.0",
|
|
39
36
|
"snazzy": "^9.0.0",
|
|
40
37
|
"standard": "^17.1.0"
|
|
41
38
|
},
|
|
42
39
|
"dependencies": {
|
|
43
|
-
"amqplib": "^0.
|
|
44
|
-
"msgpackr": "^
|
|
40
|
+
"amqplib": "^2.0.1",
|
|
41
|
+
"msgpackr": "^2.0.1",
|
|
45
42
|
"promex": "^0.1.1",
|
|
46
43
|
"reretry": "^0.1.0"
|
|
47
44
|
},
|
package/types/io.d.ts
CHANGED
|
@@ -15,7 +15,7 @@ declare namespace comq {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
interface Destroyable extends EventEmitter {
|
|
18
|
-
destroy
|
|
18
|
+
destroy(): void
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
interface Request {
|
|
@@ -25,51 +25,55 @@ declare namespace comq {
|
|
|
25
25
|
}
|
|
26
26
|
|
|
27
27
|
interface IO extends _diagnostics.Diagnosable {
|
|
28
|
-
reply
|
|
28
|
+
reply(queue: string, produce: Producer): Promise<void>
|
|
29
29
|
|
|
30
|
-
request<Reply = any, Request = any>
|
|
30
|
+
request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply> | Promise<Readable>
|
|
31
31
|
|
|
32
|
-
request
|
|
32
|
+
request(queue: string, stream: Readable, encoding?: _encoding.Encoding): Promise<Readable>
|
|
33
33
|
|
|
34
|
-
consume<T = any>
|
|
34
|
+
consume<T = any>(exchange: string, group: string, consumer: Consumer<T>): Promise<void>
|
|
35
35
|
|
|
36
|
-
consume<T = any>
|
|
36
|
+
consume<T = any>(exchange: string, consumer: Consumer<T>): Promise<void>
|
|
37
37
|
|
|
38
|
-
emit
|
|
38
|
+
emit(exchange: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
|
|
39
39
|
|
|
40
|
-
emit
|
|
40
|
+
emit(exchange: string, payload: any, properties?: _amqp.Properties): Promise<void>
|
|
41
41
|
|
|
42
|
-
emit
|
|
42
|
+
emit(exchange: string, stream: Readable, encoding?: _encoding.Encoding): Promise<void>
|
|
43
43
|
|
|
44
|
-
emit
|
|
44
|
+
emit(exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
|
|
45
45
|
|
|
46
|
-
enqueue
|
|
46
|
+
enqueue(exchange: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
|
|
47
47
|
|
|
48
|
-
enqueue
|
|
48
|
+
enqueue(exchange: string, payload: any, properties?: _amqp.Properties): Promise<void>
|
|
49
49
|
|
|
50
|
-
enqueue
|
|
50
|
+
enqueue(exchange: string, stream: Readable, encoding?: _encoding.Encoding): Promise<void>
|
|
51
51
|
|
|
52
|
-
enqueue
|
|
52
|
+
enqueue(exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
|
|
53
53
|
|
|
54
|
-
process<T = any>
|
|
54
|
+
process<T = any>(queue: string, processor: Consumer<T>): Promise<void>
|
|
55
55
|
|
|
56
|
-
seal
|
|
56
|
+
seal(): Promise<void>
|
|
57
57
|
|
|
58
|
-
close
|
|
58
|
+
close(): Promise<void>
|
|
59
59
|
|
|
60
|
-
diagnose
|
|
60
|
+
diagnose(event: 'open', listener: (index?: number) => void): void
|
|
61
61
|
|
|
62
|
-
diagnose
|
|
62
|
+
diagnose(event: 'close', listener: (index?: number) => void): void
|
|
63
63
|
|
|
64
|
-
diagnose
|
|
64
|
+
diagnose(event: 'flow', listener: (channel: _topology.type, index?: number) => void): void
|
|
65
65
|
|
|
66
|
-
diagnose
|
|
66
|
+
diagnose(event: 'drain', listener: (channel: _topology.type, index?: number) => void): void
|
|
67
67
|
|
|
68
|
-
diagnose
|
|
68
|
+
diagnose(event: 'remove', listener: (index?: number) => void): void
|
|
69
69
|
|
|
70
|
-
diagnose
|
|
70
|
+
diagnose(event: 'recover', listener: (channel: _topology.type, index?: number) => void): void
|
|
71
71
|
|
|
72
|
-
diagnose
|
|
72
|
+
diagnose(event: 'discard', listener: (channel: _topology.type, message: any, index?: number) => void): void
|
|
73
|
+
|
|
74
|
+
diagnose(event: 'pause', listener: (channel: _topology.type) => void): void
|
|
75
|
+
|
|
76
|
+
diagnose(event: 'resume', listener: (channel: _topology.type) => void): void
|
|
73
77
|
}
|
|
74
78
|
}
|
|
75
79
|
|