comq 0.15.2 → 0.15.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/features/shards.feature +16 -0
- package/features/steps/brokers.js +8 -0
- package/features/steps/context.js +6 -1
- package/features/steps/hooks.js +3 -0
- package/features/steps/parameters.js +1 -1
- package/features/steps/rpc.js +61 -1
- package/package.json +1 -1
- package/readme.md +3 -6
- package/source/connection.js +1 -1
- package/source/events.js +1 -1
- package/source/io.js +29 -53
- package/source/shards/channel.js +34 -4
- package/test/io.request.test.js +117 -0
- package/test/shards/channel.test.js +52 -0
- package/types/io.d.ts +2 -4
- package/test/io.request.timeout.test.js +0 -61
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,18 @@
|
|
|
1
|
+
## [0.15.4](https://github.com/toa-io/comq/compare/v0.15.3...v0.15.4) (2026-08-16)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* remove request() timeout ([#246](https://github.com/toa-io/comq/issues/246)) ([d432388](https://github.com/toa-io/comq/commit/d4323889bcad867a870d0ed351b92098f6aad551))
|
|
7
|
+
|
|
8
|
+
## [0.15.3](https://github.com/toa-io/comq/compare/v0.15.2...v0.15.3) (2026-08-04)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* re-send requests left unanswered by a lost shard ([c534a01](https://github.com/toa-io/comq/commit/c534a0136e5f0189cdb6976cd9cffb9a2ca5a148))
|
|
14
|
+
* re-send requests left unanswered by a lost shard ([#242](https://github.com/toa-io/comq/issues/242)) ([c821da8](https://github.com/toa-io/comq/commit/c821da895049bd78a907c6705477ebe74c160e1b))
|
|
15
|
+
|
|
1
16
|
## [0.15.2](https://github.com/toa-io/comq/compare/v0.15.1...v0.15.2) (2026-08-04)
|
|
2
17
|
|
|
3
18
|
|
package/features/shards.feature
CHANGED
|
@@ -40,6 +40,22 @@ Feature: Sharded Connection
|
|
|
40
40
|
And all replies have been received
|
|
41
41
|
Then the broker is up
|
|
42
42
|
|
|
43
|
+
Scenario: Shard crashes while requests await their replies
|
|
44
|
+
Given an active sharded connection
|
|
45
|
+
And a producer replying `slow` queue in 3000ms
|
|
46
|
+
When the consumer sends 10 requests to the `slow` queue
|
|
47
|
+
And one of the brokers has crashed
|
|
48
|
+
Then all replies have been received
|
|
49
|
+
Then the broker is up
|
|
50
|
+
|
|
51
|
+
Scenario: Shard stops responding while requests are sent
|
|
52
|
+
Given watchdog interval is set to 3000ms with 1s AMQP heartbeat
|
|
53
|
+
And an active sharded connection
|
|
54
|
+
And a producer replying `stalled` queue
|
|
55
|
+
When the consumer sends 10 requests to the `stalled` queue as a broker freezes
|
|
56
|
+
Then all replies have been received
|
|
57
|
+
Then the broker is up
|
|
58
|
+
|
|
43
59
|
Scenario: Shard crashes while publishing events
|
|
44
60
|
Given an active sharded connection
|
|
45
61
|
And events are exclusively consumed from the `flood` exchange
|
|
@@ -103,6 +103,9 @@ async function healthy (n = 0) {
|
|
|
103
103
|
|
|
104
104
|
const actions = {
|
|
105
105
|
up: async (n = 0) => {
|
|
106
|
+
// a paused broker is neither stopped nor usable, and the hooks bring every
|
|
107
|
+
// broker up between scenarios, so unpausing belongs here
|
|
108
|
+
await docker('docker', ['unpause', `comq-rmq-${n}`]).catch(() => undefined)
|
|
106
109
|
await docker('docker', ['start', `comq-rmq-${n}`])
|
|
107
110
|
await healthy(n)
|
|
108
111
|
},
|
|
@@ -111,6 +114,11 @@ const actions = {
|
|
|
111
114
|
},
|
|
112
115
|
crashed: async (n = 0) => {
|
|
113
116
|
await docker('docker', ['kill', `comq-rmq-${n}`])
|
|
117
|
+
},
|
|
118
|
+
// a frozen broker keeps its connections open while answering nothing, which is
|
|
119
|
+
// what a publisher cannot tell from a working one until the watchdog fires
|
|
120
|
+
frozen: async (n = 0) => {
|
|
121
|
+
await docker('docker', ['pause', `comq-rmq-${n}`])
|
|
114
122
|
}
|
|
115
123
|
}
|
|
116
124
|
|
|
@@ -89,7 +89,12 @@ class Context extends World {
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
#url (i, user, password) {
|
|
92
|
-
|
|
92
|
+
const url = PROTOCOL + user + ':' + password + '@' + getAddress(i)
|
|
93
|
+
const heartbeat = global.COMQ_TESTING_AMQP_HEARTBEAT
|
|
94
|
+
|
|
95
|
+
// the watchdog measures silence, so it may only be shortened along with the
|
|
96
|
+
// interval at which a healthy broker is expected to say something
|
|
97
|
+
return heartbeat === undefined ? url : url + '?heartbeat=' + heartbeat
|
|
93
98
|
}
|
|
94
99
|
}
|
|
95
100
|
|
package/features/steps/hooks.js
CHANGED
|
@@ -12,6 +12,9 @@ After(
|
|
|
12
12
|
* @this {comq.features.Context}
|
|
13
13
|
*/
|
|
14
14
|
async function () {
|
|
15
|
+
delete global.COMQ_TESTING_WATCHDOG_INTERVAL
|
|
16
|
+
delete global.COMQ_TESTING_AMQP_HEARTBEAT
|
|
17
|
+
|
|
15
18
|
await Promise.all(
|
|
16
19
|
Array.from({ length: BROKERS_AMOUNT }, (_, n) => actions.up(n))
|
|
17
20
|
)
|
package/features/steps/rpc.js
CHANGED
|
@@ -4,9 +4,10 @@ const stream = require('node:stream')
|
|
|
4
4
|
const assert = require('node:assert')
|
|
5
5
|
const { randomBytes } = require('node:crypto')
|
|
6
6
|
const { parse } = require('./yaml')
|
|
7
|
-
const { match, timeout, quantity } = require('@toa.io/generic')
|
|
7
|
+
const { match, timeout, quantity, random } = require('@toa.io/generic')
|
|
8
8
|
const { Given, When, Then } = require('@cucumber/cucumber')
|
|
9
9
|
const { Readable } = require('node:stream')
|
|
10
|
+
const { actions, BROKERS_AMOUNT } = require('./brokers')
|
|
10
11
|
|
|
11
12
|
Given('function replying {token} queue:',
|
|
12
13
|
/**
|
|
@@ -85,6 +86,12 @@ Given('idle timeout is set to {number}ms',
|
|
|
85
86
|
global.COMQ_TESTING_IDLE_INTERVAL = value
|
|
86
87
|
})
|
|
87
88
|
|
|
89
|
+
Given('watchdog interval is set to {number}ms with {number}s AMQP heartbeat',
|
|
90
|
+
function (watchdog, heartbeat) {
|
|
91
|
+
global.COMQ_TESTING_WATCHDOG_INTERVAL = watchdog
|
|
92
|
+
global.COMQ_TESTING_AMQP_HEARTBEAT = heartbeat
|
|
93
|
+
})
|
|
94
|
+
|
|
88
95
|
Given('a producer replying {token} queue',
|
|
89
96
|
/**
|
|
90
97
|
* @param {string} queue
|
|
@@ -100,6 +107,22 @@ Given('a producer replying {token} queue',
|
|
|
100
107
|
await this.io.reply(queue, producer)
|
|
101
108
|
})
|
|
102
109
|
|
|
110
|
+
Given('a producer replying {token} queue in {number}ms',
|
|
111
|
+
/**
|
|
112
|
+
* @param {string} queue
|
|
113
|
+
* @param {number} delay
|
|
114
|
+
* @this {comq.features.Context}
|
|
115
|
+
*/
|
|
116
|
+
async function (queue, delay) {
|
|
117
|
+
const producer = async (request) => {
|
|
118
|
+
await timeout(delay)
|
|
119
|
+
|
|
120
|
+
return request
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
await this.io.reply(queue, producer)
|
|
124
|
+
})
|
|
125
|
+
|
|
103
126
|
Given('function replying {token} queue is expected:',
|
|
104
127
|
/**
|
|
105
128
|
* @param {string} queue
|
|
@@ -136,6 +159,32 @@ When('the consumer sends a request to the {token} queue',
|
|
|
136
159
|
await send.call(this, queue, payload)
|
|
137
160
|
})
|
|
138
161
|
|
|
162
|
+
When('the consumer sends {number} requests to the {token} queue',
|
|
163
|
+
/**
|
|
164
|
+
* @param {number} amount
|
|
165
|
+
* @param {string} queue
|
|
166
|
+
* @this {comq.features.Context}
|
|
167
|
+
*/
|
|
168
|
+
async function (amount, queue) {
|
|
169
|
+
sendMany.call(this, amount, queue)
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
// the requests must be published while the frozen shard still looks reachable,
|
|
173
|
+
// which is a window too short to spend on a step boundary
|
|
174
|
+
When('the consumer sends {number} requests to the {token} queue as a broker freezes',
|
|
175
|
+
/**
|
|
176
|
+
* @param {number} amount
|
|
177
|
+
* @param {string} queue
|
|
178
|
+
* @this {comq.features.Context}
|
|
179
|
+
*/
|
|
180
|
+
async function (amount, queue) {
|
|
181
|
+
this.shard = random(BROKERS_AMOUNT)
|
|
182
|
+
|
|
183
|
+
await actions.frozen(this.shard)
|
|
184
|
+
|
|
185
|
+
sendMany.call(this, amount, queue)
|
|
186
|
+
})
|
|
187
|
+
|
|
139
188
|
When('the consumer requests a stream with the following request to the {token} queue:',
|
|
140
189
|
/**
|
|
141
190
|
* @param {string} queue
|
|
@@ -425,6 +474,17 @@ Then('all replies have been received',
|
|
|
425
474
|
await Promise.all(this.requestsSent)
|
|
426
475
|
})
|
|
427
476
|
|
|
477
|
+
/**
|
|
478
|
+
* @param {number} amount
|
|
479
|
+
* @param {string} queue
|
|
480
|
+
* @this {comq.features.Context}
|
|
481
|
+
*/
|
|
482
|
+
function sendMany (amount, queue) {
|
|
483
|
+
for (let i = 0; i < amount; i++) {
|
|
484
|
+
this.requestsSent.push(this.io.request(queue, randomBytes(8)))
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
428
488
|
async function send (queue, payload) {
|
|
429
489
|
if (this.expected) await this.expected
|
|
430
490
|
|
package/package.json
CHANGED
package/readme.md
CHANGED
|
@@ -103,11 +103,10 @@ await io.reply('add_numbers', ({ a, b }) => (a + b))
|
|
|
103
103
|
|
|
104
104
|
## Request
|
|
105
105
|
|
|
106
|
-
`async IO.request(queue: string, payload: any, encoding?: string
|
|
106
|
+
`async IO.request(queue: string, payload: any, encoding?: string): any`
|
|
107
107
|
|
|
108
108
|
Send encoded Request message with `replyTo` and `correlationId` properties set and
|
|
109
|
-
return decoded Reply content.
|
|
110
|
-
Reply arrives in time; encoding may be omitted when only `timeout` is passed.
|
|
109
|
+
return decoded Reply content. The promise stays pending until the Reply arrives.
|
|
111
110
|
|
|
112
111
|
On the initial call, queues for Requests and Replies are asserted.
|
|
113
112
|
|
|
@@ -115,7 +114,6 @@ On the initial call, queues for Requests and Replies are asserted.
|
|
|
115
114
|
|
|
116
115
|
```javascript
|
|
117
116
|
const sum = await io.request('add_numbers', { a: 1, b: 2 })
|
|
118
|
-
const sumOrTimeout = await io.request('add_numbers', { a: 1, b: 2 }, 5000)
|
|
119
117
|
```
|
|
120
118
|
|
|
121
119
|
## Consumption
|
|
@@ -332,8 +330,7 @@ Outgoing messages are sent to a single connection chosen at random from the shar
|
|
|
332
330
|
underlying connection or experience channel [back pressure](#flow-control) on a corresponding channel are removed from
|
|
333
331
|
the pool until the issue is resolved. Pending messages meeting these conditions are immediately routed among the
|
|
334
332
|
remaining shards in the pool. If no shards are available, `send` / `publish` / `request` wait until a shard's
|
|
335
|
-
connection is re-established
|
|
336
|
-
`ETIMEDOUT` instead of hanging while the pool is empty or a Reply never arrives.
|
|
333
|
+
connection is re-established.
|
|
337
334
|
|
|
338
335
|
Incoming messages are consumed from all shards.
|
|
339
336
|
|
package/source/connection.js
CHANGED
|
@@ -151,7 +151,7 @@ class Connection {
|
|
|
151
151
|
* @param {comq.amqp.Connection} connection
|
|
152
152
|
* @param {number} [timeoutMs]
|
|
153
153
|
*/
|
|
154
|
-
#armWatchdog (connection, timeoutMs = WATCHDOG_MS) {
|
|
154
|
+
#armWatchdog (connection, timeoutMs = global.COMQ_TESTING_WATCHDOG_INTERVAL ?? WATCHDOG_MS) {
|
|
155
155
|
const socket = connection.connection?.stream
|
|
156
156
|
|
|
157
157
|
if (socket === undefined) return
|
package/source/events.js
CHANGED
|
@@ -4,4 +4,4 @@
|
|
|
4
4
|
exports.connection = ['open', 'close', 'error']
|
|
5
5
|
|
|
6
6
|
/** @type {comq.diagnostics.Event[]} */
|
|
7
|
-
exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return']
|
|
7
|
+
exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost']
|
package/source/io.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
const stream = require('node:stream')
|
|
4
4
|
const { EventEmitter } = require('node:events')
|
|
5
5
|
const { randomBytes } = require('node:crypto')
|
|
6
|
-
const { setTimeout
|
|
6
|
+
const { setTimeout } = require('node:timers/promises')
|
|
7
7
|
const { Promex } = require('promex')
|
|
8
8
|
const { memo, failsafe, lazy, track } = require('./attributes')
|
|
9
9
|
|
|
@@ -35,8 +35,8 @@ class IO {
|
|
|
35
35
|
/** @type {comq.ReplyEmitter | null} */
|
|
36
36
|
#control = null
|
|
37
37
|
|
|
38
|
-
/** @type {
|
|
39
|
-
#pendingReplies = new
|
|
38
|
+
/** @type {Map<Promex, comq.Request>} */
|
|
39
|
+
#pendingReplies = new Map()
|
|
40
40
|
|
|
41
41
|
/** @type {Set<comq.Destroyable>} */
|
|
42
42
|
#replyStreams = new Set()
|
|
@@ -75,45 +75,24 @@ class IO {
|
|
|
75
75
|
/**
|
|
76
76
|
* @param {string} queue
|
|
77
77
|
* @param {any | Readable} payload
|
|
78
|
-
* @param {comq.Encoding
|
|
79
|
-
* @param {number} [timeout]
|
|
78
|
+
* @param {comq.Encoding} [encoding]
|
|
80
79
|
* @returns {Promise<any | Readable>}
|
|
81
80
|
*/
|
|
82
|
-
async (queue, payload, encoding
|
|
83
|
-
if (typeof encoding === 'number') {
|
|
84
|
-
timeout = encoding
|
|
85
|
-
encoding = undefined
|
|
86
|
-
}
|
|
87
|
-
|
|
81
|
+
async (queue, payload, encoding) => {
|
|
88
82
|
if (payload instanceof stream.Readable) {
|
|
89
83
|
return pipeline(
|
|
90
84
|
payload,
|
|
91
|
-
(payload) => this.request(queue, payload, encoding
|
|
85
|
+
(payload) => this.request(queue, payload, encoding),
|
|
92
86
|
this.#requests
|
|
93
87
|
)
|
|
94
88
|
}
|
|
95
89
|
|
|
96
|
-
const request = this.#createRequest(queue, payload,
|
|
90
|
+
const request = this.#createRequest(queue, payload, encoding)
|
|
97
91
|
const reply = this.#createReply(request)
|
|
98
92
|
|
|
99
|
-
|
|
100
|
-
.then(() => reply)
|
|
93
|
+
await this.#requests.send(queue, request.buffer, request.properties)
|
|
101
94
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
try {
|
|
105
|
-
return await Promise.race([
|
|
106
|
-
done,
|
|
107
|
-
delay(timeout).then(() => Promise.reject(timeoutError(timeout)))
|
|
108
|
-
])
|
|
109
|
-
} catch (exception) {
|
|
110
|
-
if (exception.code === 'ETIMEDOUT') {
|
|
111
|
-
request.emitter.removeAllListeners(request.properties.correlationId)
|
|
112
|
-
reply.reject(exception)
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
throw exception
|
|
116
|
-
}
|
|
95
|
+
return reply
|
|
117
96
|
}))
|
|
118
97
|
|
|
119
98
|
consume = lazy(this, this.#createEventChannel,
|
|
@@ -230,9 +209,14 @@ class IO {
|
|
|
230
209
|
}
|
|
231
210
|
|
|
232
211
|
#setupRetransmission () {
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
212
|
+
if (this.#requests.sharded === true) {
|
|
213
|
+
// a shard leaves the pool when it rejects a publish, and is lost when its
|
|
214
|
+
// connection drops, which leaves an already sent request unanswered
|
|
215
|
+
this.#requests.diagnose('remove', this.#retransmit)
|
|
216
|
+
this.#requests.diagnose('lost', this.#retransmit)
|
|
217
|
+
} else {
|
|
218
|
+
this.#requests.diagnose('recover', this.#retransmit)
|
|
219
|
+
}
|
|
236
220
|
}
|
|
237
221
|
|
|
238
222
|
/**
|
|
@@ -316,7 +300,7 @@ class IO {
|
|
|
316
300
|
* @return {Promex<any>}
|
|
317
301
|
*/
|
|
318
302
|
#createReply (request) {
|
|
319
|
-
const reply = this.#createPendingReply()
|
|
303
|
+
const reply = this.#createPendingReply(request)
|
|
320
304
|
|
|
321
305
|
request.emitter.once(request.properties.correlationId, this.#getReplyResolver(request, reply))
|
|
322
306
|
|
|
@@ -324,12 +308,13 @@ class IO {
|
|
|
324
308
|
}
|
|
325
309
|
|
|
326
310
|
/**
|
|
311
|
+
* @param {comq.Request} request
|
|
327
312
|
* @return {Promex}
|
|
328
313
|
*/
|
|
329
|
-
#createPendingReply () {
|
|
314
|
+
#createPendingReply (request) {
|
|
330
315
|
const reply = new Promex()
|
|
331
316
|
|
|
332
|
-
this.#pendingReplies.
|
|
317
|
+
this.#pendingReplies.set(reply, request)
|
|
333
318
|
|
|
334
319
|
reply
|
|
335
320
|
.catch(noop)
|
|
@@ -422,7 +407,7 @@ class IO {
|
|
|
422
407
|
Even if these messages are lost, the reply stream will be closed anyway,
|
|
423
408
|
either due to missing heartbeat or the deletion of the stream queue.
|
|
424
409
|
*/
|
|
425
|
-
await
|
|
410
|
+
await setTimeout(50)
|
|
426
411
|
}
|
|
427
412
|
|
|
428
413
|
/**
|
|
@@ -455,10 +440,14 @@ class IO {
|
|
|
455
440
|
}
|
|
456
441
|
|
|
457
442
|
#retransmit = () => {
|
|
458
|
-
for (const
|
|
443
|
+
for (const [reply, request] of this.#pendingReplies) {
|
|
444
|
+
// detaching this attempt alone leaves the listeners of the reply streams
|
|
445
|
+
// that are still flowing over the other shards in place
|
|
446
|
+
request.emitter.removeAllListeners(request.properties.correlationId)
|
|
459
447
|
|
|
460
|
-
|
|
461
|
-
|
|
448
|
+
// trigger failsafe attribute
|
|
449
|
+
reply.reject(RETRANSMISSION)
|
|
450
|
+
}
|
|
462
451
|
}
|
|
463
452
|
|
|
464
453
|
/**
|
|
@@ -485,19 +474,6 @@ const DEFAULT = 'application/json'
|
|
|
485
474
|
|
|
486
475
|
const RETRANSMISSION = /** @type {Error} */ Symbol('retransmission')
|
|
487
476
|
|
|
488
|
-
/**
|
|
489
|
-
* @param {number} timeout
|
|
490
|
-
* @returns {Error}
|
|
491
|
-
*/
|
|
492
|
-
function timeoutError (timeout) {
|
|
493
|
-
const error = /** @type {Error & { code: string }} */ new Error(
|
|
494
|
-
`Request timed out after ${timeout}ms`)
|
|
495
|
-
|
|
496
|
-
error.code = 'ETIMEDOUT'
|
|
497
|
-
|
|
498
|
-
return error
|
|
499
|
-
}
|
|
500
|
-
|
|
501
477
|
function noop () {}
|
|
502
478
|
|
|
503
479
|
exports.IO = IO
|
package/source/shards/channel.js
CHANGED
|
@@ -25,6 +25,9 @@ class Channel {
|
|
|
25
25
|
/** @type {Promex[]} */
|
|
26
26
|
#down = []
|
|
27
27
|
|
|
28
|
+
/** @type {boolean[]} */
|
|
29
|
+
#alive = []
|
|
30
|
+
|
|
28
31
|
/** @type {Map<comq.Channel, Promex>} */
|
|
29
32
|
#bench = new Map()
|
|
30
33
|
|
|
@@ -106,18 +109,28 @@ class Channel {
|
|
|
106
109
|
* Tracks whether a shard is reachable, so that subscribing does not wait for
|
|
107
110
|
* one that is not. The pool itself is left alone: a channel is only benched
|
|
108
111
|
* when it actually fails to publish, otherwise a lost connection would
|
|
109
|
-
* interrupt the streams it is carrying.
|
|
112
|
+
* interrupt the streams it is carrying. Losing a shard is reported instead,
|
|
113
|
+
* since a request awaiting its reply on it will never be answered.
|
|
110
114
|
*
|
|
111
115
|
* @param {comq.Connection} connection
|
|
112
116
|
* @param {number} index
|
|
113
117
|
*/
|
|
114
118
|
#watch (connection, index) {
|
|
115
119
|
this.#down[index] = new Promex()
|
|
120
|
+
this.#alive[index] = connection.connected !== false
|
|
116
121
|
|
|
117
122
|
if (connection.connected === false) this.#down[index].resolve()
|
|
118
123
|
|
|
119
|
-
connection.diagnose('close', () =>
|
|
120
|
-
|
|
124
|
+
connection.diagnose('close', () => {
|
|
125
|
+
this.#alive[index] = false
|
|
126
|
+
this.#down[index].resolve()
|
|
127
|
+
this.#diagnostics.emit(LOST, index)
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
connection.diagnose('open', () => {
|
|
131
|
+
this.#alive[index] = true
|
|
132
|
+
this.#down[index] = new Promex()
|
|
133
|
+
})
|
|
121
134
|
}
|
|
122
135
|
|
|
123
136
|
/**
|
|
@@ -141,6 +154,7 @@ class Channel {
|
|
|
141
154
|
#pipe (channel) {
|
|
142
155
|
for (const event of events.channel) {
|
|
143
156
|
if (event === RETURN) continue // returns are retried before being reported
|
|
157
|
+
if (event === LOST) continue // a shard is lost with its connection, not its channel
|
|
144
158
|
|
|
145
159
|
channel.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args, channel.index))
|
|
146
160
|
}
|
|
@@ -299,13 +313,28 @@ class Channel {
|
|
|
299
313
|
return promises
|
|
300
314
|
}
|
|
301
315
|
|
|
316
|
+
/**
|
|
317
|
+
* The channels of the shards that are known to be connected, falling back to the
|
|
318
|
+
* whole pool while none is, so that a publish is attempted rather than dropped.
|
|
319
|
+
*
|
|
320
|
+
* @return {comq.Channel[]}
|
|
321
|
+
*/
|
|
322
|
+
#reachable () {
|
|
323
|
+
const reachable = this.#pool.filter((channel) => this.#alive[channel.index] !== false)
|
|
324
|
+
|
|
325
|
+
return reachable.length === 0 ? this.#pool : reachable
|
|
326
|
+
}
|
|
327
|
+
|
|
302
328
|
/**
|
|
303
329
|
* @param {(channel: comq.Channel) => void} fn
|
|
304
330
|
*/
|
|
305
331
|
async #one (fn) {
|
|
306
332
|
if (this.#pool.length === 0) await this.#recovery
|
|
307
333
|
|
|
308
|
-
|
|
334
|
+
// publishing to a lost shard is silently accepted by the destroyed socket, so
|
|
335
|
+
// it must be avoided rather than retried: nothing would report the loss twice
|
|
336
|
+
const pool = this.#reachable()
|
|
337
|
+
const channel = pool[Math.floor(Math.random() * pool.length)]
|
|
309
338
|
|
|
310
339
|
try {
|
|
311
340
|
return await fn(channel)
|
|
@@ -333,6 +362,7 @@ async function create (connections, type) {
|
|
|
333
362
|
const DEFAULT = ''
|
|
334
363
|
const RETURN = 'return'
|
|
335
364
|
const RETURN_HEADER = 'x-return'
|
|
365
|
+
const LOST = 'lost'
|
|
336
366
|
|
|
337
367
|
function noop () {}
|
|
338
368
|
|
package/test/io.request.test.js
CHANGED
|
@@ -207,8 +207,125 @@ describe('send', () => {
|
|
|
207
207
|
|
|
208
208
|
expect(requests.send).toHaveBeenCalledTimes(2)
|
|
209
209
|
})
|
|
210
|
+
|
|
211
|
+
it('should resend unanswered Requests when a shard is lost', async () => {
|
|
212
|
+
jest.clearAllMocks()
|
|
213
|
+
|
|
214
|
+
connection = mock.connection(true)
|
|
215
|
+
io = new IO(connection)
|
|
216
|
+
|
|
217
|
+
promise = io.request(queue, payload)
|
|
218
|
+
|
|
219
|
+
// allows initializers to run
|
|
220
|
+
await immediate()
|
|
221
|
+
|
|
222
|
+
requests = await findChannel('request')
|
|
223
|
+
|
|
224
|
+
expect(requests.diagnose).toHaveBeenCalledWith('lost', expect.any(Function))
|
|
225
|
+
|
|
226
|
+
await lose()
|
|
227
|
+
|
|
228
|
+
expect(requests.send).toHaveBeenCalledTimes(2)
|
|
229
|
+
})
|
|
230
|
+
|
|
231
|
+
it('should not interrupt a confirmed reply stream when a shard is lost', async () => {
|
|
232
|
+
jest.clearAllMocks()
|
|
233
|
+
|
|
234
|
+
connection = mock.connection(true)
|
|
235
|
+
io = new IO(connection)
|
|
236
|
+
|
|
237
|
+
promise = io.request(queue, payload)
|
|
238
|
+
|
|
239
|
+
await immediate()
|
|
240
|
+
|
|
241
|
+
requests = await findChannel('request')
|
|
242
|
+
replies = await findChannel('reply')
|
|
243
|
+
|
|
244
|
+
const correlationId = requests.send.mock.calls[0][2].correlationId
|
|
245
|
+
const callback = replies.consume.mock.calls[0][1]
|
|
246
|
+
|
|
247
|
+
await callback(message(correlationId, 0, 'ok', 'control'))
|
|
248
|
+
|
|
249
|
+
const output = await promise
|
|
250
|
+
|
|
251
|
+
expect(output).toBeInstanceOf(stream.Readable)
|
|
252
|
+
|
|
253
|
+
await lose()
|
|
254
|
+
|
|
255
|
+
// the stream is carried by whichever shard replied, so it keeps flowing
|
|
256
|
+
const chunk = randomBytes(8)
|
|
257
|
+
|
|
258
|
+
await callback(message(correlationId, 1, chunk))
|
|
259
|
+
|
|
260
|
+
expect(output.read()).toStrictEqual(chunk)
|
|
261
|
+
|
|
262
|
+
output.destroy()
|
|
263
|
+
})
|
|
264
|
+
|
|
265
|
+
it('should not resend an answered Request when a shard is lost', async () => {
|
|
266
|
+
jest.clearAllMocks()
|
|
267
|
+
|
|
268
|
+
connection = mock.connection(true)
|
|
269
|
+
io = new IO(connection)
|
|
270
|
+
|
|
271
|
+
promise = io.request(queue, payload)
|
|
272
|
+
|
|
273
|
+
await immediate()
|
|
274
|
+
|
|
275
|
+
requests = await findChannel('request')
|
|
276
|
+
replies = await findChannel('reply')
|
|
277
|
+
|
|
278
|
+
const correlationId = requests.send.mock.calls[0][2].correlationId
|
|
279
|
+
const callback = replies.consume.mock.calls[0][1]
|
|
280
|
+
|
|
281
|
+
const answer = /** @type {comq.amqp.Message} */
|
|
282
|
+
{ content: randomBytes(8), properties: { correlationId } }
|
|
283
|
+
|
|
284
|
+
await callback(answer)
|
|
285
|
+
await promise
|
|
286
|
+
|
|
287
|
+
await lose()
|
|
288
|
+
|
|
289
|
+
expect(requests.send).toHaveBeenCalledTimes(1)
|
|
290
|
+
})
|
|
210
291
|
})
|
|
211
292
|
|
|
293
|
+
/**
|
|
294
|
+
* Fires the listeners the IO has attached to the shard loss diagnostic.
|
|
295
|
+
*
|
|
296
|
+
* @return {Promise<void>}
|
|
297
|
+
*/
|
|
298
|
+
async function lose () {
|
|
299
|
+
const calls = requests.diagnose.mock.calls.filter((call) => call[0] === 'lost')
|
|
300
|
+
|
|
301
|
+
for (const [, listener] of calls) listener(0)
|
|
302
|
+
|
|
303
|
+
await immediate()
|
|
304
|
+
await immediate()
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* @param {string} correlationId
|
|
309
|
+
* @param {number} index
|
|
310
|
+
* @param {any} content
|
|
311
|
+
* @param {string} [type]
|
|
312
|
+
* @return {comq.amqp.Message}
|
|
313
|
+
*/
|
|
314
|
+
function message (correlationId, index, content, type) {
|
|
315
|
+
const raw = Buffer.isBuffer(content)
|
|
316
|
+
|
|
317
|
+
return /** @type {comq.amqp.Message} */ {
|
|
318
|
+
content: raw ? content : encode(content, 'application/json'),
|
|
319
|
+
properties: {
|
|
320
|
+
correlationId,
|
|
321
|
+
type,
|
|
322
|
+
replyTo: queue,
|
|
323
|
+
contentType: raw ? 'application/octet-stream' : 'application/json',
|
|
324
|
+
headers: { index }
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
212
329
|
describe('reply', () => {
|
|
213
330
|
it.each([undefined, 'application/octet-stream'])('should return raw content if encoding is %s',
|
|
214
331
|
async (contentType) => {
|
|
@@ -29,6 +29,58 @@ it('should expose `sharded`', async () => {
|
|
|
29
29
|
expect(channel.sharded).toStrictEqual(true)
|
|
30
30
|
})
|
|
31
31
|
|
|
32
|
+
it('should report a lost shard', async () => {
|
|
33
|
+
channel = await create(connections, type)
|
|
34
|
+
|
|
35
|
+
const lost = /** @type {jest.MockedFunction} */ jest.fn()
|
|
36
|
+
|
|
37
|
+
channel.diagnose('lost', lost)
|
|
38
|
+
|
|
39
|
+
const calls = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
|
|
40
|
+
|
|
41
|
+
expect(calls.length).toBeGreaterThan(0)
|
|
42
|
+
|
|
43
|
+
for (const call of calls) call[1]()
|
|
44
|
+
|
|
45
|
+
expect(lost).toHaveBeenCalledWith(1)
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
it('should not publish to a lost shard', async () => {
|
|
49
|
+
channel = await create(connections, type)
|
|
50
|
+
|
|
51
|
+
const channels = await getCreatedChannels()
|
|
52
|
+
const calls = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
|
|
53
|
+
|
|
54
|
+
for (const call of calls) call[1]()
|
|
55
|
+
|
|
56
|
+
const buffer = randomBytes(8)
|
|
57
|
+
const label = generate()
|
|
58
|
+
|
|
59
|
+
for (let i = 0; i < 20; i++) await channel.send(label, buffer)
|
|
60
|
+
|
|
61
|
+
expect(channels[0].send).toHaveBeenCalledTimes(20)
|
|
62
|
+
expect(channels[1].send).not.toHaveBeenCalled()
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
// a lost shard is not benched, so it is used again as soon as it reconnects
|
|
66
|
+
it('should publish to a recovered shard', async () => {
|
|
67
|
+
channel = await create(connections, type)
|
|
68
|
+
|
|
69
|
+
const channels = await getCreatedChannels()
|
|
70
|
+
const close = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
|
|
71
|
+
const open = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'open')
|
|
72
|
+
|
|
73
|
+
for (const call of close) call[1]()
|
|
74
|
+
for (const call of open) call[1]()
|
|
75
|
+
|
|
76
|
+
const buffer = randomBytes(8)
|
|
77
|
+
const label = generate()
|
|
78
|
+
|
|
79
|
+
for (let i = 0; i < 20; i++) await channel.send(label, buffer)
|
|
80
|
+
|
|
81
|
+
expect(channels[1].send).toHaveBeenCalled()
|
|
82
|
+
})
|
|
83
|
+
|
|
32
84
|
it('should resolve when one of the connections has created a channel', async () => {
|
|
33
85
|
/** @type {toa.generic.Promex[]} */
|
|
34
86
|
const promises = []
|
package/types/io.d.ts
CHANGED
|
@@ -27,11 +27,9 @@ declare namespace comq {
|
|
|
27
27
|
interface IO extends _diagnostics.Diagnosable {
|
|
28
28
|
reply(queue: string, produce: Producer): Promise<void>
|
|
29
29
|
|
|
30
|
-
request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding
|
|
30
|
+
request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply> | Promise<Readable>
|
|
31
31
|
|
|
32
|
-
request
|
|
33
|
-
|
|
34
|
-
request(queue: string, stream: Readable, encoding?: _encoding.Encoding, timeout?: number): Promise<Readable>
|
|
32
|
+
request(queue: string, stream: Readable, encoding?: _encoding.Encoding): Promise<Readable>
|
|
35
33
|
|
|
36
34
|
consume<T = any>(exchange: string, group: string, consumer: Consumer<T>): Promise<void>
|
|
37
35
|
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
'use strict'
|
|
2
|
-
|
|
3
|
-
const { generate } = require('randomstring')
|
|
4
|
-
|
|
5
|
-
const mock = require('./connection.mock')
|
|
6
|
-
const { IO } = require('../source/io')
|
|
7
|
-
|
|
8
|
-
/** @type {comq.IO} */
|
|
9
|
-
let io
|
|
10
|
-
|
|
11
|
-
/** @type {jest.MockedObject<comq.Connection>} */
|
|
12
|
-
let connection
|
|
13
|
-
|
|
14
|
-
const queue = generate()
|
|
15
|
-
const payload = { [generate()]: generate() }
|
|
16
|
-
|
|
17
|
-
beforeEach(() => {
|
|
18
|
-
jest.clearAllMocks()
|
|
19
|
-
|
|
20
|
-
connection = mock.connection()
|
|
21
|
-
io = new IO(connection)
|
|
22
|
-
})
|
|
23
|
-
|
|
24
|
-
it('should reject when request times out', async () => {
|
|
25
|
-
await expect(io.request(queue, payload, 50)).rejects.toMatchObject({
|
|
26
|
-
message: expect.stringContaining('timed out'),
|
|
27
|
-
code: 'ETIMEDOUT'
|
|
28
|
-
})
|
|
29
|
-
})
|
|
30
|
-
|
|
31
|
-
it('should reject when request times out with encoding', async () => {
|
|
32
|
-
await expect(io.request(queue, payload, 'application/json', 50)).rejects.toMatchObject({
|
|
33
|
-
code: 'ETIMEDOUT'
|
|
34
|
-
})
|
|
35
|
-
})
|
|
36
|
-
|
|
37
|
-
it('should time out while send is still pending', async () => {
|
|
38
|
-
connection.createChannel.mockImplementation(async () => hangingChannel())
|
|
39
|
-
|
|
40
|
-
await expect(io.request(queue, payload, 50)).rejects.toMatchObject({
|
|
41
|
-
code: 'ETIMEDOUT'
|
|
42
|
-
})
|
|
43
|
-
})
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* @returns {jest.MockedObject<comq.Channel>}
|
|
47
|
-
*/
|
|
48
|
-
function hangingChannel () {
|
|
49
|
-
return /** @type {jest.MockedObject<comq.Channel>} */ ({
|
|
50
|
-
sharded: false,
|
|
51
|
-
consume: jest.fn(async () => undefined),
|
|
52
|
-
send: jest.fn(() => new Promise(() => {})),
|
|
53
|
-
deliver: jest.fn(async () => undefined),
|
|
54
|
-
fire: jest.fn(async () => undefined),
|
|
55
|
-
subscribe: jest.fn(async () => undefined),
|
|
56
|
-
publish: jest.fn(async () => undefined),
|
|
57
|
-
diagnose: jest.fn(),
|
|
58
|
-
seal: jest.fn(async () => undefined),
|
|
59
|
-
recover: jest.fn(async () => undefined)
|
|
60
|
-
})
|
|
61
|
-
}
|