comq 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/discussions/amqp-1.0.md +127 -0
- package/features/heartbeat.feature +16 -0
- package/features/rpc.call.feature +23 -1
- package/features/rpc.call.recovery.feature +2 -23
- package/features/rpc.parked.feature +6 -6
- package/features/rpc.shards.retransmission.feature +46 -0
- package/features/shards.feature +1 -1
- package/features/silence.feature +3 -2
- package/features/steps/.types/context.d.ts +6 -1
- package/features/steps/brokers.js +1 -1
- package/features/steps/call.js +90 -30
- package/features/steps/connection.js +23 -0
- package/features/steps/context.js +1 -2
- package/features/steps/events.js +1 -1
- package/features/steps/hooks.js +0 -1
- package/features/steps/network.js +11 -2
- package/features/steps/networks.js +31 -2
- package/features/steps/parking.js +5 -5
- package/features/steps/retransmission.js +165 -0
- package/features/steps/rpc.js +0 -6
- package/package.json +20 -4
- package/readme.md +46 -46
- package/source/channel.js +3 -1
- package/source/connection.js +10 -90
- package/source/io.js +106 -25
- package/source/shards/channel.js +73 -12
- package/test/amqplib.mock.js +1 -1
- package/test/connection.mock.js +3 -2
- package/test/connection.test.js +16 -78
- package/test/{io.request.terms.test.js → io.call.terms.test.js} +31 -30
- package/test/io.call.test.js +5 -13
- package/test/io.reply.test.js +5 -3
- package/test/io.request.test.js +277 -2
- package/test/memory.test.js +1 -3
- package/test/shards/channel.test.js +168 -2
- package/types/amqp.d.ts +1 -1
- package/types/channel.d.ts +14 -5
- package/types/io.d.ts +18 -8
- package/features/rpc.timeout.feature +0 -19
- package/source/batch.js +0 -62
- package/test/batch.test.js +0 -94
|
@@ -48,9 +48,19 @@ class Network {
|
|
|
48
48
|
/**
|
|
49
49
|
* Only the tunnels that are already open go silent: a connection established
|
|
50
50
|
* afterwards is forwarded as usual, just as it is once a machine is awake.
|
|
51
|
+
*
|
|
52
|
+
* The broker notices a silent connection by its heartbeat as soon as the client does. One
|
|
53
|
+
* that is to be held by the broker for as long as the scenario needs is kept heard from:
|
|
54
|
+
* heartbeats are sent to the broker in the client's stead, until the network lets it go.
|
|
55
|
+
*
|
|
56
|
+
* @param {boolean} [heard] whether the broker goes on hearing heartbeats
|
|
51
57
|
*/
|
|
52
|
-
silence () {
|
|
53
|
-
for (const tunnel of this.#tunnels)
|
|
58
|
+
silence (heard = false) {
|
|
59
|
+
for (const tunnel of this.#tunnels) {
|
|
60
|
+
tunnel.silent = true
|
|
61
|
+
|
|
62
|
+
if (heard) tunnel.heartbeat ??= setInterval(() => tunnel.upstream.write(HEARTBEAT), HEARTBEAT_MS)
|
|
63
|
+
}
|
|
54
64
|
}
|
|
55
65
|
|
|
56
66
|
/**
|
|
@@ -65,6 +75,17 @@ class Network {
|
|
|
65
75
|
this.#refusing = false
|
|
66
76
|
}
|
|
67
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Takes the broker away from whoever is connected through this network: the tunnels that are
|
|
80
|
+
* open are closed at both ends, and a connection attempted from now on is refused until the
|
|
81
|
+
* network admits it.
|
|
82
|
+
*/
|
|
83
|
+
cut () {
|
|
84
|
+
this.refuse()
|
|
85
|
+
|
|
86
|
+
for (const tunnel of this.#tunnels) this.#collapse(tunnel)
|
|
87
|
+
}
|
|
88
|
+
|
|
68
89
|
/**
|
|
69
90
|
* Closes the tunnels that went silent, at both ends, so the broker learns that the
|
|
70
91
|
* connection it was still holding is gone — the moment its heartbeat would otherwise tell it.
|
|
@@ -117,10 +138,18 @@ class Network {
|
|
|
117
138
|
* @param {comq.features.Tunnel} tunnel
|
|
118
139
|
*/
|
|
119
140
|
#collapse (tunnel) {
|
|
141
|
+
clearInterval(tunnel.heartbeat)
|
|
142
|
+
|
|
120
143
|
this.#tunnels.delete(tunnel)
|
|
121
144
|
tunnel.client.destroy()
|
|
122
145
|
tunnel.upstream.destroy()
|
|
123
146
|
}
|
|
124
147
|
}
|
|
125
148
|
|
|
149
|
+
/** An AMQP heartbeat frame: type 8, channel 0, no payload, and the frame end. */
|
|
150
|
+
const HEARTBEAT = Buffer.from([8, 0, 0, 0, 0, 0, 0, 0xCE])
|
|
151
|
+
|
|
152
|
+
/** Often enough for the shortest heartbeat a scenario asks for. */
|
|
153
|
+
const HEARTBEAT_MS = 250
|
|
154
|
+
|
|
126
155
|
exports.Network = Network
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const assert = require('node:assert')
|
|
4
|
-
const amqplib = require('amqplib')
|
|
4
|
+
const amqplib = require('@toa.io/amqplib')
|
|
5
5
|
const { Given, When, Then } = require('@cucumber/cucumber')
|
|
6
6
|
|
|
7
7
|
const { getAddress, USER, PASSWORD } = require('./brokers')
|
|
@@ -58,7 +58,7 @@ Then('a message parked from the {token} exchange is in the parking queue',
|
|
|
58
58
|
* Runs `read` against a connection of its own, so that what it does to the parking
|
|
59
59
|
* queue is not what comq is doing to it.
|
|
60
60
|
*
|
|
61
|
-
* @param {(channel: import('amqplib').Channel) => Promise<void>} read
|
|
61
|
+
* @param {(channel: import('@toa.io/amqplib').Channel) => Promise<void>} read
|
|
62
62
|
*/
|
|
63
63
|
async function reading (read) {
|
|
64
64
|
const connection = await amqplib.connect(`amqp://${USER}:${PASSWORD}@${getAddress(0)}`)
|
|
@@ -77,9 +77,9 @@ async function reading (read) {
|
|
|
77
77
|
* every scenario before this one left in it, so everything that does not match is given
|
|
78
78
|
* back; the match is left unacknowledged, for the caller to take or to return.
|
|
79
79
|
*
|
|
80
|
-
* @param {import('amqplib').Channel} channel
|
|
81
|
-
* @param {(message: import('amqplib').GetMessage) => boolean} matches
|
|
82
|
-
* @returns {Promise<import('amqplib').GetMessage | undefined>}
|
|
80
|
+
* @param {import('@toa.io/amqplib').Channel} channel
|
|
81
|
+
* @param {(message: import('@toa.io/amqplib').GetMessage) => boolean} matches
|
|
82
|
+
* @returns {Promise<import('@toa.io/amqplib').GetMessage | undefined>}
|
|
83
83
|
*/
|
|
84
84
|
async function pick (channel, matches) {
|
|
85
85
|
const deadline = Date.now() + DEADLINE
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const assert = require('node:assert')
|
|
4
|
+
const { Promex } = require('promex')
|
|
5
|
+
const { Given, When, Then, After } = require('@cucumber/cucumber')
|
|
6
|
+
|
|
7
|
+
const { connect } = require('../../')
|
|
8
|
+
const { getAddress, USER, PASSWORD, BROKERS_AMOUNT } = require('./brokers')
|
|
9
|
+
const { timeout, until } = require('../../test/helpers')
|
|
10
|
+
|
|
11
|
+
Given('a producer on another connection counting the requests to the {token} queue, answering when told',
|
|
12
|
+
/**
|
|
13
|
+
* Connected to every broker directly, so that nothing done to the consumer's network reaches it.
|
|
14
|
+
* It takes every Request as it comes, counts how many times it has been handed each of them, and
|
|
15
|
+
* holds the replies back until it is told to answer: whatever the consumer re-sends by then is
|
|
16
|
+
* counted, however quickly the first attempt would have been answered.
|
|
17
|
+
*
|
|
18
|
+
* @param {string} queue
|
|
19
|
+
* @this {comq.features.Context}
|
|
20
|
+
*/
|
|
21
|
+
async function (queue) {
|
|
22
|
+
const urls = Array.from({ length: BROKERS_AMOUNT }, (_, n) =>
|
|
23
|
+
`amqp://${USER}:${PASSWORD}@${getAddress(n)}`)
|
|
24
|
+
|
|
25
|
+
this.producer = await connect(...urls)
|
|
26
|
+
this.executions = new Map()
|
|
27
|
+
this.answering = new Promex()
|
|
28
|
+
|
|
29
|
+
await this.producer.reply(queue, async ({ id }) => {
|
|
30
|
+
this.executions.set(id, (this.executions.get(id) ?? 0) + 1)
|
|
31
|
+
|
|
32
|
+
await this.answering
|
|
33
|
+
|
|
34
|
+
return id
|
|
35
|
+
})
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
When('the producer answers',
|
|
39
|
+
/**
|
|
40
|
+
* @this {comq.features.Context}
|
|
41
|
+
*/
|
|
42
|
+
function () {
|
|
43
|
+
this.answering.resolve()
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
When('the consumer sends {number} numbered requests to the {token} queue',
|
|
47
|
+
/**
|
|
48
|
+
* @param {number} amount
|
|
49
|
+
* @param {string} queue
|
|
50
|
+
* @this {comq.features.Context}
|
|
51
|
+
*/
|
|
52
|
+
function (amount, queue) {
|
|
53
|
+
this.numbered = Array.from({ length: amount }, (_, id) => {
|
|
54
|
+
const reply = this.io.request(queue, { id })
|
|
55
|
+
|
|
56
|
+
// asserted on by a later step, and an unhandled rejection would end the run before it
|
|
57
|
+
reply.catch(() => undefined)
|
|
58
|
+
|
|
59
|
+
return reply
|
|
60
|
+
})
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
When('the consumer\'s connection to broker {number} is cut',
|
|
64
|
+
/**
|
|
65
|
+
* The broker is taken away from the consumer alone: the connection closes at both ends, and the
|
|
66
|
+
* consumer is refused until it is restored. Returns once the consumer has noticed, so that what
|
|
67
|
+
* it sends next goes to the other broker.
|
|
68
|
+
*
|
|
69
|
+
* @param {number} broker
|
|
70
|
+
* @this {comq.features.Context}
|
|
71
|
+
*/
|
|
72
|
+
async function (broker) {
|
|
73
|
+
let lost = false
|
|
74
|
+
|
|
75
|
+
this.io.diagnose('close', (_error, index) => { if (index === broker) lost = true })
|
|
76
|
+
this.networks[broker].cut()
|
|
77
|
+
|
|
78
|
+
assert.equal(await until(() => lost, 10_000), true,
|
|
79
|
+
`The consumer's connection to broker ${broker} was not lost`)
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
When('the consumer\'s connection to broker {number} is restored',
|
|
83
|
+
/**
|
|
84
|
+
* Returns once the consumer's channels to that broker have recovered, which is when the
|
|
85
|
+
* requests it may have lost with it are re-sent.
|
|
86
|
+
*
|
|
87
|
+
* @param {number} broker
|
|
88
|
+
* @this {comq.features.Context}
|
|
89
|
+
*/
|
|
90
|
+
async function (broker) {
|
|
91
|
+
const recovered = new Set()
|
|
92
|
+
|
|
93
|
+
this.io.diagnose('recover', (type, index) => { if (index === broker) recovered.add(type) })
|
|
94
|
+
this.networks[broker].admit()
|
|
95
|
+
|
|
96
|
+
const restored = () => recovered.has('request') && recovered.has('reply')
|
|
97
|
+
|
|
98
|
+
assert.equal(await until(restored, 20_000), true,
|
|
99
|
+
`The consumer's channels to broker ${broker} have not recovered`)
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
When('the consumer\'s network to broker {number} goes silent',
|
|
103
|
+
/**
|
|
104
|
+
* @param {number} broker
|
|
105
|
+
* @this {comq.features.Context}
|
|
106
|
+
*/
|
|
107
|
+
function (broker) {
|
|
108
|
+
this.networks[broker].silence()
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
Then('every numbered request is answered within {number} seconds',
|
|
112
|
+
/**
|
|
113
|
+
* @param {number} seconds
|
|
114
|
+
* @this {comq.features.Context}
|
|
115
|
+
*/
|
|
116
|
+
async function (seconds) {
|
|
117
|
+
const answered = new Array(this.numbered.length).fill(false)
|
|
118
|
+
|
|
119
|
+
this.numbered.forEach((reply, id) => reply.then(() => { answered[id] = true }, () => undefined))
|
|
120
|
+
|
|
121
|
+
await until(() => answered.every(Boolean), seconds * 1000)
|
|
122
|
+
|
|
123
|
+
const unanswered = answered.flatMap((yes, id) => yes ? [] : [id])
|
|
124
|
+
|
|
125
|
+
assert.deepEqual(unanswered, [], `Requests ${unanswered.join(', ')} are not answered`)
|
|
126
|
+
|
|
127
|
+
for (const [id, reply] of this.numbered.entries()) assert.equal(await reply, id, 'Reply mismatch')
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
Then('every numbered request has been executed {word}',
|
|
131
|
+
/**
|
|
132
|
+
* A re-sent Request is taken as soon as it arrives, so a moment is given for any copy still on
|
|
133
|
+
* its way to the producer to be counted.
|
|
134
|
+
*
|
|
135
|
+
* @param {'once' | 'twice'} times
|
|
136
|
+
* @this {comq.features.Context}
|
|
137
|
+
*/
|
|
138
|
+
async function (times) {
|
|
139
|
+
const expected = TIMES[times]
|
|
140
|
+
|
|
141
|
+
await timeout(SETTLE_MS)
|
|
142
|
+
|
|
143
|
+
const executed = this.numbered.map((_, id) => this.executions.get(id) ?? 0)
|
|
144
|
+
const wrong = executed.flatMap((count, id) => count === expected ? [] : [`${id}: ${count}`])
|
|
145
|
+
|
|
146
|
+
assert.deepEqual(wrong, [], `Requests executed other than ${times} (id: times): ${wrong.join(', ')}`)
|
|
147
|
+
})
|
|
148
|
+
|
|
149
|
+
After(
|
|
150
|
+
/**
|
|
151
|
+
* @this {comq.features.Context}
|
|
152
|
+
*/
|
|
153
|
+
async function () {
|
|
154
|
+
this.answering?.resolve()
|
|
155
|
+
await this.producer?.close()
|
|
156
|
+
|
|
157
|
+
this.producer = undefined
|
|
158
|
+
this.executions = undefined
|
|
159
|
+
this.answering = undefined
|
|
160
|
+
this.numbered = undefined
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
const TIMES = { once: 1, twice: 2 }
|
|
164
|
+
|
|
165
|
+
const SETTLE_MS = 1000
|
package/features/steps/rpc.js
CHANGED
|
@@ -88,12 +88,6 @@ Given('idle timeout is set to {number}ms',
|
|
|
88
88
|
global.COMQ_TESTING_IDLE_INTERVAL = value
|
|
89
89
|
})
|
|
90
90
|
|
|
91
|
-
Given('watchdog interval is set to {number}ms with {number}s AMQP heartbeat',
|
|
92
|
-
function (watchdog, heartbeat) {
|
|
93
|
-
global.COMQ_TESTING_WATCHDOG_INTERVAL = watchdog
|
|
94
|
-
global.COMQ_TESTING_AMQP_HEARTBEAT = heartbeat
|
|
95
|
-
})
|
|
96
|
-
|
|
97
91
|
Given('a producer replying {token} queue',
|
|
98
92
|
/**
|
|
99
93
|
* @param {string} queue
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "comq",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.24.1",
|
|
4
4
|
"description": "ComQ",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Artem Gurtovoi",
|
|
@@ -34,7 +34,6 @@
|
|
|
34
34
|
"@semantic-release/npm": "^13.1.2",
|
|
35
35
|
"@semantic-release/release-notes-generator": "^14.1.0",
|
|
36
36
|
"@testcontainers/rabbitmq": "^12.1.0",
|
|
37
|
-
"@types/amqplib": "^0.10.8",
|
|
38
37
|
"@types/jest": "^30.0.0",
|
|
39
38
|
"conventional-changelog-conventionalcommits": "^8.0.0",
|
|
40
39
|
"fast-glob": "^3.3.3",
|
|
@@ -49,13 +48,16 @@
|
|
|
49
48
|
"testcontainers": "^12.0.4"
|
|
50
49
|
},
|
|
51
50
|
"dependencies": {
|
|
52
|
-
"amqplib": "^
|
|
51
|
+
"@toa.io/amqplib": "^0.1.0",
|
|
53
52
|
"promex": "^0.1.1",
|
|
54
53
|
"reretry": "^0.1.0"
|
|
55
54
|
},
|
|
55
|
+
"engines": {
|
|
56
|
+
"node": ">=24.0.0"
|
|
57
|
+
},
|
|
56
58
|
"scripts": {
|
|
57
59
|
"prepare": "husky",
|
|
58
|
-
"test": "standard && jest",
|
|
60
|
+
"test": "standard && NODE_OPTIONS=--experimental-vm-modules jest",
|
|
59
61
|
"features": "cucumber-js --fail-fast -t 'not @manual'",
|
|
60
62
|
"features:fast": "cucumber-js --fail-fast -t 'not @manual and not @heavy'",
|
|
61
63
|
"benchmark": "node benchmarks/run.js",
|
|
@@ -65,5 +67,19 @@
|
|
|
65
67
|
"env": [
|
|
66
68
|
"jest"
|
|
67
69
|
]
|
|
70
|
+
},
|
|
71
|
+
"overrides": {
|
|
72
|
+
"test-exclude": "^8.0.0",
|
|
73
|
+
"rimraf": "^6.1.3",
|
|
74
|
+
"archiver-utils": {
|
|
75
|
+
"glob": "^13.0.6"
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"allowScripts": {
|
|
79
|
+
"@parcel/watcher": false,
|
|
80
|
+
"cpu-features": false,
|
|
81
|
+
"protobufjs": false,
|
|
82
|
+
"ssh2": false,
|
|
83
|
+
"unrs-resolver": false
|
|
68
84
|
}
|
|
69
85
|
}
|
package/readme.md
CHANGED
|
@@ -6,8 +6,8 @@ for distributed, eventually consistent systems running on Node.js.
|
|
|
6
6
|
## Features
|
|
7
7
|
|
|
8
8
|
- [Dynamic topology](#topology)
|
|
9
|
-
- [Request](#request)-[reply](#reply) (RPC)
|
|
10
|
-
- [Addressed requests](#addressed-requests) to the one connection holding a Key
|
|
9
|
+
- [Request](#request)-[reply](#reply) (RPC)
|
|
10
|
+
- [Addressed requests](#addressed-requests) to the one connection holding a Key, with a [timeout](#timeout)
|
|
11
11
|
- Events ([pub](#emission)/[sub](#consumption)), fanned out or [routed](#routing)
|
|
12
12
|
- [Tasks](#tasks)
|
|
13
13
|
- [Pipelines](#pipelines)
|
|
@@ -107,51 +107,22 @@ await io.reply('add_numbers', ({ a, b }) => (a + b))
|
|
|
107
107
|
|
|
108
108
|
## Request
|
|
109
109
|
|
|
110
|
-
`async IO.request(queue: string, payload: any,
|
|
110
|
+
`async IO.request(queue: string, payload: any, encoding?: string): any`
|
|
111
111
|
|
|
112
112
|
Send encoded Request message with `replyTo` and `correlationId` properties set and
|
|
113
|
-
return decoded Reply content. The promise stays pending until the Reply arrives,
|
|
114
|
-
|
|
113
|
+
return decoded Reply content. The promise stays pending until the Reply arrives, however long
|
|
114
|
+
that takes: a Request has no timeout and cannot be withdrawn. One no Producer can answer is
|
|
115
|
+
[parked](#parked-messages), and can still be answered while its caller waits.
|
|
115
116
|
|
|
116
117
|
On the initial call, the queue for Requests is asserted, and so is the one this `IO` receives
|
|
117
118
|
every Reply on.
|
|
118
119
|
|
|
119
|
-
`options` is the encoding, or an object:
|
|
120
|
-
|
|
121
|
-
| Option | Type | Default |
|
|
122
|
-
|------------|---------------|--------------------|
|
|
123
|
-
| `encoding` | `string` | `application/json` |
|
|
124
|
-
| `timeout` | `number`, ms | none |
|
|
125
|
-
| `signal` | `AbortSignal` | none |
|
|
126
|
-
|
|
127
120
|
### Example
|
|
128
121
|
|
|
129
122
|
```javascript
|
|
130
123
|
const sum = await io.request('add_numbers', { a: 1, b: 2 })
|
|
131
124
|
```
|
|
132
125
|
|
|
133
|
-
### Timeout
|
|
134
|
-
|
|
135
|
-
A Request with a `timeout` rejects once it passes, with the `TimeoutError` of
|
|
136
|
-
[`AbortSignal.timeout`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static).
|
|
137
|
-
It is published with that much
|
|
138
|
-
[expiration](https://www.rabbitmq.com/docs/ttl#per-message-ttl-in-publishers), so a Request no
|
|
139
|
-
Producer has taken by then is dropped by the broker and never processed. A Request a Producer has
|
|
140
|
-
already taken is processed to the end, and its Reply is discarded. A Request re-sent after a lost
|
|
141
|
-
connection carries the time it has left.
|
|
142
|
-
|
|
143
|
-
A `signal` rejects the Request with its reason once aborted, within the `timeout` where both are
|
|
144
|
-
given. It ends the wait and leaves the Request where it is: in its queue until the `timeout`
|
|
145
|
-
passes, or, without a `timeout`, until a Producer takes it. A Request abandoned by its `signal` may
|
|
146
|
-
therefore still be processed.
|
|
147
|
-
|
|
148
|
-
A Request that failed and waits for its [next attempt](#retries) loses its expiration on the way
|
|
149
|
-
back to its queue, so it may be processed after its caller has stopped waiting.
|
|
150
|
-
|
|
151
|
-
```javascript
|
|
152
|
-
const sum = await io.request('add_numbers', { a: 1, b: 2 }, { timeout: 5000 })
|
|
153
|
-
```
|
|
154
|
-
|
|
155
126
|
## Consumption
|
|
156
127
|
|
|
157
128
|
`async IO.consume(exchange: string, group?: string, consumer): void`
|
|
@@ -231,7 +202,7 @@ await io.route('records', 'store.customers', { id: 2 }) // not delivered to the
|
|
|
231
202
|
|
|
232
203
|
`async IO.back(exchange: string, key: string, producer): void`
|
|
233
204
|
|
|
234
|
-
`async IO.call(exchange: string, key: string, payload: any, options?: string |
|
|
205
|
+
`async IO.call(exchange: string, key: string, payload: any, options?: string | CallOptions): any`
|
|
235
206
|
|
|
236
207
|
A Request to the one connection holding a Key, where a [Request](#request) goes to whichever
|
|
237
208
|
Producer takes it first.
|
|
@@ -245,8 +216,15 @@ is let go when its connection closes, however it closes — once the broker has
|
|
|
245
216
|
connection that went without a word. `back` returns once a broker holds the Key. The rest of the
|
|
246
217
|
connection works throughout, and a connection that is restored claims its Keys again the same way.
|
|
247
218
|
|
|
248
|
-
`call` publishes the encoded Request to the exchange under the `key` and returns the decoded Reply
|
|
249
|
-
|
|
219
|
+
`call` publishes the encoded Request to the exchange under the `key` and returns the decoded Reply.
|
|
220
|
+
|
|
221
|
+
`options` is the encoding, or an object:
|
|
222
|
+
|
|
223
|
+
| Option | Type | Default |
|
|
224
|
+
|------------|---------------|--------------------|
|
|
225
|
+
| `encoding` | `string` | `application/json` |
|
|
226
|
+
| `timeout` | `number`, ms | none |
|
|
227
|
+
| `signal` | `AbortSignal` | none |
|
|
250
228
|
|
|
251
229
|
A call ends in one of three ways:
|
|
252
230
|
|
|
@@ -272,6 +250,24 @@ same Key can therefore hold it on different shards and both answer calls, and `t
|
|
|
272
250
|
so. A call returned by one shard is published on the next, and refused once every shard has
|
|
273
251
|
returned it.
|
|
274
252
|
|
|
253
|
+
### Timeout
|
|
254
|
+
|
|
255
|
+
A call with a `timeout` rejects once it passes, with the `TimeoutError` of
|
|
256
|
+
[`AbortSignal.timeout`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static).
|
|
257
|
+
It is published with that much
|
|
258
|
+
[expiration](https://www.rabbitmq.com/docs/ttl#per-message-ttl-in-publishers), so a call its holder
|
|
259
|
+
has not taken by then is dropped by the broker and never processed. A call the holder has already
|
|
260
|
+
taken is processed to the end, and its Reply is discarded. A call re-sent after a lost connection
|
|
261
|
+
carries the time it has left.
|
|
262
|
+
|
|
263
|
+
A `signal` rejects the call with its reason once aborted, within the `timeout` where both are
|
|
264
|
+
given. It ends the wait and leaves the call where it is: in its queue until the `timeout` passes,
|
|
265
|
+
or, without a `timeout`, until its holder takes it. A call abandoned by its `signal` may therefore
|
|
266
|
+
still be processed.
|
|
267
|
+
|
|
268
|
+
A call that failed and waits for its [next attempt](#retries) loses its expiration on the way back
|
|
269
|
+
to its queue, so it may be processed after its caller has stopped waiting.
|
|
270
|
+
|
|
275
271
|
### Example
|
|
276
272
|
|
|
277
273
|
```javascript
|
|
@@ -461,8 +457,9 @@ Reconnection attempts will be made indefinitely, with intervals increasing up to
|
|
|
461
457
|
Unless the URL sets one, a 15 second heartbeat is requested, so that a connection that is gone
|
|
462
458
|
without a word, such as after a machine wakes from sleep, is noticed within a minute instead of
|
|
463
459
|
being left to whatever the broker suggests.
|
|
464
|
-
A connection that
|
|
465
|
-
the operating system have reported, since neither is guaranteed to report anything
|
|
460
|
+
A connection that has received nothing for two heartbeats is closed and restored regardless of what
|
|
461
|
+
the broker and the operating system have reported, since neither is guaranteed to report anything
|
|
462
|
+
at all. A connection that is merely idle receives the broker's heartbeats, and is kept.
|
|
466
463
|
Requesting `heartbeat=0` disables both.
|
|
467
464
|
If the broker rejects the connection, for example, due to access being denied, an exception will be thrown.
|
|
468
465
|
Once reconnected, the topology will be recovered, and any unanswered requests and unconfirmed events will be
|
|
@@ -484,6 +481,10 @@ connection is re-established.
|
|
|
484
481
|
|
|
485
482
|
Incoming messages are consumed from all shards.
|
|
486
483
|
|
|
484
|
+
A Reply goes back through the shard its Request arrived on, while that shard is reachable. When a
|
|
485
|
+
shard is lost, the Requests sent through it that are still unanswered are re-sent through the
|
|
486
|
+
others, since their Replies may be lost with it; a Request sent through another shard is not.
|
|
487
|
+
|
|
487
488
|
`async connect(...shards: string[]): IO`
|
|
488
489
|
|
|
489
490
|
Returns an instance of `IO` once connections to the shards are established.
|
|
@@ -661,10 +662,9 @@ The queue grows until somebody drains it, which is deliberate — the alternativ
|
|
|
661
662
|
evidence. Nothing comq does removes it or what it holds. Alert on [`discard`](#diagnostics), and
|
|
662
663
|
do not delete `comq.retry.*` or `comq.parked` on a running system.
|
|
663
664
|
|
|
664
|
-
> **A parked Request is never answered.** A Consumer awaiting its Reply waits
|
|
665
|
-
> [timeout](#timeout)
|
|
666
|
-
>
|
|
667
|
-
> it — and it keeps `replyTo` and `correlationId`, so a Reply can still be produced from it by
|
|
665
|
+
> **A parked Request is never answered.** A Consumer awaiting its Reply waits indefinitely — a
|
|
666
|
+
> call until its [timeout](#timeout) — and with a limited prefetch that can deadlock it. Parking
|
|
667
|
+
> keeps the Request rather than deleting it — and it keeps `replyTo` and `correlationId`, so a Reply can still be produced from it by
|
|
668
668
|
> hand while the caller is alive — but comq itself sends no Reply and reports no error to the
|
|
669
669
|
> caller.
|
|
670
670
|
|
|
@@ -800,8 +800,8 @@ Subscribe to one of the diagnostic events:
|
|
|
800
800
|
an argument.
|
|
801
801
|
- `drain`: back pressure is removed from a channel. Channel type is passed.
|
|
802
802
|
- `remove`: channel is removed from the [pool](#sharded-connection), having failed to publish.
|
|
803
|
-
- `lost`: a shard has lost its connection, hence the requests
|
|
804
|
-
re-sent. Channel type is passed.
|
|
803
|
+
- `lost`: a shard has lost its connection, hence the requests sent through it that are awaiting
|
|
804
|
+
their replies are re-sent. Channel type is passed.
|
|
805
805
|
- `recover`: channel's topology is recovered. Channel type is passed.
|
|
806
806
|
- `discard`: message is [parked](#parked-messages), having run out of attempts. Channel type,
|
|
807
807
|
raw [amqp message object](https://amqp-node.github.io/amqplib/channel_api.html#channel_consume)
|
package/source/channel.js
CHANGED
|
@@ -70,7 +70,9 @@ class Channel {
|
|
|
70
70
|
this.#failfast = index !== undefined
|
|
71
71
|
this.#release = release
|
|
72
72
|
|
|
73
|
-
|
|
73
|
+
// a shard's publish fails rather than waits for its connection to come back: the pool
|
|
74
|
+
// fails it over to another shard, and whoever published it is told where it went
|
|
75
|
+
if (this.#failfast) failsafe.disable(this.send, this.publish, this.route)
|
|
74
76
|
}
|
|
75
77
|
|
|
76
78
|
async create () {
|
package/source/connection.js
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const { setTimeout: delay } = require('node:timers/promises')
|
|
4
|
-
const amqp = require('amqplib')
|
|
4
|
+
const amqp = require('@toa.io/amqplib')
|
|
5
5
|
const { Promex } = require('promex')
|
|
6
6
|
const { retry } = require('reretry')
|
|
7
7
|
|
|
8
8
|
const { failsafe } = require('./attributes')
|
|
9
|
-
const { batch } = require('./batch')
|
|
10
9
|
const presets = require('./topology')
|
|
11
10
|
const channels = require('./channel')
|
|
12
11
|
const emitter = require('./emitter')
|
|
@@ -36,18 +35,9 @@ class Connection {
|
|
|
36
35
|
/** @type {boolean} */
|
|
37
36
|
#closed = false
|
|
38
37
|
|
|
39
|
-
/** @type {NodeJS.Timeout | null} */
|
|
40
|
-
#heartbeatTimer = null
|
|
41
|
-
|
|
42
38
|
/** @type {comq.topology.Overrides} */
|
|
43
39
|
#overrides
|
|
44
40
|
|
|
45
|
-
/** @type {import('node:net').Socket | null} */
|
|
46
|
-
#heartbeatSocket = null
|
|
47
|
-
|
|
48
|
-
/** @type {(() => void) | null} */
|
|
49
|
-
#heartbeatReset = null
|
|
50
|
-
|
|
51
41
|
#diagnostics = emitter.create()
|
|
52
42
|
|
|
53
43
|
/**
|
|
@@ -148,9 +138,6 @@ class Connection {
|
|
|
148
138
|
connection.on('close', (error) => this.#close(connection, error))
|
|
149
139
|
this.#connection = connection
|
|
150
140
|
|
|
151
|
-
batch(connection.connection?.stream)
|
|
152
|
-
|
|
153
|
-
this.#armWatchdog(connection)
|
|
154
141
|
this.#diagnostics.emit('open')
|
|
155
142
|
|
|
156
143
|
try {
|
|
@@ -180,11 +167,14 @@ class Connection {
|
|
|
180
167
|
#close = (connection, error) => {
|
|
181
168
|
if (this.#connection !== connection) return
|
|
182
169
|
|
|
183
|
-
this.#disarmWatchdog()
|
|
184
170
|
this.#diagnostics.emit('close', error)
|
|
185
171
|
connection.removeAllListeners()
|
|
186
172
|
this.#connection = undefined
|
|
187
173
|
|
|
174
|
+
// amqplib only ends the socket of a connection it has given up on, and a peer that has gone
|
|
175
|
+
// silent never ends its side, which would leave the socket to the kernel's keepalive
|
|
176
|
+
if (error !== undefined) connection.connection?.stream?.destroy()
|
|
177
|
+
|
|
188
178
|
if (error !== undefined && !this.#closed) {
|
|
189
179
|
this.#reopen().catch((exception) => this.#diagnostics.emit('error', exception))
|
|
190
180
|
}
|
|
@@ -227,10 +217,7 @@ class Connection {
|
|
|
227
217
|
* @param {comq.amqp.Connection} connection
|
|
228
218
|
*/
|
|
229
219
|
#drop (connection) {
|
|
230
|
-
if (this.#connection === connection)
|
|
231
|
-
this.#disarmWatchdog()
|
|
232
|
-
this.#connection = undefined
|
|
233
|
-
}
|
|
220
|
+
if (this.#connection === connection) this.#connection = undefined
|
|
234
221
|
|
|
235
222
|
connection.removeAllListeners()
|
|
236
223
|
|
|
@@ -297,46 +284,6 @@ class Connection {
|
|
|
297
284
|
}
|
|
298
285
|
}
|
|
299
286
|
|
|
300
|
-
/**
|
|
301
|
-
* @param {comq.amqp.Connection} connection
|
|
302
|
-
* @param {number} [timeoutMs]
|
|
303
|
-
*/
|
|
304
|
-
#armWatchdog (connection, timeoutMs = tolerance(connection)) {
|
|
305
|
-
this.#disarmWatchdog()
|
|
306
|
-
|
|
307
|
-
const socket = connection.connection?.stream
|
|
308
|
-
|
|
309
|
-
if (socket === undefined || timeoutMs === undefined) return
|
|
310
|
-
|
|
311
|
-
const reset = () => {
|
|
312
|
-
clearTimeout(this.#heartbeatTimer)
|
|
313
|
-
// destroying a socket without an error tells amqplib nothing: it only
|
|
314
|
-
// listens for 'error' and 'end', so a bare destroy() leaves the connection
|
|
315
|
-
// silently dead — no 'close' event, no recovery, every pending operation
|
|
316
|
-
// hanging forever
|
|
317
|
-
this.#heartbeatTimer = setTimeout(() => socket.destroy(silence()), timeoutMs)
|
|
318
|
-
this.#heartbeatTimer.unref()
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
this.#heartbeatSocket = socket
|
|
322
|
-
this.#heartbeatReset = reset
|
|
323
|
-
|
|
324
|
-
reset()
|
|
325
|
-
socket.on('data', reset)
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
#disarmWatchdog () {
|
|
329
|
-
clearTimeout(this.#heartbeatTimer)
|
|
330
|
-
this.#heartbeatTimer = null
|
|
331
|
-
|
|
332
|
-
// otherwise a byte arriving on a replaced socket rearms the watchdog of a
|
|
333
|
-
// connection that is already gone, leaving the current one unguarded
|
|
334
|
-
this.#heartbeatSocket?.off('data', this.#heartbeatReset)
|
|
335
|
-
|
|
336
|
-
this.#heartbeatSocket = null
|
|
337
|
-
this.#heartbeatReset = null
|
|
338
|
-
}
|
|
339
|
-
|
|
340
287
|
#transient (exception) {
|
|
341
288
|
if (this.#running) return true
|
|
342
289
|
if (TRANSIENT_CODES.has(exception.code)) return true
|
|
@@ -351,6 +298,10 @@ class Connection {
|
|
|
351
298
|
* to suggest one that leaves a connection lost for minutes before anything
|
|
352
299
|
* notices, and RabbitMQ suggests 60 by default.
|
|
353
300
|
*
|
|
301
|
+
* It is amqplib that tells a connection gone silent: one that has received nothing
|
|
302
|
+
* for two heartbeats is closed with an error, whatever the broker and the operating
|
|
303
|
+
* system have reported, which is what restores it.
|
|
304
|
+
*
|
|
354
305
|
* @type {number}
|
|
355
306
|
*/
|
|
356
307
|
const HEARTBEAT_S = 15
|
|
@@ -358,15 +309,6 @@ const HEARTBEAT_S = 15
|
|
|
358
309
|
/** @type {number} */
|
|
359
310
|
const KEEPALIVE_MS = 10_000
|
|
360
311
|
|
|
361
|
-
/** How many heartbeats a connection may miss before it is destroyed. */
|
|
362
|
-
const MISSED_HEARTBEATS = 3
|
|
363
|
-
|
|
364
|
-
/** @type {number} */
|
|
365
|
-
const WATCHDOG_MIN_MS = 15_000
|
|
366
|
-
|
|
367
|
-
/** The watchdog interval used when the negotiated heartbeat is unknown. */
|
|
368
|
-
const WATCHDOG_MS = 60_000
|
|
369
|
-
|
|
370
312
|
/** @type {number} */
|
|
371
313
|
const CONNECT_MS = 30_000
|
|
372
314
|
|
|
@@ -425,28 +367,6 @@ function heartbeaten (url) {
|
|
|
425
367
|
return url + (url.includes('?') ? '&' : '?') + 'heartbeat=' + HEARTBEAT_S
|
|
426
368
|
}
|
|
427
369
|
|
|
428
|
-
/**
|
|
429
|
-
* The watchdog measures silence, so it cannot be shorter than the interval at
|
|
430
|
-
* which a healthy broker is expected to say something, which is a heartbeat
|
|
431
|
-
* frame every half a heartbeat. A connection that has agreed to no heartbeats
|
|
432
|
-
* says nothing at all while it is idle, leaving nothing to measure.
|
|
433
|
-
*
|
|
434
|
-
* @param {comq.amqp.Connection} connection
|
|
435
|
-
* @return {number | undefined}
|
|
436
|
-
*/
|
|
437
|
-
function tolerance (connection) {
|
|
438
|
-
const override = global.COMQ_TESTING_WATCHDOG_INTERVAL
|
|
439
|
-
|
|
440
|
-
if (override !== undefined) return override
|
|
441
|
-
|
|
442
|
-
const heartbeat = connection.connection?.heartbeat
|
|
443
|
-
|
|
444
|
-
if (heartbeat === undefined) return WATCHDOG_MS
|
|
445
|
-
if (heartbeat === 0) return undefined
|
|
446
|
-
|
|
447
|
-
return Math.max(heartbeat * MISSED_HEARTBEATS * 1000, WATCHDOG_MIN_MS)
|
|
448
|
-
}
|
|
449
|
-
|
|
450
370
|
/**
|
|
451
371
|
* @return {Error}
|
|
452
372
|
*/
|