comq 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/discussions/amqp-1.0.md +127 -0
- package/discussions/one-parking-queue.md +63 -0
- package/discussions/one-reply-queue.md +44 -0
- package/discussions/poison-message-handling.md +2 -1
- package/docs/headers.md +1 -1
- package/features/parking.feature +32 -0
- package/features/rpc.call.feature +23 -1
- package/features/rpc.call.recovery.feature +0 -21
- package/features/rpc.parked.feature +6 -6
- package/features/rpc.replies.feature +26 -0
- package/features/steps/brokers.js +25 -1
- package/features/steps/call.js +90 -30
- package/features/steps/events.js +1 -1
- package/features/steps/parked.js +7 -18
- package/features/steps/parking.js +113 -0
- package/features/steps/poison.js +17 -19
- package/features/steps/replies.js +35 -0
- package/package.json +6 -4
- package/readme.md +54 -75
- package/source/.io/ReplyPipe.js +16 -1
- package/source/channel.js +19 -27
- package/source/connection.js +1 -4
- package/source/index.js +0 -2
- package/source/io.js +48 -59
- package/test/ReplyPipe.test.js +40 -0
- package/test/amqplib.mock.js +1 -1
- package/test/channel.test.js +29 -18
- package/test/connection.test.js +1 -1
- package/test/{io.request.terms.test.js → io.call.terms.test.js} +31 -30
- package/test/io.call.test.js +0 -10
- package/test/io.request.test.js +26 -3
- package/test/memory.test.js +1 -1
- package/types/amqp.d.ts +1 -1
- package/types/channel.d.ts +1 -1
- package/types/index.d.ts +0 -6
- package/types/io.d.ts +9 -14
- package/features/closing.feature +0 -32
- package/features/rpc.timeout.feature +0 -19
- package/features/steps/closing.js +0 -109
- package/source/abandoned.js +0 -15
- package/source/batch.js +0 -62
- package/test/batch.test.js +0 -94
package/features/steps/parked.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const assert = require('node:assert')
|
|
4
|
-
const amqplib = require('amqplib')
|
|
5
4
|
const { Given, When, Then } = require('@cucumber/cucumber')
|
|
6
5
|
|
|
7
|
-
const { getAddress, USER, PASSWORD } = require('./brokers')
|
|
8
6
|
const { until, timeout } = require('../../test/helpers')
|
|
7
|
+
const { reading, pick } = require('./parking')
|
|
9
8
|
|
|
10
9
|
Given('a producer failing every request to the {token} queue',
|
|
11
10
|
/**
|
|
@@ -52,18 +51,9 @@ When('the parked request from the {token} queue is answered by hand',
|
|
|
52
51
|
* @this {comq.features.Context}
|
|
53
52
|
*/
|
|
54
53
|
async function (queue) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
try {
|
|
59
|
-
let parked
|
|
60
|
-
|
|
61
|
-
await channel.consume('comq.parked.' + queue, (message) => {
|
|
62
|
-
parked = message
|
|
63
|
-
channel.ack(message)
|
|
64
|
-
})
|
|
65
|
-
|
|
66
|
-
await until(() => parked !== undefined)
|
|
54
|
+
await reading(async (channel) => {
|
|
55
|
+
const parked = await pick(channel,
|
|
56
|
+
(one) => one.properties.headers?.['x-comq-queue'] === queue)
|
|
67
57
|
|
|
68
58
|
assert.notEqual(parked, undefined, `Nothing was parked from ${queue}`)
|
|
69
59
|
|
|
@@ -74,15 +64,14 @@ When('the parked request from the {token} queue is answered by hand',
|
|
|
74
64
|
|
|
75
65
|
this.answer = { answered: true, was: JSON.parse(parked.content.toString()) }
|
|
76
66
|
|
|
67
|
+
channel.ack(parked)
|
|
68
|
+
|
|
77
69
|
channel.sendToQueue(replyTo, Buffer.from(JSON.stringify(this.answer)),
|
|
78
70
|
{ correlationId, contentType })
|
|
79
71
|
|
|
80
72
|
// let the frame reach the broker before the connection goes
|
|
81
73
|
await timeout(100)
|
|
82
|
-
}
|
|
83
|
-
await channel.close()
|
|
84
|
-
await connection.close()
|
|
85
|
-
}
|
|
74
|
+
})
|
|
86
75
|
})
|
|
87
76
|
|
|
88
77
|
Then('the caller receives the reply',
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const assert = require('node:assert')
|
|
4
|
+
const amqplib = require('@toa.io/amqplib')
|
|
5
|
+
const { Given, When, Then } = require('@cucumber/cucumber')
|
|
6
|
+
|
|
7
|
+
const { getAddress, USER, PASSWORD } = require('./brokers')
|
|
8
|
+
const { timeout } = require('../../test/helpers')
|
|
9
|
+
|
|
10
|
+
Given('(that )groupless events from the {token} exchange are causing exceptions',
|
|
11
|
+
/**
|
|
12
|
+
* @param {string} exchange
|
|
13
|
+
* @this {comq.features.Context}
|
|
14
|
+
*/
|
|
15
|
+
async function (exchange) {
|
|
16
|
+
await this.io.consume(exchange, (payload, properties) => {
|
|
17
|
+
this.attempts.push(properties.headers?.['x-comq-attempt'] ?? 1)
|
|
18
|
+
|
|
19
|
+
throw new Error('Expected exception')
|
|
20
|
+
})
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
When('the connection is closed',
|
|
24
|
+
/**
|
|
25
|
+
* @this {comq.features.Context}
|
|
26
|
+
*/
|
|
27
|
+
async function () {
|
|
28
|
+
await this.disconnect()
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
Then('a message parked from the {token} exchange is in the parking queue',
|
|
32
|
+
/**
|
|
33
|
+
* @param {string} exchange
|
|
34
|
+
* @this {comq.features.Context}
|
|
35
|
+
*/
|
|
36
|
+
async function (exchange) {
|
|
37
|
+
await reading(async (channel) => {
|
|
38
|
+
const message = await pick(channel,
|
|
39
|
+
(one) => one.properties.headers?.['x-comq-exchange'] === exchange)
|
|
40
|
+
|
|
41
|
+
assert.notEqual(message, undefined,
|
|
42
|
+
`Nothing in '${PARKED}' came from the '${exchange}' exchange`)
|
|
43
|
+
|
|
44
|
+
const headers = message.properties.headers
|
|
45
|
+
|
|
46
|
+
// a groupless subscriber consumes a name the broker chose, so what is asserted
|
|
47
|
+
// here is that the message says which queue, not which name that is
|
|
48
|
+
assert.notEqual(headers['x-comq-queue'], undefined,
|
|
49
|
+
'The parked message does not name the queue it came from')
|
|
50
|
+
|
|
51
|
+
assert.notEqual(headers['x-comq-reason'], undefined, 'The parked message has no reason')
|
|
52
|
+
|
|
53
|
+
channel.nack(message, false, true)
|
|
54
|
+
})
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Runs `read` against a connection of its own, so that what it does to the parking
|
|
59
|
+
* queue is not what comq is doing to it.
|
|
60
|
+
*
|
|
61
|
+
* @param {(channel: import('@toa.io/amqplib').Channel) => Promise<void>} read
|
|
62
|
+
*/
|
|
63
|
+
async function reading (read) {
|
|
64
|
+
const connection = await amqplib.connect(`amqp://${USER}:${PASSWORD}@${getAddress(0)}`)
|
|
65
|
+
const channel = await connection.createChannel()
|
|
66
|
+
|
|
67
|
+
try {
|
|
68
|
+
await read(channel)
|
|
69
|
+
} finally {
|
|
70
|
+
await channel.close().catch(noop)
|
|
71
|
+
await connection.close().catch(noop)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The first message in the parking queue that `matches`, waited for. One queue holds what
|
|
77
|
+
* every scenario before this one left in it, so everything that does not match is given
|
|
78
|
+
* back; the match is left unacknowledged, for the caller to take or to return.
|
|
79
|
+
*
|
|
80
|
+
* @param {import('@toa.io/amqplib').Channel} channel
|
|
81
|
+
* @param {(message: import('@toa.io/amqplib').GetMessage) => boolean} matches
|
|
82
|
+
* @returns {Promise<import('@toa.io/amqplib').GetMessage | undefined>}
|
|
83
|
+
*/
|
|
84
|
+
async function pick (channel, matches) {
|
|
85
|
+
const deadline = Date.now() + DEADLINE
|
|
86
|
+
|
|
87
|
+
do {
|
|
88
|
+
const taken = []
|
|
89
|
+
|
|
90
|
+
let delivery
|
|
91
|
+
|
|
92
|
+
while ((delivery = await channel.get(PARKED, { noAck: false })) !== false) taken.push(delivery)
|
|
93
|
+
|
|
94
|
+
const match = taken.find(matches)
|
|
95
|
+
|
|
96
|
+
for (const one of taken) if (one !== match) channel.nack(one, false, true)
|
|
97
|
+
|
|
98
|
+
if (match !== undefined) return match
|
|
99
|
+
|
|
100
|
+
await timeout(50)
|
|
101
|
+
} while (Date.now() < deadline)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const PARKED = 'comq.parked'
|
|
105
|
+
|
|
106
|
+
/** The ladder the suite runs on is half a second; this is room for it and the broker. */
|
|
107
|
+
const DEADLINE = 5000
|
|
108
|
+
|
|
109
|
+
function noop () {}
|
|
110
|
+
|
|
111
|
+
exports.PARKED = PARKED
|
|
112
|
+
exports.reading = reading
|
|
113
|
+
exports.pick = pick
|
package/features/steps/poison.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
const assert = require('node:assert')
|
|
4
4
|
const { until } = require('../../test/helpers')
|
|
5
5
|
const { Then } = require('@cucumber/cucumber')
|
|
6
|
+
const { PARKED, reading, pick } = require('./parking')
|
|
6
7
|
|
|
7
8
|
Then('the event is attempted {int} time(s)',
|
|
8
9
|
/**
|
|
@@ -35,32 +36,29 @@ Then('the parked message is kept, and says it came from the {token} exchange',
|
|
|
35
36
|
*/
|
|
36
37
|
async function (exchange) {
|
|
37
38
|
const queue = exchange + '..exceptions' // the group the throwing consumer uses
|
|
38
|
-
const parked = 'comq.parked.' + queue
|
|
39
39
|
|
|
40
|
-
await
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
await until(() => this.parked[queue] !== undefined)
|
|
40
|
+
await reading(async (channel) => {
|
|
41
|
+
const message = await pick(channel,
|
|
42
|
+
(one) => one.properties.headers?.['x-comq-queue'] === queue)
|
|
45
43
|
|
|
46
|
-
|
|
44
|
+
assert.notEqual(message, undefined, `Nothing in '${PARKED}' came from ${queue}`)
|
|
47
45
|
|
|
48
|
-
|
|
46
|
+
const headers = message.properties.headers
|
|
49
47
|
|
|
50
|
-
|
|
48
|
+
assert.notEqual(headers['x-comq-reason'], undefined, 'The parked message has no reason')
|
|
51
49
|
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
// the message came back through the default exchange on every retry, so this
|
|
51
|
+
// is only right if the origin was recorded on the first failure
|
|
52
|
+
assert.equal(headers['x-comq-exchange'], exchange,
|
|
53
|
+
`The parked message says it came from '${headers['x-comq-exchange']}'`)
|
|
54
54
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
55
|
+
// absent on a message parked from its first delivery, exactly as a consumer reads it;
|
|
56
|
+
// derived rather than stated, so it cannot drift from the configured ladder
|
|
57
|
+
assert.equal(headers['x-comq-attempt'] ?? 1, this.attempts.length,
|
|
58
|
+
'The parked message did not have as many attempts as the consumer saw')
|
|
59
59
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
assert.equal(headers['x-comq-attempt'] ?? 1, this.attempts.length,
|
|
63
|
-
'The parked message did not have as many attempts as the consumer saw')
|
|
60
|
+
channel.ack(message)
|
|
61
|
+
})
|
|
64
62
|
})
|
|
65
63
|
|
|
66
64
|
Then('{token} has received {int} event(s)',
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const assert = require('node:assert')
|
|
4
|
+
const { Then } = require('@cucumber/cucumber')
|
|
5
|
+
|
|
6
|
+
const { queues, BROKERS_AMOUNT } = require('./brokers')
|
|
7
|
+
|
|
8
|
+
Then('the broker holds {int} reply queue(s)',
|
|
9
|
+
/**
|
|
10
|
+
* @param {number} count
|
|
11
|
+
*/
|
|
12
|
+
async function (count) {
|
|
13
|
+
await held(0, count)
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
Then('each broker holds {int} reply queue(s)',
|
|
17
|
+
/**
|
|
18
|
+
* @param {number} count
|
|
19
|
+
*/
|
|
20
|
+
async function (count) {
|
|
21
|
+
for (let n = 0; n < BROKERS_AMOUNT; n++) await held(n, count)
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @param {number} broker
|
|
26
|
+
* @param {number} count
|
|
27
|
+
*/
|
|
28
|
+
async function held (broker, count) {
|
|
29
|
+
const replies = (await queues(broker)).filter((name) => name.startsWith(PREFIX))
|
|
30
|
+
|
|
31
|
+
assert.equal(replies.length, count,
|
|
32
|
+
`Broker ${broker} holds ${replies.length} reply queue(s): ${replies.join(', ')}`)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const PREFIX = 'comq.reply'
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "comq",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.24.0",
|
|
4
4
|
"description": "ComQ",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Artem Gurtovoi",
|
|
@@ -34,7 +34,6 @@
|
|
|
34
34
|
"@semantic-release/npm": "^13.1.2",
|
|
35
35
|
"@semantic-release/release-notes-generator": "^14.1.0",
|
|
36
36
|
"@testcontainers/rabbitmq": "^12.1.0",
|
|
37
|
-
"@types/amqplib": "^0.10.8",
|
|
38
37
|
"@types/jest": "^30.0.0",
|
|
39
38
|
"conventional-changelog-conventionalcommits": "^8.0.0",
|
|
40
39
|
"fast-glob": "^3.3.3",
|
|
@@ -49,13 +48,16 @@
|
|
|
49
48
|
"testcontainers": "^12.0.4"
|
|
50
49
|
},
|
|
51
50
|
"dependencies": {
|
|
52
|
-
"amqplib": "^
|
|
51
|
+
"@toa.io/amqplib": "^0.1.0",
|
|
53
52
|
"promex": "^0.1.1",
|
|
54
53
|
"reretry": "^0.1.0"
|
|
55
54
|
},
|
|
55
|
+
"engines": {
|
|
56
|
+
"node": ">=24.0.0"
|
|
57
|
+
},
|
|
56
58
|
"scripts": {
|
|
57
59
|
"prepare": "husky",
|
|
58
|
-
"test": "standard && jest",
|
|
60
|
+
"test": "standard && NODE_OPTIONS=--experimental-vm-modules jest",
|
|
59
61
|
"features": "cucumber-js --fail-fast -t 'not @manual'",
|
|
60
62
|
"features:fast": "cucumber-js --fail-fast -t 'not @manual and not @heavy'",
|
|
61
63
|
"benchmark": "node benchmarks/run.js",
|
package/readme.md
CHANGED
|
@@ -6,8 +6,8 @@ for distributed, eventually consistent systems running on Node.js.
|
|
|
6
6
|
## Features
|
|
7
7
|
|
|
8
8
|
- [Dynamic topology](#topology)
|
|
9
|
-
- [Request](#request)-[reply](#reply) (RPC)
|
|
10
|
-
- [Addressed requests](#addressed-requests) to the one connection holding a Key
|
|
9
|
+
- [Request](#request)-[reply](#reply) (RPC)
|
|
10
|
+
- [Addressed requests](#addressed-requests) to the one connection holding a Key, with a [timeout](#timeout)
|
|
11
11
|
- Events ([pub](#emission)/[sub](#consumption)), fanned out or [routed](#routing)
|
|
12
12
|
- [Tasks](#tasks)
|
|
13
13
|
- [Pipelines](#pipelines)
|
|
@@ -107,21 +107,15 @@ await io.reply('add_numbers', ({ a, b }) => (a + b))
|
|
|
107
107
|
|
|
108
108
|
## Request
|
|
109
109
|
|
|
110
|
-
`async IO.request(queue: string, payload: any,
|
|
110
|
+
`async IO.request(queue: string, payload: any, encoding?: string): any`
|
|
111
111
|
|
|
112
112
|
Send encoded Request message with `replyTo` and `correlationId` properties set and
|
|
113
|
-
return decoded Reply content. The promise stays pending until the Reply arrives,
|
|
114
|
-
|
|
113
|
+
return decoded Reply content. The promise stays pending until the Reply arrives, however long
|
|
114
|
+
that takes: a Request has no timeout and cannot be withdrawn. One no Producer can answer is
|
|
115
|
+
[parked](#parked-messages), and can still be answered while its caller waits.
|
|
115
116
|
|
|
116
|
-
On the initial call,
|
|
117
|
-
|
|
118
|
-
`options` is the encoding, or an object:
|
|
119
|
-
|
|
120
|
-
| Option | Type | Default |
|
|
121
|
-
|------------|---------------|--------------------|
|
|
122
|
-
| `encoding` | `string` | `application/json` |
|
|
123
|
-
| `timeout` | `number`, ms | none |
|
|
124
|
-
| `signal` | `AbortSignal` | none |
|
|
117
|
+
On the initial call, the queue for Requests is asserted, and so is the one this `IO` receives
|
|
118
|
+
every Reply on.
|
|
125
119
|
|
|
126
120
|
### Example
|
|
127
121
|
|
|
@@ -129,28 +123,6 @@ On the initial call, queues for Requests and Replies are asserted.
|
|
|
129
123
|
const sum = await io.request('add_numbers', { a: 1, b: 2 })
|
|
130
124
|
```
|
|
131
125
|
|
|
132
|
-
### Timeout
|
|
133
|
-
|
|
134
|
-
A Request with a `timeout` rejects once it passes, with the `TimeoutError` of
|
|
135
|
-
[`AbortSignal.timeout`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static).
|
|
136
|
-
It is published with that much
|
|
137
|
-
[expiration](https://www.rabbitmq.com/docs/ttl#per-message-ttl-in-publishers), so a Request no
|
|
138
|
-
Producer has taken by then is dropped by the broker and never processed. A Request a Producer has
|
|
139
|
-
already taken is processed to the end, and its Reply is discarded. A Request re-sent after a lost
|
|
140
|
-
connection carries the time it has left.
|
|
141
|
-
|
|
142
|
-
A `signal` rejects the Request with its reason once aborted, within the `timeout` where both are
|
|
143
|
-
given. It ends the wait and leaves the Request where it is: in its queue until the `timeout`
|
|
144
|
-
passes, or, without a `timeout`, until a Producer takes it. A Request abandoned by its `signal` may
|
|
145
|
-
therefore still be processed.
|
|
146
|
-
|
|
147
|
-
A Request that failed and waits for its [next attempt](#retries) loses its expiration on the way
|
|
148
|
-
back to its queue, so it may be processed after its caller has stopped waiting.
|
|
149
|
-
|
|
150
|
-
```javascript
|
|
151
|
-
const sum = await io.request('add_numbers', { a: 1, b: 2 }, { timeout: 5000 })
|
|
152
|
-
```
|
|
153
|
-
|
|
154
126
|
## Consumption
|
|
155
127
|
|
|
156
128
|
`async IO.consume(exchange: string, group?: string, consumer): void`
|
|
@@ -230,7 +202,7 @@ await io.route('records', 'store.customers', { id: 2 }) // not delivered to the
|
|
|
230
202
|
|
|
231
203
|
`async IO.back(exchange: string, key: string, producer): void`
|
|
232
204
|
|
|
233
|
-
`async IO.call(exchange: string, key: string, payload: any, options?: string |
|
|
205
|
+
`async IO.call(exchange: string, key: string, payload: any, options?: string | CallOptions): any`
|
|
234
206
|
|
|
235
207
|
A Request to the one connection holding a Key, where a [Request](#request) goes to whichever
|
|
236
208
|
Producer takes it first.
|
|
@@ -244,8 +216,15 @@ is let go when its connection closes, however it closes — once the broker has
|
|
|
244
216
|
connection that went without a word. `back` returns once a broker holds the Key. The rest of the
|
|
245
217
|
connection works throughout, and a connection that is restored claims its Keys again the same way.
|
|
246
218
|
|
|
247
|
-
`call` publishes the encoded Request to the exchange under the `key` and returns the decoded Reply
|
|
248
|
-
|
|
219
|
+
`call` publishes the encoded Request to the exchange under the `key` and returns the decoded Reply.
|
|
220
|
+
|
|
221
|
+
`options` is the encoding, or an object:
|
|
222
|
+
|
|
223
|
+
| Option | Type | Default |
|
|
224
|
+
|------------|---------------|--------------------|
|
|
225
|
+
| `encoding` | `string` | `application/json` |
|
|
226
|
+
| `timeout` | `number`, ms | none |
|
|
227
|
+
| `signal` | `AbortSignal` | none |
|
|
249
228
|
|
|
250
229
|
A call ends in one of three ways:
|
|
251
230
|
|
|
@@ -271,6 +250,24 @@ same Key can therefore hold it on different shards and both answer calls, and `t
|
|
|
271
250
|
so. A call returned by one shard is published on the next, and refused once every shard has
|
|
272
251
|
returned it.
|
|
273
252
|
|
|
253
|
+
### Timeout
|
|
254
|
+
|
|
255
|
+
A call with a `timeout` rejects once it passes, with the `TimeoutError` of
|
|
256
|
+
[`AbortSignal.timeout`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static).
|
|
257
|
+
It is published with that much
|
|
258
|
+
[expiration](https://www.rabbitmq.com/docs/ttl#per-message-ttl-in-publishers), so a call its holder
|
|
259
|
+
has not taken by then is dropped by the broker and never processed. A call the holder has already
|
|
260
|
+
taken is processed to the end, and its Reply is discarded. A call re-sent after a lost connection
|
|
261
|
+
carries the time it has left.
|
|
262
|
+
|
|
263
|
+
A `signal` rejects the call with its reason once aborted, within the `timeout` where both are
|
|
264
|
+
given. It ends the wait and leaves the call where it is: in its queue until the `timeout` passes,
|
|
265
|
+
or, without a `timeout`, until its holder takes it. A call abandoned by its `signal` may therefore
|
|
266
|
+
still be processed.
|
|
267
|
+
|
|
268
|
+
A call that failed and waits for its [next attempt](#retries) loses its expiration on the way back
|
|
269
|
+
to its queue, so it may be processed after its caller has stopped waiting.
|
|
270
|
+
|
|
274
271
|
### Example
|
|
275
272
|
|
|
276
273
|
```javascript
|
|
@@ -550,7 +547,9 @@ requests and are expecting replies.
|
|
|
550
547
|
- An exchange is asserted as _fanout_ for [Emission](#emission)
|
|
551
548
|
and [Consumption](#consumption), and as _direct_ for [Routing](#routing). One name is one or
|
|
552
549
|
the other: asserting it as both is what the broker refuses.
|
|
553
|
-
-
|
|
550
|
+
- One queue takes every Reply an `IO` receives, `comq.reply..<id>`, _exclusive_ and gone with the
|
|
551
|
+
connection. A Reply is matched by its `correlationId`, which is unique across processes, so the
|
|
552
|
+
queue names neither what was called nor what is waiting.
|
|
554
553
|
- A queue [`back`](#addressed-requests) holds is _exclusive_, bound under its Key to a _direct_
|
|
555
554
|
exchange.
|
|
556
555
|
|
|
@@ -558,7 +557,8 @@ comq declares two kinds of queue of its own, for [failed messages](#retries):
|
|
|
558
557
|
|
|
559
558
|
- `comq.retry.<delay>`, with a fanout exchange of the same name, one pair per distinct rung of
|
|
560
559
|
the [backoff ladder](#retries), shared by every queue that uses it.
|
|
561
|
-
- `comq.parked
|
|
560
|
+
- `comq.parked`, one queue for everything parked, whatever it was consumed from, always
|
|
561
|
+
_durable_.
|
|
562
562
|
|
|
563
563
|
See [queue assertion options](https://amqp-node.github.io/amqplib/channel_api.html#channel_assertQueue).
|
|
564
564
|
|
|
@@ -638,8 +638,10 @@ verdict's own message.
|
|
|
638
638
|
|
|
639
639
|
#### Parked messages
|
|
640
640
|
|
|
641
|
-
A message that has run out of attempts is published to `comq.parked
|
|
642
|
-
only once the broker confirms it.
|
|
641
|
+
A message that has run out of attempts is published to `comq.parked` and acknowledged
|
|
642
|
+
only once the broker confirms it. There is one such queue, and it holds what every queue this
|
|
643
|
+
connection consumes could not process — including a groupless subscription's, which the broker
|
|
644
|
+
removes with the connection while what it could not process stays. It is not deleted, and it does not depend on a broker-side
|
|
643
645
|
policy. The [`discard`](#diagnostics) diagnostic event is emitted when it happens, and
|
|
644
646
|
[`retry`](#diagnostics) on every attempt before it.
|
|
645
647
|
|
|
@@ -648,14 +650,16 @@ name where it was originally published, `x-comq-queue` the queue it was consumed
|
|
|
648
650
|
`x-comq-reason` the exception's message, and `x-comq-at` when it was parked. Its original
|
|
649
651
|
properties are kept as they were.
|
|
650
652
|
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
653
|
+
A message is found by the queue it names rather than by the queue it is in: `comq.parked` holds
|
|
654
|
+
messages from every source, so reading one source's means matching on `x-comq-queue`.
|
|
655
|
+
|
|
656
|
+
The queue grows until somebody drains it, which is deliberate — the alternative is deleting
|
|
657
|
+
evidence. Nothing comq does removes it or what it holds. Alert on [`discard`](#diagnostics), and
|
|
658
|
+
do not delete `comq.retry.*` or `comq.parked` on a running system.
|
|
654
659
|
|
|
655
|
-
> **A parked Request is never answered.** A Consumer awaiting its Reply waits
|
|
656
|
-
> [timeout](#timeout)
|
|
657
|
-
>
|
|
658
|
-
> it — and it keeps `replyTo` and `correlationId`, so a Reply can still be produced from it by
|
|
660
|
+
> **A parked Request is never answered.** A Consumer awaiting its Reply waits indefinitely — a
|
|
661
|
+
> call until its [timeout](#timeout) — and with a limited prefetch that can deadlock it. Parking
|
|
662
|
+
> keeps the Request rather than deleting it — and it keeps `replyTo` and `correlationId`, so a Reply can still be produced from it by
|
|
659
663
|
> hand while the caller is alive — but comq itself sends no Reply and reports no error to the
|
|
660
664
|
> caller.
|
|
661
665
|
|
|
@@ -734,28 +738,6 @@ Sending Requests, receiving Replies, and emitting Events will still be available
|
|
|
734
738
|
Keys held by [`back`](#addressed-requests) are withdrawn first, so a call published from then on is
|
|
735
739
|
refused.
|
|
736
740
|
|
|
737
|
-
### Abandoning
|
|
738
|
-
|
|
739
|
-
`async IO.abandon(): void`
|
|
740
|
-
|
|
741
|
-
Stop waiting for Replies.
|
|
742
|
-
|
|
743
|
-
A [close](#disconnection) waits for every `producer` and `consumer` call to return, and such a
|
|
744
|
-
call may itself be waiting on a Request of its own. Where the answer to that Request is the very
|
|
745
|
-
thing going away — the process it would come from is shutting down alongside this one — the close
|
|
746
|
-
waits for as long as the answer takes.
|
|
747
|
-
|
|
748
|
-
`IO.abandon()` ends that wait:
|
|
749
|
-
|
|
750
|
-
- every outstanding Reply is rejected with `Abandoned`, so a caller inside a `producer` or
|
|
751
|
-
`consumer` is told at once rather than held;
|
|
752
|
-
- a Request made from then on is refused with `Abandoned`, so a callback cannot start another
|
|
753
|
-
wait on the way out.
|
|
754
|
-
|
|
755
|
-
The Requests those Replies belonged to were sent and may well be processed; what is given up is
|
|
756
|
-
this side's interest in the answers. It may be called before a close or while one is already under
|
|
757
|
-
way, which is how a close that has begun is told to stop waiting.
|
|
758
|
-
|
|
759
741
|
### Disconnection
|
|
760
742
|
|
|
761
743
|
`async IO.close(): void`
|
|
@@ -764,9 +746,6 @@ way, which is how a close that has begun is told to stop waiting.
|
|
|
764
746
|
2. Wait for any outstanding messages to be processed[^2] and acknowledged.
|
|
765
747
|
3. Close the connection.
|
|
766
748
|
|
|
767
|
-
Step 2 still runs where the Replies have been [abandoned](#abandoning): each message is processed
|
|
768
|
-
to a verdict and acknowledged, and what was given up is the waiting rather than the drain.
|
|
769
|
-
|
|
770
749
|
[^2]: Therefore, if the underlying connection is lost, `.close()` will only be completed once the
|
|
771
750
|
connection is [recovered](#connection-tolerance).
|
|
772
751
|
|
package/source/.io/ReplyPipe.js
CHANGED
|
@@ -60,12 +60,25 @@ class ReplyPipe extends EventEmitter {
|
|
|
60
60
|
ok: { correlationId, replyTo: feedback.queue, ...CONTROL, headers: FLOW }
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
// The source is not read until the confirmation is published, and a blocked broker holds the
|
|
64
|
+
// publish for as long as it stays blocked. A source failing in the meantime — a reply stream
|
|
65
|
+
// being relayed times out — would raise an `error` nobody listens to. How it failed is not
|
|
66
|
+
// reported to the consumer anyway: `#pump` ends the stream once it finds the source destroyed.
|
|
67
|
+
stream.on('error', noop)
|
|
68
|
+
|
|
63
69
|
channel.diagnose('return', this.#onReturn)
|
|
64
70
|
feedback.on(correlationId, this.#control)
|
|
65
71
|
}
|
|
66
72
|
|
|
67
73
|
async pipe () {
|
|
68
|
-
|
|
74
|
+
try {
|
|
75
|
+
await this.#transmit(control.ok, this.#properties.ok)
|
|
76
|
+
} catch (exception) {
|
|
77
|
+
// nobody is going to read the source now
|
|
78
|
+
this.#interrupt()
|
|
79
|
+
|
|
80
|
+
throw exception
|
|
81
|
+
}
|
|
69
82
|
|
|
70
83
|
if (this.#closed) return
|
|
71
84
|
|
|
@@ -198,4 +211,6 @@ const CONTROL = { type: 'control', mandatory: true }
|
|
|
198
211
|
|
|
199
212
|
const FLOW = { [FLOW_HEADER]: true }
|
|
200
213
|
|
|
214
|
+
function noop () {}
|
|
215
|
+
|
|
201
216
|
exports.ReplyPipe = ReplyPipe
|
package/source/channel.js
CHANGED
|
@@ -27,14 +27,6 @@ class Channel {
|
|
|
27
27
|
/** @type {string[]} */
|
|
28
28
|
#tags = []
|
|
29
29
|
|
|
30
|
-
/**
|
|
31
|
-
* How each queue was declared, so that the queue holding what it could not
|
|
32
|
-
* process is declared to live exactly as long as it does.
|
|
33
|
-
*
|
|
34
|
-
* @type {Map<string, comq.amqp.options.Queue>}
|
|
35
|
-
*/
|
|
36
|
-
#queues = new Map()
|
|
37
|
-
|
|
38
30
|
/**
|
|
39
31
|
* The exchange and key each held queue is bound under, so that sealing withdraws them.
|
|
40
32
|
*
|
|
@@ -88,8 +80,7 @@ class Channel {
|
|
|
88
80
|
// the consumers of the previous channel went down with it, their tags mean nothing here
|
|
89
81
|
this.#tags = []
|
|
90
82
|
|
|
91
|
-
// a fresh channel has
|
|
92
|
-
this.#queues.clear()
|
|
83
|
+
// a fresh channel has bound nothing
|
|
93
84
|
this.#held.clear()
|
|
94
85
|
|
|
95
86
|
await this.#channel.prefetch(this.#topology.prefetch)
|
|
@@ -384,8 +375,6 @@ class Channel {
|
|
|
384
375
|
|
|
385
376
|
const { queue } = await this.#channel.assertQueue(name, options)
|
|
386
377
|
|
|
387
|
-
this.#queues.set(queue, options)
|
|
388
|
-
|
|
389
378
|
return [queue]
|
|
390
379
|
}
|
|
391
380
|
|
|
@@ -485,21 +474,23 @@ class Channel {
|
|
|
485
474
|
* The queue a message is kept in once it has run out of attempts. It has no consumer
|
|
486
475
|
* either: what is in it is waiting for a person.
|
|
487
476
|
*
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
*
|
|
492
|
-
*
|
|
477
|
+
* One queue serves every queue this channel consumes, as one retry queue serves every
|
|
478
|
+
* source that shares its delay. A parked message carries the queue it came from, so a
|
|
479
|
+
* queue per source would name what the message already says, and stay on the broker
|
|
480
|
+
* long after the queue it was named for has gone.
|
|
481
|
+
*
|
|
482
|
+
* It is durable whatever the channel is. A groupless subscriber's own queue goes with
|
|
483
|
+
* its connection, but what it could not process is waiting for a person, and there is
|
|
484
|
+
* nobody left to tell.
|
|
493
485
|
*
|
|
494
|
-
* @param {string} queue the queue the message was consumed from
|
|
495
486
|
* @returns {Promise<void>}
|
|
496
487
|
*/
|
|
497
|
-
async #assertParkedQueue (
|
|
488
|
+
async #assertParkedQueue () {
|
|
498
489
|
if (!this.#topology.acknowledgments) return
|
|
499
490
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
await this.#assertQueue(
|
|
491
|
+
// declared here rather than when a message is first parked, for the reason the retry
|
|
492
|
+
// queues are: the failure handler is the least likely moment for a declare to succeed
|
|
493
|
+
await this.#assertQueue(PARKED, DURABLE)
|
|
503
494
|
}
|
|
504
495
|
|
|
505
496
|
/** The waits between attempts, as a ladder even when it is one rung. */
|
|
@@ -556,8 +547,9 @@ class Channel {
|
|
|
556
547
|
}
|
|
557
548
|
|
|
558
549
|
/**
|
|
559
|
-
* The retry topology is asserted once per channel
|
|
560
|
-
*
|
|
550
|
+
* The retry and parking topology is asserted once per channel: `lazy` keys an
|
|
551
|
+
* initializer by the arguments it takes, and neither takes any. Both again after
|
|
552
|
+
* a recovery.
|
|
561
553
|
*/
|
|
562
554
|
#consume = lazy(this, [this.#assertRetryQueue, this.#assertParkedQueue],
|
|
563
555
|
/**
|
|
@@ -660,7 +652,7 @@ class Channel {
|
|
|
660
652
|
[PARKED_AT_HEADER]: Date.now()
|
|
661
653
|
})
|
|
662
654
|
|
|
663
|
-
await this.#publish(DEFAULT,
|
|
655
|
+
await this.#publish(DEFAULT, PARKED, message.content, properties)
|
|
664
656
|
|
|
665
657
|
this.#channel.ack(message)
|
|
666
658
|
|
|
@@ -775,9 +767,9 @@ const INTERRUPTION = /** @type {Error} */ Symbol('internal interruption')
|
|
|
775
767
|
const RESOURCE_LOCKED = 405
|
|
776
768
|
|
|
777
769
|
const RETRY_PREFIX = 'comq.retry.'
|
|
778
|
-
const PARKED_PREFIX = 'comq.parked.'
|
|
779
770
|
|
|
780
|
-
|
|
771
|
+
/** Where every message that has run out of attempts is kept, whatever it was consumed from. */
|
|
772
|
+
const PARKED = 'comq.parked'
|
|
781
773
|
|
|
782
774
|
// Everything comq writes onto a message, under its own prefix: AMQP defines no retry
|
|
783
775
|
// counter, so this one is comq's invention rather than a convention, and an unprefixed
|
package/source/connection.js
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const { setTimeout: delay } = require('node:timers/promises')
|
|
4
|
-
const amqp = require('amqplib')
|
|
4
|
+
const amqp = require('@toa.io/amqplib')
|
|
5
5
|
const { Promex } = require('promex')
|
|
6
6
|
const { retry } = require('reretry')
|
|
7
7
|
|
|
8
8
|
const { failsafe } = require('./attributes')
|
|
9
|
-
const { batch } = require('./batch')
|
|
10
9
|
const presets = require('./topology')
|
|
11
10
|
const channels = require('./channel')
|
|
12
11
|
const emitter = require('./emitter')
|
|
@@ -148,8 +147,6 @@ class Connection {
|
|
|
148
147
|
connection.on('close', (error) => this.#close(connection, error))
|
|
149
148
|
this.#connection = connection
|
|
150
149
|
|
|
151
|
-
batch(connection.connection?.stream)
|
|
152
|
-
|
|
153
150
|
this.#armWatchdog(connection)
|
|
154
151
|
this.#diagnostics.emit('open')
|
|
155
152
|
|