@onlineapps/mq-client-core 4.2.0 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +768 -0
- package/README.md +149 -19
- package/package.json +1 -1
- package/src/BaseClient.js +126 -31
- package/src/buffer/InMemoryBuffer.js +98 -13
- package/src/buffer/MessageBuffer.js +5 -3
- package/src/config/configSchema.js +33 -10
- package/src/config/defaultConfig.js +21 -10
- package/src/config/deliveryPolicy.js +86 -6
- package/src/config/queueConfig.js +68 -39
- package/src/config.js +5 -1
- package/src/defaults.js +7 -1
- package/src/index.js +41 -4
- package/src/layers/PublishLayer.js +264 -27
- package/src/monitoring-initiated.js +6 -9
- package/src/monitoring-publish.js +89 -40
- package/src/scheduler-request.js +32 -11
- package/src/transports/rabbitmqClient.js +1782 -390
- package/src/utils/decodeMessage.js +123 -0
- package/src/utils/publishErrors.js +256 -17
- package/jest.config.js +0 -15
- package/jest.integration.config.js +0 -16
- package/src/workers/RecoveryWorker.js +0 -130
package/src/index.js
CHANGED
|
@@ -8,9 +8,6 @@
|
|
|
8
8
|
*
|
|
9
9
|
* NOTE: Infrastructure orchestration utilities (waitForInfrastructureReady, initInfrastructureQueues)
|
|
10
10
|
* have been moved to @onlineapps/infrastructure-tools to keep this library focused on core MQ operations.
|
|
11
|
-
*
|
|
12
|
-
* @version 1.0.49
|
|
13
|
-
* @updated 2024-12-08 - Test publish-library.sh workflow
|
|
14
11
|
*/
|
|
15
12
|
|
|
16
13
|
const BaseClient = require('./BaseClient');
|
|
@@ -48,9 +45,11 @@ const {
|
|
|
48
45
|
buildCancelRequest,
|
|
49
46
|
publishSchedulerRequest,
|
|
50
47
|
createSchedulerPublisher,
|
|
48
|
+
SCHEDULER_REQUEST_INVALID,
|
|
51
49
|
} = require('./scheduler-request');
|
|
52
50
|
const queueConfig = require('./config/queueConfig');
|
|
53
51
|
const deliveryPolicy = require('./config/deliveryPolicy');
|
|
52
|
+
const { decodeMessage } = require('./utils/decodeMessage');
|
|
54
53
|
const { redactUrl, UNPARSEABLE_PLACEHOLDER } = require('@onlineapps/logger-contract');
|
|
55
54
|
|
|
56
55
|
// Export BaseClient as default (constructor), with additional named exports
|
|
@@ -119,6 +118,8 @@ module.exports.scheduler = {
|
|
|
119
118
|
buildCancelRequest,
|
|
120
119
|
publishSchedulerRequest,
|
|
121
120
|
createSchedulerPublisher,
|
|
121
|
+
// The code on every refusal of a builder, and on nothing a failed send throws.
|
|
122
|
+
SCHEDULER_REQUEST_INVALID,
|
|
122
123
|
};
|
|
123
124
|
/** The only public way to reach the central queue classification; `src/config/queueConfig` is an internal path, not a contract. */
|
|
124
125
|
module.exports.queueConfig = queueConfig;
|
|
@@ -175,7 +176,25 @@ module.exports.topology = Object.freeze({
|
|
|
175
176
|
/** The same consumer's metric-sample intake on that exchange. */
|
|
176
177
|
telemetryMetricsQueue: queueConfig.queueName('telemetry', 'metrics.queue'),
|
|
177
178
|
/** The same consumer's span intake on that exchange. */
|
|
178
|
-
telemetryTracesQueue: queueConfig.queueName('telemetry', 'traces.queue')
|
|
179
|
+
telemetryTracesQueue: queueConfig.queueName('telemetry', 'traces.queue'),
|
|
180
|
+
/**
|
|
181
|
+
* Topic exchange: every telemetry producer on the platform publishes logs, metrics and
|
|
182
|
+
* spans here with a routing key of the shape `telemetry.<signal>.<service_name>`, and
|
|
183
|
+
* the three queues above are bound to it.
|
|
184
|
+
*
|
|
185
|
+
* Declared in `queueConfig` since d.708, which ended the state where one name had three
|
|
186
|
+
* owners: a module constant in the monitoring consumer
|
|
187
|
+
* (`api/infra/api_monitoring/src/consumer/topology.js`) and a config default in each of
|
|
188
|
+
* monitoring-core's two publishers. The declaration alone does not finish that move —
|
|
189
|
+
* the consumer resolves every platform name it does not own through THIS export, and
|
|
190
|
+
* `src/config/queueConfig` is an internal path rather than a contract (see the note on
|
|
191
|
+
* the module export above). So the name it must import belongs here, the same step d.461
|
|
192
|
+
* made for the three queues it is bound to.
|
|
193
|
+
*
|
|
194
|
+
* @see api/infra/api_monitoring/src/consumer/index.js — assertMonitoringTopology()
|
|
195
|
+
* @see api/shared/monitoring/monitoring-core/src/exporters/rabbitmq.js
|
|
196
|
+
*/
|
|
197
|
+
telemetryExchange: queueConfig.queueName('telemetry', 'exchange')
|
|
179
198
|
});
|
|
180
199
|
/**
|
|
181
200
|
* The dead-letter policy of `consume()` as a contract, not as an implementation
|
|
@@ -186,6 +205,24 @@ module.exports.topology = Object.freeze({
|
|
|
186
205
|
* return one of those two words.
|
|
187
206
|
*/
|
|
188
207
|
module.exports.deliveryPolicy = deliveryPolicy;
|
|
208
|
+
/**
|
|
209
|
+
* The one rail from a delivery to the message it carries.
|
|
210
|
+
*
|
|
211
|
+
* `consume()` hands its handler the RAW amqplib delivery, so every consumer wrote that
|
|
212
|
+
* translation itself: d.788 measured FOUR copies (dispatcher, scheduler, service-wrapper,
|
|
213
|
+
* registry) which disagree about what happens to a BAD message — dead-letter, loss rail,
|
|
214
|
+
* `INVALID_JSON` reply, or requeue for ever — and two of which pass a delivery with no
|
|
215
|
+
* `content` on AS the body.
|
|
216
|
+
*
|
|
217
|
+
* The decoding was never the missing part (`utils/serializer.js` has done it all along);
|
|
218
|
+
* the missing part was this door, because `src/…` is an internal path rather than a
|
|
219
|
+
* contract. Exported ADDITIVELY: nothing in this package calls it and `consume()` is
|
|
220
|
+
* unchanged, because converting those four changes the fate of a malformed message —
|
|
221
|
+
* BREAKING, and for the registry only after its queue has a dead-letter route (d.784).
|
|
222
|
+
*
|
|
223
|
+
* @see ./utils/decodeMessage.js — what it refuses, and with which of this package's classes
|
|
224
|
+
*/
|
|
225
|
+
module.exports.decodeMessage = decodeMessage;
|
|
189
226
|
/**
|
|
190
227
|
* Redaction of the credential a connection URL carries, as a DECLARED export.
|
|
191
228
|
*
|
|
@@ -51,10 +51,10 @@ class PublishLayer {
|
|
|
51
51
|
// transport constructor reads it from there (d.292 §5).
|
|
52
52
|
//
|
|
53
53
|
// Asked with `=== undefined`, never with `||`. `||` means "if falsy", so the
|
|
54
|
-
// three numbers below lost a legally written `0` — `
|
|
55
|
-
// means "
|
|
56
|
-
//
|
|
57
|
-
//
|
|
54
|
+
// three numbers below lost a legally written `0` — `publishRetryBaseDelay: 0`
|
|
55
|
+
// means "retry immediately", and it arrived here as 100. d.292 removed exactly
|
|
56
|
+
// that shape from the transport and said in as many words that this layer still
|
|
57
|
+
// carried its own copies; this is that floor (d.311).
|
|
58
58
|
this._retryEnabled = options.retryEnabled !== undefined
|
|
59
59
|
? options.retryEnabled
|
|
60
60
|
: (this._client._publishRetryEnabled !== false);
|
|
@@ -71,6 +71,45 @@ class PublishLayer {
|
|
|
71
71
|
? options.retryBackoffMultiplier
|
|
72
72
|
: this._client._publishRetryBackoffMultiplier;
|
|
73
73
|
|
|
74
|
+
// `publishMaxRetries` counts ATTEMPTS, the first one included — that is what the
|
|
75
|
+
// key means in every place it is written down (`config/configSchema.js`,
|
|
76
|
+
// `config/defaultConfig.js`, `docs/architecture/mq-publish-reliability.md` § the
|
|
77
|
+
// config table, `docs/standards/rabbitmq-channel-best-practices.md`), and what
|
|
78
|
+
// `_publishWithRetry()` below does for every value from 1 up.
|
|
79
|
+
//
|
|
80
|
+
// So `0` is not "do not retry" — three comments in this package said it was, and
|
|
81
|
+
// the loop obeyed the key instead: `while (attempt < this._maxRetries)` never runs
|
|
82
|
+
// its body, so the message is never offered to the broker, is NOT buffered on the
|
|
83
|
+
// way out, and the caller is handed an error blaming this layer for a defect its
|
|
84
|
+
// own configuration caused (measured, d.684). "One attempt, no retry" is `1`, and
|
|
85
|
+
// leaving the retry rail entirely is `publishRetryEnabled: false`; both are
|
|
86
|
+
// explicit, and neither needs a magic zero (`architecture-principles.md` §8).
|
|
87
|
+
//
|
|
88
|
+
// Refused HERE, at construction, because this is the one place both doors meet —
|
|
89
|
+
// the caller's `options.maxRetries` and the client's resolved config — and a value
|
|
90
|
+
// that can only fail is refused before it is used, not when a publish needs it
|
|
91
|
+
// (§4 fail-fast).
|
|
92
|
+
//
|
|
93
|
+
// `config/configSchema.js` refuses the same value too since d.685b (`minimum: 1`),
|
|
94
|
+
// and that is not a second rail for one concern: the schema answers what may be
|
|
95
|
+
// WRITTEN in a client's configuration, this answers what may be USED as a number of
|
|
96
|
+
// attempts — and the second is reachable without the first. Measured 2026-09-20: the
|
|
97
|
+
// schema is compiled and applied in exactly one place, `BaseClient`'s constructor,
|
|
98
|
+
// while the only `new RabbitMQClient` in the workspace is
|
|
99
|
+
// `transports/transportFactory.js`, which validates nothing. The two also answer at
|
|
100
|
+
// different moments, because `BaseClient` builds the transport in `connect()` rather
|
|
101
|
+
// than in its constructor — so a client built past the front desk still arrives here
|
|
102
|
+
// with whatever it was given.
|
|
103
|
+
if (!Number.isInteger(this._maxRetries) || this._maxRetries < 1) {
|
|
104
|
+
throw new ValidationError(
|
|
105
|
+
`[PublishLayer] publishMaxRetries must be at least 1 - got ${JSON.stringify(this._maxRetries)}. `
|
|
106
|
+
+ 'Expected: a whole number of publish attempts, because it counts ATTEMPTS, the first one '
|
|
107
|
+
+ 'included — 0 means the message is never offered to the broker at all. '
|
|
108
|
+
+ 'Fix: write publishMaxRetries: 1 for one attempt and no retry, or publishRetryEnabled: false '
|
|
109
|
+
+ 'to leave the retry rail entirely.'
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
|
|
74
113
|
// Buffer
|
|
75
114
|
this._buffer = new MessageBuffer({
|
|
76
115
|
inMemory: options.bufferConfig?.inMemory,
|
|
@@ -83,8 +122,35 @@ class PublishLayer {
|
|
|
83
122
|
* @param {string} queue
|
|
84
123
|
* @param {Buffer} buffer
|
|
85
124
|
* @param {Object} [options]
|
|
125
|
+
* @param {boolean} [options.bufferOnFailure=true] - Whether a message whose publish failed
|
|
126
|
+
* transiently is held in the buffer and replayed after the next reconnect. `false` is for
|
|
127
|
+
* a caller that keeps the message durably elsewhere (conf `step-publish-failure` 001):
|
|
128
|
+
* nothing is held, so nothing replays behind its back. On each of the three transient
|
|
129
|
+
* paths (retries exhausted, retry disabled, the awaited reconnect failed) the thrown
|
|
130
|
+
* error carries `buffered` (boolean) — `true` only when the buffer really holds the
|
|
131
|
+
* message. A non-boolean value is refused before any attempt.
|
|
132
|
+
*
|
|
133
|
+
* Every error this method throws also carries `sent` (boolean, d.1145, conf
|
|
134
|
+
* `mq-publish-buffer` 001): `true` when ANY attempt of this call wrote the frame to the
|
|
135
|
+
* channel, so the broker may hold — and after a memory alarm deliver — the message even
|
|
136
|
+
* though the call failed; `false` when no attempt got that far. The buffer entry records the
|
|
137
|
+
* same fact, and the flush after `unblocked` reads it.
|
|
138
|
+
* @throws {ValidationError} When `options.bufferOnFailure` is present and not a boolean.
|
|
86
139
|
*/
|
|
87
140
|
async publish(queue, buffer, options = {}) {
|
|
141
|
+
// Refused HERE, before any attempt: a value that cannot say yes or no must not be read
|
|
142
|
+
// as either — `'false'` is truthy, and `0` would be read as "no" by accident
|
|
143
|
+
// (`architecture-principles.md` §3, §4). The one owner of the option is this layer;
|
|
144
|
+
// `BaseClient.publish()` only carries it.
|
|
145
|
+
if (options.bufferOnFailure !== undefined && typeof options.bufferOnFailure !== 'boolean') {
|
|
146
|
+
throw new ValidationError(
|
|
147
|
+
`[PublishLayer] options.bufferOnFailure must be a boolean - got ${JSON.stringify(options.bufferOnFailure)} `
|
|
148
|
+
+ `for queue "${queue}". Expected: true (hold a transiently failed message and replay it after `
|
|
149
|
+
+ 'the reconnect — the default) or false (hold nothing; the caller keeps the message). '
|
|
150
|
+
+ 'Fix: pass bufferOnFailure: true or bufferOnFailure: false, or omit it.'
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
88
154
|
// `priority` still orders the buffer's replay (critical first). What it no
|
|
89
155
|
// longer does is choose a DIFFERENT buffer: the persistent one was a stub
|
|
90
156
|
// that stored nothing, so routing critical messages to it dropped them
|
|
@@ -106,8 +172,38 @@ class PublishLayer {
|
|
|
106
172
|
try {
|
|
107
173
|
await this._client._publishOnce(queue, buffer, options);
|
|
108
174
|
} catch (err) {
|
|
109
|
-
|
|
175
|
+
const sent = this._sentOf(err, queue);
|
|
176
|
+
return await this._handlePublishError(err, queue, buffer, options, priority, sent);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Whether the transport says the failed attempt's frame left the client.
|
|
182
|
+
*
|
|
183
|
+
* The transport writes `sent` on every error of `_publishOnce()` (d.1145). An error without
|
|
184
|
+
* it is a defect of the neighbour, and it is refused here by name rather than read as
|
|
185
|
+
* `false`: a message whose frame did leave would then be re-sent after `unblocked`, and
|
|
186
|
+
* that is the duplicate conf `mq-publish-buffer` 001 exists to prevent.
|
|
187
|
+
*
|
|
188
|
+
* @param {*} err - what `client._publishOnce()` threw
|
|
189
|
+
* @param {string} queue
|
|
190
|
+
* @returns {boolean}
|
|
191
|
+
* @throws {Error} when the error carries no boolean `sent`
|
|
192
|
+
* @private
|
|
193
|
+
*/
|
|
194
|
+
_sentOf(err, queue) {
|
|
195
|
+
if (err !== null && typeof err === 'object' && typeof err.sent === 'boolean') {
|
|
196
|
+
return err.sent;
|
|
110
197
|
}
|
|
198
|
+
throw new Error(
|
|
199
|
+
'[PublishLayer] The transport reported a failed attempt without saying whether its frame was sent - '
|
|
200
|
+
+ `queue "${queue}", error: ${err && err.message ? err.message : String(err)}. `
|
|
201
|
+
+ 'Expected: every error of client._publishOnce() carries sent (boolean) — RabbitMQClient writes it on '
|
|
202
|
+
+ 'every attempt (conf mq-publish-buffer 001). Fix: a stand-in transport must write err.sent as the '
|
|
203
|
+
+ 'real one does; the message is neither buffered nor retried, because nobody can say whether '
|
|
204
|
+
+ 'the broker may already hold it.',
|
|
205
|
+
{ cause: err }
|
|
206
|
+
);
|
|
111
207
|
}
|
|
112
208
|
|
|
113
209
|
/**
|
|
@@ -118,7 +214,56 @@ class PublishLayer {
|
|
|
118
214
|
let lastError = null;
|
|
119
215
|
let attempt = 0;
|
|
120
216
|
|
|
217
|
+
/**
|
|
218
|
+
* The broker's numbered refusal of THIS message, if it arrived after the confirmation
|
|
219
|
+
* budget had already answered an attempt.
|
|
220
|
+
*
|
|
221
|
+
* Measured (d.801): when the budget is shorter than the broker's answer, the attempt
|
|
222
|
+
* is answered "transient" while the frame underneath says 404 — and the loop below
|
|
223
|
+
* used to publish the message again, `publishMaxRetries` times, and then buffer it,
|
|
224
|
+
* where the flush re-offers it on every reconnect until its TTL drops it. d.801b made
|
|
225
|
+
* that verdict audible (`publish:refused-late`); this listener is what ACTS on it.
|
|
226
|
+
*
|
|
227
|
+
* No new evidence and no new timer: the frame is read where it has always been read
|
|
228
|
+
* (the transport's `channel._closeError` + `_closeFrameNamesThisSend()`), the event is
|
|
229
|
+
* the one already emitted, and the subscription lives exactly as long as this publish.
|
|
230
|
+
*/
|
|
231
|
+
// Whether ANY attempt of this call wrote the frame to the channel (d.1145). Written on
|
|
232
|
+
// every error this loop throws — `said()` — and on the buffer entry.
|
|
233
|
+
let sentAny = false;
|
|
234
|
+
const said = (error) => {
|
|
235
|
+
error.sent = sentAny;
|
|
236
|
+
return error;
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
let lateRefusal = null;
|
|
240
|
+
const onRefusedLate = (event) => {
|
|
241
|
+
if (event.queue === queue && event.verdict === 'PermanentPublishError' && event.error) {
|
|
242
|
+
lateRefusal = event.error;
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
this._client.on('publish:refused-late', onRefusedLate);
|
|
246
|
+
|
|
247
|
+
try {
|
|
121
248
|
while (attempt < this._maxRetries) {
|
|
249
|
+
// A message the broker has REFUSED is not published again. The question is asked
|
|
250
|
+
// before every attempt but the first, because the first is what earns the answer.
|
|
251
|
+
if (attempt > 0 && lateRefusal !== null) {
|
|
252
|
+
this._logger.warn(
|
|
253
|
+
`[PublishLayer] The broker refused this message while attempt ${attempt} was in flight, `
|
|
254
|
+
+ `so attempt ${attempt + 1} of ${this._maxRetries} is not made for queue "${queue}" - `
|
|
255
|
+
+ `${lateRefusal.message}`
|
|
256
|
+
);
|
|
257
|
+
this._client.emit('publish:failed', {
|
|
258
|
+
queue,
|
|
259
|
+
attempt,
|
|
260
|
+
error: lateRefusal.message,
|
|
261
|
+
retryable: false,
|
|
262
|
+
refusedLate: true
|
|
263
|
+
});
|
|
264
|
+
throw said(lateRefusal);
|
|
265
|
+
}
|
|
266
|
+
|
|
122
267
|
attempt++;
|
|
123
268
|
|
|
124
269
|
try {
|
|
@@ -133,7 +278,12 @@ class PublishLayer {
|
|
|
133
278
|
this._logger.debug(`[PublishLayer] Retry attempt ${attempt}/${this._maxRetries} for queue "${queue}"`);
|
|
134
279
|
}
|
|
135
280
|
|
|
136
|
-
|
|
281
|
+
// WHICH attempt this is travels with the send, and is read by exactly one thing:
|
|
282
|
+
// the transport's report of a broker verdict that arrived after the confirmation
|
|
283
|
+
// budget had already answered (`publish:refused-late`, d.801b). It reaches no
|
|
284
|
+
// broker property — `buildAmqpMessageProperties()` copies only the AMQP keys it
|
|
285
|
+
// declares — and it changes nothing about the publish itself.
|
|
286
|
+
await this._client._publishOnce(queue, buffer, { ...options, attempt });
|
|
137
287
|
|
|
138
288
|
// Track success (always emit, even for first attempt)
|
|
139
289
|
this._client.emit('publish:success', {
|
|
@@ -149,6 +299,7 @@ class PublishLayer {
|
|
|
149
299
|
|
|
150
300
|
} catch (err) {
|
|
151
301
|
lastError = err;
|
|
302
|
+
sentAny = this._sentOf(err, queue) || sentAny;
|
|
152
303
|
|
|
153
304
|
// Non-retryable errors - fail immediately
|
|
154
305
|
if (err instanceof PermanentPublishError || err instanceof QueueNotFoundError) {
|
|
@@ -158,7 +309,7 @@ class PublishLayer {
|
|
|
158
309
|
error: err.message,
|
|
159
310
|
retryable: false
|
|
160
311
|
});
|
|
161
|
-
throw err;
|
|
312
|
+
throw said(err);
|
|
162
313
|
}
|
|
163
314
|
|
|
164
315
|
// Transient errors - check if we should retry or buffer
|
|
@@ -173,8 +324,8 @@ class PublishLayer {
|
|
|
173
324
|
this._logger.debug(`[PublishLayer] ✓ Reconnection completed, proceeding with retry ${attempt + 1}/${this._maxRetries}`);
|
|
174
325
|
continue; // Retry immediately after reconnect
|
|
175
326
|
} catch (reconnectErr) {
|
|
176
|
-
// Reconnection failed - buffer and fail
|
|
177
|
-
await this.
|
|
327
|
+
// Reconnection failed - buffer (unless the caller said not to) and fail
|
|
328
|
+
const held = await this._holdOnFailure(queue, buffer, options, priority, err, sentAny);
|
|
178
329
|
this._client.emit('publish:failed', {
|
|
179
330
|
queue,
|
|
180
331
|
attempt,
|
|
@@ -182,13 +333,15 @@ class PublishLayer {
|
|
|
182
333
|
retryable: false,
|
|
183
334
|
reconnectFailed: true
|
|
184
335
|
});
|
|
185
|
-
|
|
336
|
+
const failure = new PublishError(
|
|
186
337
|
`[PublishLayer] Publish failed: reconnection failed after ${attempt} attempts for queue "${queue}" - `
|
|
187
338
|
+ 'Expected: the transport to re-establish the connection before the retry. '
|
|
188
|
-
+
|
|
339
|
+
+ `Fix: read error.cause for the reconnect reason; ${held.sentence}`,
|
|
189
340
|
queue,
|
|
190
341
|
reconnectErr
|
|
191
342
|
);
|
|
343
|
+
failure.buffered = held.buffered;
|
|
344
|
+
throw said(failure);
|
|
192
345
|
}
|
|
193
346
|
}
|
|
194
347
|
|
|
@@ -198,12 +351,30 @@ class PublishLayer {
|
|
|
198
351
|
this._retryMaxDelay
|
|
199
352
|
);
|
|
200
353
|
this._logger.warn(`[PublishLayer] Retryable error for queue "${queue}" (attempt ${attempt}/${this._maxRetries}), waiting ${delay}ms before retry: ${err.message}`);
|
|
354
|
+
// THE BARE TIMER IS THE POINT, and it is not a copy of the transport's rail.
|
|
355
|
+
//
|
|
356
|
+
// The transport sleeps through `_setTimeout()`, which files the handle in
|
|
357
|
+
// `_activeTimers` so that a teardown can END the wait (d.759e). That is right
|
|
358
|
+
// THERE: what waits is the client's own recovery loop, and a teardown must
|
|
359
|
+
// stop it. It is wrong HERE, because what waits is the CALLER's `publish()`.
|
|
360
|
+
// Měřeno d.766 on this runtime: a sleep armed through `_setTimeout()` and
|
|
361
|
+
// swept by `_clearAllTimers()` mid-wait NEVER settles (the same sleep on the
|
|
362
|
+
// bare timer always does), so the one rail would turn a disconnect during a
|
|
363
|
+
// backoff into a `publish()` that never returns. The layer HAS the client
|
|
364
|
+
// (injected, required in the constructor), so this is a difference of
|
|
365
|
+
// obligation and not of reach.
|
|
366
|
+
//
|
|
367
|
+
// One owner at this level: the retry wait is the layer's, it is always
|
|
368
|
+
// awaited by the caller, and it always ends.
|
|
369
|
+
// @see ../transports/rabbitmqClient.js — `_setInterval` / `_clearAllTimers`
|
|
370
|
+
// @see ../../tests/unit/publish-max-retries-counts-attempts.test.js — the
|
|
371
|
+
// control case that fails if this is ever "unified"
|
|
201
372
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
202
373
|
continue;
|
|
203
374
|
}
|
|
204
375
|
|
|
205
|
-
// Max retries exceeded - buffer and fail
|
|
206
|
-
await this.
|
|
376
|
+
// Max retries exceeded - buffer (unless the caller said not to) and fail
|
|
377
|
+
const held = await this._holdOnFailure(queue, buffer, options, priority, err, sentAny);
|
|
207
378
|
this._client.emit('publish:failed', {
|
|
208
379
|
queue,
|
|
209
380
|
attempt,
|
|
@@ -211,21 +382,23 @@ class PublishLayer {
|
|
|
211
382
|
retryable: true,
|
|
212
383
|
maxRetriesExceeded: true
|
|
213
384
|
});
|
|
214
|
-
|
|
385
|
+
const failure = new PublishError(
|
|
215
386
|
`[PublishLayer] Publish failed after ${attempt} attempts for queue "${queue}" - `
|
|
216
387
|
+ 'Expected: a retryable broker error to clear within maxRetries. '
|
|
217
|
-
+
|
|
388
|
+
+ `Fix: read error.cause for the last broker reason; ${held.sentence}`,
|
|
218
389
|
queue,
|
|
219
390
|
err
|
|
220
391
|
);
|
|
392
|
+
failure.buffered = held.buffered;
|
|
393
|
+
throw said(failure);
|
|
221
394
|
}
|
|
222
395
|
|
|
223
396
|
// Unknown error - fail immediately
|
|
224
|
-
throw err;
|
|
397
|
+
throw said(err);
|
|
225
398
|
}
|
|
226
399
|
}
|
|
227
400
|
|
|
228
|
-
throw (
|
|
401
|
+
throw said(
|
|
229
402
|
lastError
|
|
230
403
|
|| new PublishError(
|
|
231
404
|
`[PublishLayer] Publish failed for queue "${queue}" after ${attempt} attempts - `
|
|
@@ -235,50 +408,114 @@ class PublishLayer {
|
|
|
235
408
|
null
|
|
236
409
|
)
|
|
237
410
|
);
|
|
411
|
+
} finally {
|
|
412
|
+
// The subscription lives exactly as long as this publish: no listener survives the
|
|
413
|
+
// call that opened it, whichever way the loop ended.
|
|
414
|
+
this._client.removeListener('publish:refused-late', onRefusedLate);
|
|
415
|
+
}
|
|
238
416
|
}
|
|
239
417
|
|
|
240
418
|
/**
|
|
241
419
|
* Handle publish error - buffer transient errors
|
|
242
420
|
* @private
|
|
243
421
|
*/
|
|
244
|
-
async _handlePublishError(err, queue, buffer, options, priority) {
|
|
422
|
+
async _handlePublishError(err, queue, buffer, options, priority, sent) {
|
|
245
423
|
if (err instanceof QueueNotFoundError) {
|
|
246
424
|
this._logger.error(`[PublishLayer] QueueNotFoundError for '${queue}' (infra=${err.isInfrastructure}): ${err.message}`);
|
|
247
425
|
throw err;
|
|
248
426
|
}
|
|
249
427
|
|
|
250
428
|
if (err instanceof TransientPublishError) {
|
|
251
|
-
await this.
|
|
429
|
+
const held = await this._holdOnFailure(queue, buffer, options, priority, err, sent);
|
|
430
|
+
// The transport's own error, thrown as it came — this path has always done that —
|
|
431
|
+
// with the one fact only this layer knows written on it.
|
|
432
|
+
err.buffered = held.buffered;
|
|
252
433
|
throw err;
|
|
253
434
|
}
|
|
254
435
|
|
|
255
436
|
throw err;
|
|
256
437
|
}
|
|
257
438
|
|
|
439
|
+
/**
|
|
440
|
+
* Hold a transiently failed message in the buffer — or, when the caller passed
|
|
441
|
+
* `bufferOnFailure: false`, deliberately do not — and say which happened.
|
|
442
|
+
*
|
|
443
|
+
* The one decision point of the three transient paths, so the option cannot be honoured
|
|
444
|
+
* on one path and forgotten on another.
|
|
445
|
+
*
|
|
446
|
+
* @private
|
|
447
|
+
* @returns {Promise<{buffered: boolean, sentence: string}>} `buffered` is `true` only when
|
|
448
|
+
* the buffer really holds the message; `sentence` ends the thrown error's `Fix:` with
|
|
449
|
+
* what the caller can rely on.
|
|
450
|
+
*/
|
|
451
|
+
async _holdOnFailure(queue, buffer, options, priority, err, sent) {
|
|
452
|
+
if (options.bufferOnFailure === false) {
|
|
453
|
+
this._logger.warn(
|
|
454
|
+
`[PublishLayer] Message for queue '${queue}' NOT buffered (bufferOnFailure: false) after a transient error: ${err.message}`
|
|
455
|
+
);
|
|
456
|
+
return {
|
|
457
|
+
buffered: false,
|
|
458
|
+
sentence: 'the message is NOT buffered (bufferOnFailure: false), so nothing replays it — '
|
|
459
|
+
+ 'the caller holds the only copy (error.buffered === false).'
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
if (await this._bufferMessage(queue, buffer, options, priority, err, sent)) {
|
|
463
|
+
return { buffered: true, sentence: 'the message is buffered, so it is not lost.' };
|
|
464
|
+
}
|
|
465
|
+
return {
|
|
466
|
+
buffered: false,
|
|
467
|
+
sentence: 'the message is NOT buffered (the buffer refused it — see the error log line before this one), '
|
|
468
|
+
+ 'so nothing replays it (error.buffered === false).'
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
|
|
258
472
|
/**
|
|
259
473
|
* Buffer message při transient error
|
|
260
474
|
* @private
|
|
475
|
+
* @returns {Promise<boolean>} whether the buffer now holds the message
|
|
261
476
|
*/
|
|
262
|
-
async _bufferMessage(queue, buffer, options, priority, err) {
|
|
477
|
+
async _bufferMessage(queue, buffer, options, priority, err, sent) {
|
|
263
478
|
try {
|
|
264
|
-
await this._buffer.add(queue, buffer, options, { priority });
|
|
479
|
+
await this._buffer.add(queue, buffer, options, { priority, sent });
|
|
265
480
|
this._logger.warn(`[PublishLayer] Buffered message for queue '${queue}' due to transient error: ${err.message}`);
|
|
266
481
|
this._client.emit('publish:buffered', { queue });
|
|
482
|
+
return true;
|
|
267
483
|
} catch (bufferErr) {
|
|
268
484
|
this._logger.error(`[PublishLayer] Failed to buffer message for queue '${queue}': ${bufferErr.message}`);
|
|
485
|
+
return false;
|
|
269
486
|
}
|
|
270
487
|
}
|
|
271
488
|
|
|
272
489
|
/**
|
|
273
|
-
*
|
|
490
|
+
* Re-send what the buffer holds — the one flush, with two triggers (d.1145, conf
|
|
491
|
+
* `mq-publish-buffer` 001):
|
|
492
|
+
*
|
|
493
|
+
* - `'reconnected'` — a new connection: every entry is re-sent, as it always was;
|
|
494
|
+
* - `'unblocked'` — the broker ended a memory alarm on the SAME connection: only the
|
|
495
|
+
* entries whose frame never left the client are re-sent; the sent ones are dropped with a
|
|
496
|
+
* warning, because the broker delivers those itself (d.1143 measured it).
|
|
497
|
+
*
|
|
498
|
+
* Every entry is re-sent with the publish options it was held with — nothing else is
|
|
499
|
+
* merged in (d.1145 removed the `flushOptions` argument nobody passed).
|
|
500
|
+
*
|
|
501
|
+
* @param {'reconnected'|'unblocked'} trigger - what ended the outage
|
|
502
|
+
* @returns {Promise<number>} how many entries were re-sent successfully
|
|
503
|
+
* @throws {Error} when the trigger is not one of the two
|
|
274
504
|
*/
|
|
275
|
-
async flushBuffered(
|
|
505
|
+
async flushBuffered(trigger) {
|
|
506
|
+
const SENT_ENTRIES = { reconnected: 'resend', unblocked: 'drop' };
|
|
507
|
+
if (!Object.prototype.hasOwnProperty.call(SENT_ENTRIES, trigger)) {
|
|
508
|
+
throw new Error(
|
|
509
|
+
`[PublishLayer] flushBuffered() needs its trigger - got ${JSON.stringify(trigger)}. `
|
|
510
|
+
+ "Expected: 'reconnected' or 'unblocked' — they differ in what happens to a message whose frame "
|
|
511
|
+
+ 'was already sent (conf mq-publish-buffer 001). Fix: pass the event that ended the outage.'
|
|
512
|
+
);
|
|
513
|
+
}
|
|
276
514
|
const result = await this._buffer.flush(async (queue, buf, opts) => {
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
});
|
|
515
|
+
await this._client._publishOnce(queue, buf, opts);
|
|
516
|
+
}, { sentEntries: SENT_ENTRIES[trigger] });
|
|
280
517
|
|
|
281
|
-
this._logger.info(`[PublishLayer] Flushed ${result} buffered message(s)`);
|
|
518
|
+
this._logger.info(`[PublishLayer] Flushed ${result} buffered message(s) after ${trigger}`);
|
|
282
519
|
return result;
|
|
283
520
|
}
|
|
284
521
|
}
|
|
@@ -8,15 +8,12 @@
|
|
|
8
8
|
* run `active` on it (`api/infra/api_monitoring/src/consumer/index.js`). A service that
|
|
9
9
|
* does not file it starts runs that are invisible for their whole life.
|
|
10
10
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* the
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* half of the same wire (`docs/governance/confirmations/scheduler.md` 009). Both copies
|
|
18
|
-
* are retired by their own services in the change that adopts this module — no period
|
|
19
|
-
* with both (`architecture-principles.md` §11).
|
|
11
|
+
* One producer rail, by the owner's decision (`docs/governance/confirmations/scheduler.md`
|
|
12
|
+
* 009 and 024): the event is built and filed by this module, in the library that already
|
|
13
|
+
* owns the transport half of the same wire (`./monitoring-publish.js`). Every service that
|
|
14
|
+
* starts a run files it through here — the decision names the gateway and the scheduler.
|
|
15
|
+
* A service that builds the event in its own words is a second implementation of one
|
|
16
|
+
* concern, a defect by default (`change-discipline.md` § One rail per concern).
|
|
20
17
|
*
|
|
21
18
|
* WHAT THIS MODULE DOES NOT DO, deliberately:
|
|
22
19
|
*
|