@babelqueue/pulsar 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -4
- package/dist/index.cjs +206 -17
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +120 -6
- package/dist/index.d.ts +120 -6
- package/dist/index.js +206 -18
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.d.cts
CHANGED
|
@@ -20,7 +20,7 @@ import { Envelope, HeaderCarrier, IncomingEnvelope } from '@babelqueue/core';
|
|
|
20
20
|
* });
|
|
21
21
|
* const consumer = new PulsarConsumer(sub, {
|
|
22
22
|
* "urn:babel:orders:created": async (env) => { ... },
|
|
23
|
-
* });
|
|
23
|
+
* }, { subscriptionType: "Shared" }); // declare it (or pass `client` together with `topic`): see PulsarConsumerOptions
|
|
24
24
|
* await consumer.run();
|
|
25
25
|
*
|
|
26
26
|
* This implements §5 of the broker-bindings contract: the canonical envelope is the message
|
|
@@ -28,7 +28,12 @@ import { Envelope, HeaderCarrier, IncomingEnvelope } from '@babelqueue/core';
|
|
|
28
28
|
* `bq-trace-id` = trace_id, `bq-message-id` = meta.id, plus `bq-schema-version` /
|
|
29
29
|
* `bq-source-lang` / `bq-attempts`. The envelope is unchanged (`schema_version` stays 1);
|
|
30
30
|
* Pulsar is purely additive. A failed handler `negativeAcknowledge`s the message, so it is
|
|
31
|
-
* redelivered (at-least-once) and `getRedeliveryCount()` is incremented
|
|
31
|
+
* redelivered (at-least-once) and `getRedeliveryCount()` is incremented — up to `maxAttempts`
|
|
32
|
+
* deliveries (default 3, ADR-0036): the message that exhausts them is written to `<queue>.dlq`
|
|
33
|
+
* (when a `deadLetterProducer` is given) or acknowledged and dropped. A body that does not decode
|
|
34
|
+
* into an envelope (including an object with no valid identity) is poison: it never reaches a
|
|
35
|
+
* handler, and with dead-lettering on it is recorded in `<queue>.dlq` in the ADR-0035 shape without
|
|
36
|
+
* retry. Per §5.2 / §5.7 the
|
|
32
37
|
* consumer routes on the `bq-job` property (the body's URN is used only when the property is
|
|
33
38
|
* absent or blank) and takes `attempts` from the authoritative `bq-attempts` property (the
|
|
34
39
|
* body's value is used only when the property is absent or not a plain ASCII-decimal integer
|
|
@@ -79,10 +84,27 @@ interface PulsarReceivedMessage {
|
|
|
79
84
|
* satisfies it structurally; a fake satisfies it in tests.
|
|
80
85
|
*/
|
|
81
86
|
interface PulsarConsumerClient {
|
|
87
|
+
/**
|
|
88
|
+
* The topic the subscription reads. pulsar-client's `Consumer` has **no** such method (only its `Producer`
|
|
89
|
+
* does), so with it the topic comes from the {@link PulsarConsumerOptions.topic} option; a custom
|
|
90
|
+
* subscription wrapper may provide this to make `topic` unnecessary.
|
|
91
|
+
*/
|
|
92
|
+
getTopic?(): string;
|
|
82
93
|
receive(timeoutMs?: number): Promise<PulsarReceivedMessage>;
|
|
83
94
|
acknowledge(message: PulsarReceivedMessage): Promise<void> | void;
|
|
84
95
|
negativeAcknowledge(message: PulsarReceivedMessage): Promise<void> | void;
|
|
85
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* The subset of the Pulsar client this adapter calls to build its own retry producer. A `Client` from
|
|
99
|
+
* `pulsar-client` satisfies it structurally.
|
|
100
|
+
*/
|
|
101
|
+
interface PulsarClientLike {
|
|
102
|
+
createProducer(config: {
|
|
103
|
+
topic: string;
|
|
104
|
+
}): Promise<PulsarProducer>;
|
|
105
|
+
}
|
|
106
|
+
/** The subscription type, as pulsar-client names it (`Key_Shared` is accepted as an alias of `KeyShared`). */
|
|
107
|
+
type PulsarSubscriptionType = "Exclusive" | "Shared" | "Failover" | "KeyShared" | "Key_Shared";
|
|
86
108
|
/**
|
|
87
109
|
* Project the envelope's contract fields onto native Pulsar message properties
|
|
88
110
|
* (string→string): `bq-job` = URN, `bq-trace-id` = trace_id, `bq-message-id` = meta.id, plus
|
|
@@ -153,26 +175,118 @@ interface PulsarConsumerOptions {
|
|
|
153
175
|
* rejects with) is swallowed — it never changes how the message is handled or acknowledged.
|
|
154
176
|
*/
|
|
155
177
|
onRoutingMismatch?: (propertyUrn: string, bodyUrn: string, envelope: IncomingEnvelope) => unknown;
|
|
178
|
+
/**
|
|
179
|
+
* The total number of deliveries a message gets, the first included (default 3; `1` = no retry;
|
|
180
|
+
* ADR-0036 decision 1). A failing handler, an unmapped URN without {@link onUnknownUrn} and a poison
|
|
181
|
+
* body with dead-lettering off are retried (`negativeAcknowledge`) while `reconciled attempts + 1`
|
|
182
|
+
* is below it; the delivery that reaches it is dead-lettered (when {@link deadLetterProducer} is
|
|
183
|
+
* set) or acknowledged and dropped. Must be an integer of at least 1.
|
|
184
|
+
*/
|
|
185
|
+
maxAttempts?: number;
|
|
186
|
+
/**
|
|
187
|
+
* A producer on the dead-letter topic (by convention `<queue>.dlq`). Dead-lettering is on exactly
|
|
188
|
+
* when it is set; the SDK does not use Pulsar's native `DeadLetterPolicy`. The original message is
|
|
189
|
+
* acknowledged only after the record was sent; a failed send leaves it negatively acknowledged so
|
|
190
|
+
* it is redelivered and the write is tried again.
|
|
191
|
+
*/
|
|
192
|
+
deadLetterProducer?: PulsarProducer;
|
|
193
|
+
/**
|
|
194
|
+
* A producer on the consumer's **own** topic. A retry re-publishes the message to that topic with
|
|
195
|
+
* `bq-attempts` (and the body's `attempts`) raised to the counted value and then acknowledges the
|
|
196
|
+
* original — the Go model. The count then lives in the message itself, so the bound holds on every
|
|
197
|
+
* subscription type. Optional when {@link client} is given (the consumer builds the producer itself);
|
|
198
|
+
* a failed re-publish leaves the original negatively acknowledged (never lost).
|
|
199
|
+
*
|
|
200
|
+
* Without a producer or a client a retry is a `negativeAcknowledge`, which advances the native
|
|
201
|
+
* redelivery count only on `Shared` / `Key_Shared`: on `Exclusive` / `Failover` it stays 0 and the retry
|
|
202
|
+
* would never reach `maxAttempts`, so that combination is rejected at construction (see
|
|
203
|
+
* {@link subscriptionType}).
|
|
204
|
+
*/
|
|
205
|
+
retryProducer?: PulsarProducer;
|
|
206
|
+
/**
|
|
207
|
+
* The Pulsar client the consumer builds its retry producer from, lazily on the first retry, on
|
|
208
|
+
* {@link topic}. Needs `topic` (pulsar-client's `Consumer` cannot tell its topic). Ignored when
|
|
209
|
+
* {@link retryProducer} is given.
|
|
210
|
+
*/
|
|
211
|
+
client?: PulsarClientLike;
|
|
212
|
+
/**
|
|
213
|
+
* The topic the retry producer is created on — the topic the consumer reads. Required together with
|
|
214
|
+
* {@link client} (a pulsar-client `Consumer` has no `getTopic()`); a subscription wrapper that provides
|
|
215
|
+
* `getTopic()` makes it optional.
|
|
216
|
+
*/
|
|
217
|
+
topic?: string;
|
|
218
|
+
/**
|
|
219
|
+
* The subscription type the consumer was created with. `Exclusive` and `Failover` cannot bound a
|
|
220
|
+
* nack-driven retry (a negative acknowledgement does not advance the redelivery count there), so with
|
|
221
|
+
* `maxAttempts > 1` they require a {@link retryProducer} or a {@link client}. **Not declaring it counts as
|
|
222
|
+
* `Exclusive`** (pulsar-client's own default, and the fail-closed choice): without a `retryProducer` or a
|
|
223
|
+
* `client` the constructor throws `BabelQueueError`. Declare `"Shared"` or `"Key_Shared"` to keep the
|
|
224
|
+
* `negativeAcknowledge` retry without a producer.
|
|
225
|
+
*/
|
|
226
|
+
subscriptionType?: PulsarSubscriptionType;
|
|
227
|
+
/**
|
|
228
|
+
* The queue name written to `dead_letter.original_queue` of a poison record (a decodable envelope
|
|
229
|
+
* uses its own `meta.queue`). Defaults to the {@link deadLetterProducer} topic without its `.dlq`
|
|
230
|
+
* suffix.
|
|
231
|
+
*/
|
|
232
|
+
queue?: string;
|
|
233
|
+
/** Receives the "message dropped" and "dead-letter write failed" warnings (default `console.warn`). */
|
|
234
|
+
onWarn?: (message: string) => void;
|
|
156
235
|
}
|
|
236
|
+
/** Default total number of deliveries (ADR-0036 decision 1). */
|
|
237
|
+
declare const DEFAULT_MAX_ATTEMPTS = 3;
|
|
157
238
|
/**
|
|
158
239
|
* Receives from a Pulsar subscription, decodes + validates each message, routes it to the
|
|
159
240
|
* handler for its URN, and acknowledges it on success. A throwing handler
|
|
160
241
|
* `negativeAcknowledge`s the message — the broker redelivers it and increments
|
|
161
|
-
* `getRedeliveryCount()` (at-least-once
|
|
162
|
-
*
|
|
163
|
-
* `
|
|
242
|
+
* `getRedeliveryCount()` (at-least-once; with a `retryProducer` the message is instead re-published with
|
|
243
|
+
* `bq-attempts` + 1, which also bounds `Exclusive` / `Failover` subscriptions) — until `maxAttempts` deliveries are used up; the last one is
|
|
244
|
+
* dead-lettered to `<queue>.dlq` or acknowledged and dropped (ADR-0036). Routing uses the `bq-job`
|
|
245
|
+
* property (body URN as the fallback); `attempts` is the `bq-attempts` property (body value as the
|
|
246
|
+
* fallback) reconciled to `max(.., getRedeliveryCount())` for the handler.
|
|
164
247
|
*/
|
|
165
248
|
declare class PulsarConsumer {
|
|
166
249
|
private readonly consumer;
|
|
167
250
|
private readonly handlers;
|
|
168
251
|
private readonly options;
|
|
252
|
+
private readonly maxAttempts;
|
|
253
|
+
private builtRetryProducer;
|
|
254
|
+
private readonly warn;
|
|
169
255
|
constructor(consumer: PulsarConsumerClient, handlers: BabelHandlers, options?: PulsarConsumerOptions);
|
|
256
|
+
/**
|
|
257
|
+
* Setup check: on `Exclusive` / `Failover` (and an undeclared type) a negative acknowledgement does not advance the native
|
|
258
|
+
* redelivery count, so a retry needs a re-publish. Fail loudly here instead of looping at run time.
|
|
259
|
+
*/
|
|
260
|
+
private checkRetryBound;
|
|
261
|
+
private retryTopic;
|
|
262
|
+
/** The retry producer: the one given, else one built (once) from the client; undefined when neither exists. */
|
|
263
|
+
private resolveRetryProducer;
|
|
170
264
|
/** Receive one message (up to the receive timeout), route + settle it. Returns 1, or 0 on timeout. */
|
|
171
265
|
poll(): Promise<number>;
|
|
172
266
|
/** Poll until `signal` aborts. */
|
|
173
267
|
run(signal?: AbortSignal): Promise<void>;
|
|
174
268
|
private handle;
|
|
269
|
+
/**
|
|
270
|
+
* The settle decision of a retryable outcome (ADR-0036 decisions 1 and 2). `counted` is the reconciled
|
|
271
|
+
* attempts plus one for this delivery. A poison body with dead-lettering on is recorded without retry;
|
|
272
|
+
* every other outcome retries (`negativeAcknowledge`: the native redelivery count advances, so the
|
|
273
|
+
* next delivery counts one more) while `counted < maxAttempts`, then dead-letters or drops.
|
|
274
|
+
*/
|
|
275
|
+
private settle;
|
|
276
|
+
/**
|
|
277
|
+
* Retry one delivery. With a {@link PulsarConsumerOptions.retryProducer}: re-publish to the consumer's
|
|
278
|
+
* own topic carrying `bq-attempts = counted` (reconcile = max(bq-attempts, RedeliveryCount) on the next
|
|
279
|
+
* delivery), then acknowledge the original — only after the send succeeded. Otherwise `negativeAcknowledge`.
|
|
280
|
+
*/
|
|
281
|
+
private retry;
|
|
282
|
+
private retryMessage;
|
|
283
|
+
/**
|
|
284
|
+
* Write the DLQ record, then acknowledge the original — only after the send succeeded. A failed send
|
|
285
|
+
* leaves the original negatively acknowledged, so it is redelivered and the write is tried again.
|
|
286
|
+
*/
|
|
287
|
+
private deadLetter;
|
|
288
|
+
private deadLetterMessage;
|
|
175
289
|
private notifyRoutingMismatch;
|
|
176
290
|
}
|
|
177
291
|
|
|
178
|
-
export { type BabelHandler, type BabelHandlers, type PublishOptions, PulsarConsumer, type PulsarConsumerClient, type PulsarConsumerOptions, type PulsarProducer, type PulsarProducerMessage, PulsarPublisher, type PulsarReceivedMessage, headersOf, pulsarProperties, toPulsarMessage };
|
|
292
|
+
export { type BabelHandler, type BabelHandlers, DEFAULT_MAX_ATTEMPTS, type PublishOptions, type PulsarClientLike, PulsarConsumer, type PulsarConsumerClient, type PulsarConsumerOptions, type PulsarProducer, type PulsarProducerMessage, PulsarPublisher, type PulsarReceivedMessage, type PulsarSubscriptionType, headersOf, pulsarProperties, toPulsarMessage };
|
package/dist/index.d.ts
CHANGED
|
@@ -20,7 +20,7 @@ import { Envelope, HeaderCarrier, IncomingEnvelope } from '@babelqueue/core';
|
|
|
20
20
|
* });
|
|
21
21
|
* const consumer = new PulsarConsumer(sub, {
|
|
22
22
|
* "urn:babel:orders:created": async (env) => { ... },
|
|
23
|
-
* });
|
|
23
|
+
* }, { subscriptionType: "Shared" }); // declare it (or pass `client` together with `topic`): see PulsarConsumerOptions
|
|
24
24
|
* await consumer.run();
|
|
25
25
|
*
|
|
26
26
|
* This implements §5 of the broker-bindings contract: the canonical envelope is the message
|
|
@@ -28,7 +28,12 @@ import { Envelope, HeaderCarrier, IncomingEnvelope } from '@babelqueue/core';
|
|
|
28
28
|
* `bq-trace-id` = trace_id, `bq-message-id` = meta.id, plus `bq-schema-version` /
|
|
29
29
|
* `bq-source-lang` / `bq-attempts`. The envelope is unchanged (`schema_version` stays 1);
|
|
30
30
|
* Pulsar is purely additive. A failed handler `negativeAcknowledge`s the message, so it is
|
|
31
|
-
* redelivered (at-least-once) and `getRedeliveryCount()` is incremented
|
|
31
|
+
* redelivered (at-least-once) and `getRedeliveryCount()` is incremented — up to `maxAttempts`
|
|
32
|
+
* deliveries (default 3, ADR-0036): the message that exhausts them is written to `<queue>.dlq`
|
|
33
|
+
* (when a `deadLetterProducer` is given) or acknowledged and dropped. A body that does not decode
|
|
34
|
+
* into an envelope (including an object with no valid identity) is poison: it never reaches a
|
|
35
|
+
* handler, and with dead-lettering on it is recorded in `<queue>.dlq` in the ADR-0035 shape without
|
|
36
|
+
* retry. Per §5.2 / §5.7 the
|
|
32
37
|
* consumer routes on the `bq-job` property (the body's URN is used only when the property is
|
|
33
38
|
* absent or blank) and takes `attempts` from the authoritative `bq-attempts` property (the
|
|
34
39
|
* body's value is used only when the property is absent or not a plain ASCII-decimal integer
|
|
@@ -79,10 +84,27 @@ interface PulsarReceivedMessage {
|
|
|
79
84
|
* satisfies it structurally; a fake satisfies it in tests.
|
|
80
85
|
*/
|
|
81
86
|
interface PulsarConsumerClient {
|
|
87
|
+
/**
|
|
88
|
+
* The topic the subscription reads. pulsar-client's `Consumer` has **no** such method (only its `Producer`
|
|
89
|
+
* does), so with it the topic comes from the {@link PulsarConsumerOptions.topic} option; a custom
|
|
90
|
+
* subscription wrapper may provide this to make `topic` unnecessary.
|
|
91
|
+
*/
|
|
92
|
+
getTopic?(): string;
|
|
82
93
|
receive(timeoutMs?: number): Promise<PulsarReceivedMessage>;
|
|
83
94
|
acknowledge(message: PulsarReceivedMessage): Promise<void> | void;
|
|
84
95
|
negativeAcknowledge(message: PulsarReceivedMessage): Promise<void> | void;
|
|
85
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* The subset of the Pulsar client this adapter calls to build its own retry producer. A `Client` from
|
|
99
|
+
* `pulsar-client` satisfies it structurally.
|
|
100
|
+
*/
|
|
101
|
+
interface PulsarClientLike {
|
|
102
|
+
createProducer(config: {
|
|
103
|
+
topic: string;
|
|
104
|
+
}): Promise<PulsarProducer>;
|
|
105
|
+
}
|
|
106
|
+
/** The subscription type, as pulsar-client names it (`Key_Shared` is accepted as an alias of `KeyShared`). */
|
|
107
|
+
type PulsarSubscriptionType = "Exclusive" | "Shared" | "Failover" | "KeyShared" | "Key_Shared";
|
|
86
108
|
/**
|
|
87
109
|
* Project the envelope's contract fields onto native Pulsar message properties
|
|
88
110
|
* (string→string): `bq-job` = URN, `bq-trace-id` = trace_id, `bq-message-id` = meta.id, plus
|
|
@@ -153,26 +175,118 @@ interface PulsarConsumerOptions {
|
|
|
153
175
|
* rejects with) is swallowed — it never changes how the message is handled or acknowledged.
|
|
154
176
|
*/
|
|
155
177
|
onRoutingMismatch?: (propertyUrn: string, bodyUrn: string, envelope: IncomingEnvelope) => unknown;
|
|
178
|
+
/**
|
|
179
|
+
* The total number of deliveries a message gets, the first included (default 3; `1` = no retry;
|
|
180
|
+
* ADR-0036 decision 1). A failing handler, an unmapped URN without {@link onUnknownUrn} and a poison
|
|
181
|
+
* body with dead-lettering off are retried (`negativeAcknowledge`) while `reconciled attempts + 1`
|
|
182
|
+
* is below it; the delivery that reaches it is dead-lettered (when {@link deadLetterProducer} is
|
|
183
|
+
* set) or acknowledged and dropped. Must be an integer of at least 1.
|
|
184
|
+
*/
|
|
185
|
+
maxAttempts?: number;
|
|
186
|
+
/**
|
|
187
|
+
* A producer on the dead-letter topic (by convention `<queue>.dlq`). Dead-lettering is on exactly
|
|
188
|
+
* when it is set; the SDK does not use Pulsar's native `DeadLetterPolicy`. The original message is
|
|
189
|
+
* acknowledged only after the record was sent; a failed send leaves it negatively acknowledged so
|
|
190
|
+
* it is redelivered and the write is tried again.
|
|
191
|
+
*/
|
|
192
|
+
deadLetterProducer?: PulsarProducer;
|
|
193
|
+
/**
|
|
194
|
+
* A producer on the consumer's **own** topic. A retry re-publishes the message to that topic with
|
|
195
|
+
* `bq-attempts` (and the body's `attempts`) raised to the counted value and then acknowledges the
|
|
196
|
+
* original — the Go model. The count then lives in the message itself, so the bound holds on every
|
|
197
|
+
* subscription type. Optional when {@link client} is given (the consumer builds the producer itself);
|
|
198
|
+
* a failed re-publish leaves the original negatively acknowledged (never lost).
|
|
199
|
+
*
|
|
200
|
+
* Without a producer or a client a retry is a `negativeAcknowledge`, which advances the native
|
|
201
|
+
* redelivery count only on `Shared` / `Key_Shared`: on `Exclusive` / `Failover` it stays 0 and the retry
|
|
202
|
+
* would never reach `maxAttempts`, so that combination is rejected at construction (see
|
|
203
|
+
* {@link subscriptionType}).
|
|
204
|
+
*/
|
|
205
|
+
retryProducer?: PulsarProducer;
|
|
206
|
+
/**
|
|
207
|
+
* The Pulsar client the consumer builds its retry producer from, lazily on the first retry, on
|
|
208
|
+
* {@link topic}. Needs `topic` (pulsar-client's `Consumer` cannot tell its topic). Ignored when
|
|
209
|
+
* {@link retryProducer} is given.
|
|
210
|
+
*/
|
|
211
|
+
client?: PulsarClientLike;
|
|
212
|
+
/**
|
|
213
|
+
* The topic the retry producer is created on — the topic the consumer reads. Required together with
|
|
214
|
+
* {@link client} (a pulsar-client `Consumer` has no `getTopic()`); a subscription wrapper that provides
|
|
215
|
+
* `getTopic()` makes it optional.
|
|
216
|
+
*/
|
|
217
|
+
topic?: string;
|
|
218
|
+
/**
|
|
219
|
+
* The subscription type the consumer was created with. `Exclusive` and `Failover` cannot bound a
|
|
220
|
+
* nack-driven retry (a negative acknowledgement does not advance the redelivery count there), so with
|
|
221
|
+
* `maxAttempts > 1` they require a {@link retryProducer} or a {@link client}. **Not declaring it counts as
|
|
222
|
+
* `Exclusive`** (pulsar-client's own default, and the fail-closed choice): without a `retryProducer` or a
|
|
223
|
+
* `client` the constructor throws `BabelQueueError`. Declare `"Shared"` or `"Key_Shared"` to keep the
|
|
224
|
+
* `negativeAcknowledge` retry without a producer.
|
|
225
|
+
*/
|
|
226
|
+
subscriptionType?: PulsarSubscriptionType;
|
|
227
|
+
/**
|
|
228
|
+
* The queue name written to `dead_letter.original_queue` of a poison record (a decodable envelope
|
|
229
|
+
* uses its own `meta.queue`). Defaults to the {@link deadLetterProducer} topic without its `.dlq`
|
|
230
|
+
* suffix.
|
|
231
|
+
*/
|
|
232
|
+
queue?: string;
|
|
233
|
+
/** Receives the "message dropped" and "dead-letter write failed" warnings (default `console.warn`). */
|
|
234
|
+
onWarn?: (message: string) => void;
|
|
156
235
|
}
|
|
236
|
+
/** Default total number of deliveries (ADR-0036 decision 1). */
|
|
237
|
+
declare const DEFAULT_MAX_ATTEMPTS = 3;
|
|
157
238
|
/**
|
|
158
239
|
* Receives from a Pulsar subscription, decodes + validates each message, routes it to the
|
|
159
240
|
* handler for its URN, and acknowledges it on success. A throwing handler
|
|
160
241
|
* `negativeAcknowledge`s the message — the broker redelivers it and increments
|
|
161
|
-
* `getRedeliveryCount()` (at-least-once
|
|
162
|
-
*
|
|
163
|
-
* `
|
|
242
|
+
* `getRedeliveryCount()` (at-least-once; with a `retryProducer` the message is instead re-published with
|
|
243
|
+
* `bq-attempts` + 1, which also bounds `Exclusive` / `Failover` subscriptions) — until `maxAttempts` deliveries are used up; the last one is
|
|
244
|
+
* dead-lettered to `<queue>.dlq` or acknowledged and dropped (ADR-0036). Routing uses the `bq-job`
|
|
245
|
+
* property (body URN as the fallback); `attempts` is the `bq-attempts` property (body value as the
|
|
246
|
+
* fallback) reconciled to `max(.., getRedeliveryCount())` for the handler.
|
|
164
247
|
*/
|
|
165
248
|
declare class PulsarConsumer {
|
|
166
249
|
private readonly consumer;
|
|
167
250
|
private readonly handlers;
|
|
168
251
|
private readonly options;
|
|
252
|
+
private readonly maxAttempts;
|
|
253
|
+
private builtRetryProducer;
|
|
254
|
+
private readonly warn;
|
|
169
255
|
constructor(consumer: PulsarConsumerClient, handlers: BabelHandlers, options?: PulsarConsumerOptions);
|
|
256
|
+
/**
|
|
257
|
+
* Setup check: on `Exclusive` / `Failover` (and an undeclared type) a negative acknowledgement does not advance the native
|
|
258
|
+
* redelivery count, so a retry needs a re-publish. Fail loudly here instead of looping at run time.
|
|
259
|
+
*/
|
|
260
|
+
private checkRetryBound;
|
|
261
|
+
private retryTopic;
|
|
262
|
+
/** The retry producer: the one given, else one built (once) from the client; undefined when neither exists. */
|
|
263
|
+
private resolveRetryProducer;
|
|
170
264
|
/** Receive one message (up to the receive timeout), route + settle it. Returns 1, or 0 on timeout. */
|
|
171
265
|
poll(): Promise<number>;
|
|
172
266
|
/** Poll until `signal` aborts. */
|
|
173
267
|
run(signal?: AbortSignal): Promise<void>;
|
|
174
268
|
private handle;
|
|
269
|
+
/**
|
|
270
|
+
* The settle decision of a retryable outcome (ADR-0036 decisions 1 and 2). `counted` is the reconciled
|
|
271
|
+
* attempts plus one for this delivery. A poison body with dead-lettering on is recorded without retry;
|
|
272
|
+
* every other outcome retries (`negativeAcknowledge`: the native redelivery count advances, so the
|
|
273
|
+
* next delivery counts one more) while `counted < maxAttempts`, then dead-letters or drops.
|
|
274
|
+
*/
|
|
275
|
+
private settle;
|
|
276
|
+
/**
|
|
277
|
+
* Retry one delivery. With a {@link PulsarConsumerOptions.retryProducer}: re-publish to the consumer's
|
|
278
|
+
* own topic carrying `bq-attempts = counted` (reconcile = max(bq-attempts, RedeliveryCount) on the next
|
|
279
|
+
* delivery), then acknowledge the original — only after the send succeeded. Otherwise `negativeAcknowledge`.
|
|
280
|
+
*/
|
|
281
|
+
private retry;
|
|
282
|
+
private retryMessage;
|
|
283
|
+
/**
|
|
284
|
+
* Write the DLQ record, then acknowledge the original — only after the send succeeded. A failed send
|
|
285
|
+
* leaves the original negatively acknowledged, so it is redelivered and the write is tried again.
|
|
286
|
+
*/
|
|
287
|
+
private deadLetter;
|
|
288
|
+
private deadLetterMessage;
|
|
175
289
|
private notifyRoutingMismatch;
|
|
176
290
|
}
|
|
177
291
|
|
|
178
|
-
export { type BabelHandler, type BabelHandlers, type PublishOptions, PulsarConsumer, type PulsarConsumerClient, type PulsarConsumerOptions, type PulsarProducer, type PulsarProducerMessage, PulsarPublisher, type PulsarReceivedMessage, headersOf, pulsarProperties, toPulsarMessage };
|
|
292
|
+
export { type BabelHandler, type BabelHandlers, DEFAULT_MAX_ATTEMPTS, type PublishOptions, type PulsarClientLike, PulsarConsumer, type PulsarConsumerClient, type PulsarConsumerOptions, type PulsarProducer, type PulsarProducerMessage, PulsarPublisher, type PulsarReceivedMessage, type PulsarSubscriptionType, headersOf, pulsarProperties, toPulsarMessage };
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/index.ts
|
|
2
|
-
import { BabelQueueError, EnvelopeCodec, SCHEMA_VERSION, UnknownUrnError } from "@babelqueue/core";
|
|
2
|
+
import { BabelQueueError, EnvelopeCodec, SCHEMA_VERSION, UnknownUrnError, annotate, buildPoisonRecord } from "@babelqueue/core";
|
|
3
3
|
|
|
4
4
|
// src/headers.ts
|
|
5
5
|
function sanitizeHeaders(headers) {
|
|
@@ -77,12 +77,19 @@ var PulsarPublisher = class {
|
|
|
77
77
|
return envelope.meta.id;
|
|
78
78
|
}
|
|
79
79
|
};
|
|
80
|
+
var DEFAULT_MAX_ATTEMPTS = 3;
|
|
80
81
|
var MAX_ATTEMPTS = 2 ** 31 - 1;
|
|
81
82
|
function parseAttempts(raw) {
|
|
82
83
|
if (typeof raw !== "string" || !/^[0-9]+$/.test(raw)) return void 0;
|
|
83
84
|
const value = Number(raw);
|
|
84
85
|
return value <= MAX_ATTEMPTS ? value : void 0;
|
|
85
86
|
}
|
|
87
|
+
function reconcileAttempts(properties, bodyAttempts, redeliveryCount) {
|
|
88
|
+
let attempts = bodyAttempts;
|
|
89
|
+
const fromProperty = parseAttempts(properties["bq-attempts"]);
|
|
90
|
+
if (fromProperty !== void 0) attempts = fromProperty;
|
|
91
|
+
return redeliveryCount > attempts ? redeliveryCount : attempts;
|
|
92
|
+
}
|
|
86
93
|
var BLANK_ASCII = /^[ \t\n\v\f\r]*$/;
|
|
87
94
|
function unsupportedSchemaVersion(properties) {
|
|
88
95
|
const raw = properties["bq-schema-version"];
|
|
@@ -101,10 +108,62 @@ var PulsarConsumer = class {
|
|
|
101
108
|
this.consumer = consumer;
|
|
102
109
|
this.handlers = handlers;
|
|
103
110
|
this.options = options;
|
|
111
|
+
const maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
112
|
+
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
|
113
|
+
throw new BabelQueueError(`maxAttempts must be an integer of at least 1, got ${String(maxAttempts)}.`);
|
|
114
|
+
}
|
|
115
|
+
this.maxAttempts = maxAttempts;
|
|
116
|
+
this.checkRetryBound(maxAttempts);
|
|
117
|
+
this.warn = options.onWarn ?? ((message) => console.warn(message));
|
|
104
118
|
}
|
|
105
119
|
consumer;
|
|
106
120
|
handlers;
|
|
107
121
|
options;
|
|
122
|
+
maxAttempts;
|
|
123
|
+
builtRetryProducer;
|
|
124
|
+
warn;
|
|
125
|
+
/**
|
|
126
|
+
* Setup check: on `Exclusive` / `Failover` (and an undeclared type) a negative acknowledgement does not advance the native
|
|
127
|
+
* redelivery count, so a retry needs a re-publish. Fail loudly here instead of looping at run time.
|
|
128
|
+
*/
|
|
129
|
+
checkRetryBound(maxAttempts) {
|
|
130
|
+
const declared = this.options.subscriptionType;
|
|
131
|
+
if (declared !== void 0 && !["Exclusive", "Shared", "Failover", "KeyShared"].includes(declared.replace("_", ""))) {
|
|
132
|
+
throw new BabelQueueError(`Unknown subscriptionType "${String(declared)}".`);
|
|
133
|
+
}
|
|
134
|
+
const type = declared === void 0 ? "Exclusive" : declared.replace("_", "");
|
|
135
|
+
if (type !== "Exclusive" && type !== "Failover") return;
|
|
136
|
+
if (maxAttempts === 1 || this.options.retryProducer !== void 0) return;
|
|
137
|
+
if (this.options.client !== void 0 && this.retryTopic() !== void 0) return;
|
|
138
|
+
const shown = declared === void 0 ? "An undeclared subscriptionType (treated as Exclusive)" : `A ${type} subscription`;
|
|
139
|
+
if (this.options.client !== void 0) {
|
|
140
|
+
throw new BabelQueueError(
|
|
141
|
+
`${shown} with a client needs the \`topic\` option (the topic this consumer reads) to build the retry producer: pulsar-client's Consumer cannot report its own topic.`
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
throw new BabelQueueError(
|
|
145
|
+
`${shown} needs a retryProducer (or a client plus a topic) to bound retries: a negative acknowledgement does not advance the redelivery count there, so a retry would never reach maxAttempts. Pass a client plus topic (the topic you subscribed to), or a retryProducer, or declare subscriptionType: "Shared" / "Key_Shared".`
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
retryTopic() {
|
|
149
|
+
const configured = this.options.topic;
|
|
150
|
+
if (configured !== void 0 && configured !== "") return configured;
|
|
151
|
+
const own = typeof this.consumer.getTopic === "function" ? this.consumer.getTopic() : void 0;
|
|
152
|
+
return own !== void 0 && own !== "" ? own : void 0;
|
|
153
|
+
}
|
|
154
|
+
/** The retry producer: the one given, else one built (once) from the client; undefined when neither exists. */
|
|
155
|
+
async resolveRetryProducer() {
|
|
156
|
+
if (this.options.retryProducer !== void 0) return this.options.retryProducer;
|
|
157
|
+
const client = this.options.client;
|
|
158
|
+
const topic = this.retryTopic();
|
|
159
|
+
if (client === void 0 || topic === void 0) return void 0;
|
|
160
|
+
this.builtRetryProducer ??= client.createProducer({ topic }).catch((error) => {
|
|
161
|
+
this.builtRetryProducer = void 0;
|
|
162
|
+
this.warn(`babelqueue: creating the retry producer on ${topic} failed (${errorText(error)}).`);
|
|
163
|
+
return void 0;
|
|
164
|
+
});
|
|
165
|
+
return this.builtRetryProducer;
|
|
166
|
+
}
|
|
108
167
|
/** Receive one message (up to the receive timeout), route + settle it. Returns 1, or 0 on timeout. */
|
|
109
168
|
async poll() {
|
|
110
169
|
let message;
|
|
@@ -137,29 +196,23 @@ var PulsarConsumer = class {
|
|
|
137
196
|
await this.consumer.negativeAcknowledge(message);
|
|
138
197
|
return;
|
|
139
198
|
}
|
|
140
|
-
const envelope = EnvelopeCodec.decode(decodePayload(message.getData()));
|
|
141
199
|
const redeliveryCount = typeof message.getRedeliveryCount === "function" ? message.getRedeliveryCount() : 0;
|
|
142
|
-
const
|
|
143
|
-
let attempts = typeof bodyAttempts === "number" && Number.isInteger(bodyAttempts) ? bodyAttempts : 0;
|
|
144
|
-
const fromProperty = parseAttempts(properties["bq-attempts"]);
|
|
145
|
-
if (fromProperty !== void 0) attempts = fromProperty;
|
|
146
|
-
if (redeliveryCount > attempts) attempts = redeliveryCount;
|
|
147
|
-
if (typeof bodyAttempts === "number" && Number.isInteger(bodyAttempts) && bodyAttempts !== attempts) {
|
|
148
|
-
envelope.attempts = attempts;
|
|
149
|
-
}
|
|
200
|
+
const envelope = EnvelopeCodec.decode(decodePayload(message.getData()));
|
|
150
201
|
const bodyUrn = EnvelopeCodec.urn(envelope);
|
|
151
202
|
const rawJob = properties["bq-job"];
|
|
152
203
|
const propertyUrn = typeof rawJob === "string" ? rawJob.trim() : "";
|
|
153
204
|
const urn = propertyUrn !== "" ? propertyUrn : bodyUrn;
|
|
154
205
|
if (!EnvelopeCodec.accepts(envelope)) {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
message
|
|
159
|
-
);
|
|
160
|
-
await this.consumer.negativeAcknowledge(message);
|
|
206
|
+
const error = new BabelQueueError("Rejected a non-conformant BabelQueue envelope from Apache Pulsar.");
|
|
207
|
+
this.options.onError?.(error, envelope, message);
|
|
208
|
+
await this.settle("poison", reconcileAttempts(properties, 0, redeliveryCount) + 1, null, message, error);
|
|
161
209
|
return;
|
|
162
210
|
}
|
|
211
|
+
const bodyAttempts = envelope.attempts;
|
|
212
|
+
const attempts = reconcileAttempts(properties, bodyAttempts, redeliveryCount);
|
|
213
|
+
if (bodyAttempts !== attempts) {
|
|
214
|
+
envelope.attempts = attempts;
|
|
215
|
+
}
|
|
163
216
|
if (propertyUrn !== "" && bodyUrn !== "" && propertyUrn !== bodyUrn) {
|
|
164
217
|
this.notifyRoutingMismatch(propertyUrn, bodyUrn, envelope);
|
|
165
218
|
}
|
|
@@ -169,8 +222,9 @@ var PulsarConsumer = class {
|
|
|
169
222
|
await this.options.onUnknownUrn(envelope, message);
|
|
170
223
|
await this.consumer.acknowledge(message);
|
|
171
224
|
} else {
|
|
172
|
-
|
|
173
|
-
|
|
225
|
+
const error = new UnknownUrnError(urn);
|
|
226
|
+
this.options.onError?.(error, envelope, message);
|
|
227
|
+
await this.settle("unknown_urn", attempts + 1, envelope, message, error);
|
|
174
228
|
}
|
|
175
229
|
return;
|
|
176
230
|
}
|
|
@@ -179,8 +233,108 @@ var PulsarConsumer = class {
|
|
|
179
233
|
await this.consumer.acknowledge(message);
|
|
180
234
|
} catch (error) {
|
|
181
235
|
this.options.onError?.(error, envelope, message);
|
|
236
|
+
await this.settle("failed", attempts + 1, envelope, message, error);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* The settle decision of a retryable outcome (ADR-0036 decisions 1 and 2). `counted` is the reconciled
|
|
241
|
+
* attempts plus one for this delivery. A poison body with dead-lettering on is recorded without retry;
|
|
242
|
+
* every other outcome retries (`negativeAcknowledge`: the native redelivery count advances, so the
|
|
243
|
+
* next delivery counts one more) while `counted < maxAttempts`, then dead-letters or drops.
|
|
244
|
+
*/
|
|
245
|
+
async settle(kind, counted, envelope, message, error) {
|
|
246
|
+
const producer = this.options.deadLetterProducer;
|
|
247
|
+
if (kind === "poison" && producer !== void 0) {
|
|
248
|
+
await this.deadLetter(producer, kind, counted, null, message, error);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
if (counted < this.maxAttempts) {
|
|
252
|
+
await this.retry(counted, envelope, message);
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
if (producer !== void 0) {
|
|
256
|
+
await this.deadLetter(producer, kind, counted, envelope, message, error);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
this.warn(
|
|
260
|
+
`babelqueue: dropped a ${kind} Apache Pulsar message after ${counted} of ${this.maxAttempts} deliveries (dead-lettering is off).`
|
|
261
|
+
);
|
|
262
|
+
await this.consumer.acknowledge(message);
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Retry one delivery. With a {@link PulsarConsumerOptions.retryProducer}: re-publish to the consumer's
|
|
266
|
+
* own topic carrying `bq-attempts = counted` (reconcile = max(bq-attempts, RedeliveryCount) on the next
|
|
267
|
+
* delivery), then acknowledge the original — only after the send succeeded. Otherwise `negativeAcknowledge`.
|
|
268
|
+
*/
|
|
269
|
+
async retry(counted, envelope, message) {
|
|
270
|
+
const producer = await this.resolveRetryProducer();
|
|
271
|
+
if (producer === void 0) {
|
|
272
|
+
await this.consumer.negativeAcknowledge(message);
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
try {
|
|
276
|
+
await producer.send(this.retryMessage(counted, envelope, message));
|
|
277
|
+
} catch (writeError) {
|
|
278
|
+
this.warn(
|
|
279
|
+
`babelqueue: retry re-publish to ${producer.getTopic()} failed (${errorText(writeError)}); the message is left for redelivery.`
|
|
280
|
+
);
|
|
281
|
+
await this.consumer.negativeAcknowledge(message);
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
await this.consumer.acknowledge(message);
|
|
285
|
+
}
|
|
286
|
+
retryMessage(counted, envelope, message) {
|
|
287
|
+
const carried = headersOf(message);
|
|
288
|
+
delete carried["bq-delay"];
|
|
289
|
+
if (envelope !== null) {
|
|
290
|
+
return keepRoute(toPulsarMessage({ ...envelope, attempts: counted }, carried), message);
|
|
291
|
+
}
|
|
292
|
+
const data = message.getData();
|
|
293
|
+
return keepRoute(
|
|
294
|
+
{
|
|
295
|
+
data: typeof data === "string" ? Buffer.from(data, "utf8") : Buffer.from(data),
|
|
296
|
+
properties: { ...carried, "bq-attempts": String(counted) }
|
|
297
|
+
},
|
|
298
|
+
message
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Write the DLQ record, then acknowledge the original — only after the send succeeded. A failed send
|
|
303
|
+
* leaves the original negatively acknowledged, so it is redelivered and the write is tried again.
|
|
304
|
+
*/
|
|
305
|
+
async deadLetter(producer, kind, counted, envelope, message, error) {
|
|
306
|
+
try {
|
|
307
|
+
await producer.send(this.deadLetterMessage(producer, kind, counted, envelope, message, error));
|
|
308
|
+
} catch (writeError) {
|
|
309
|
+
this.warn(
|
|
310
|
+
`babelqueue: dead-letter write to ${producer.getTopic()} failed (${errorText(writeError)}); the message is left for redelivery.`
|
|
311
|
+
);
|
|
182
312
|
await this.consumer.negativeAcknowledge(message);
|
|
313
|
+
return;
|
|
183
314
|
}
|
|
315
|
+
await this.consumer.acknowledge(message);
|
|
316
|
+
}
|
|
317
|
+
deadLetterMessage(producer, kind, counted, envelope, message, error) {
|
|
318
|
+
const details = {
|
|
319
|
+
attempts: counted,
|
|
320
|
+
error: error instanceof Error ? error.message : null,
|
|
321
|
+
exception: error instanceof Error ? error.name : null
|
|
322
|
+
};
|
|
323
|
+
if (envelope === null) {
|
|
324
|
+
const original2 = this.options.queue ?? stripDlqSuffix(topicName(producer.getTopic()));
|
|
325
|
+
const delivered = counted - 1;
|
|
326
|
+
const deadLetter = annotate({ attempts: delivered }, kind, original2, {
|
|
327
|
+
...details,
|
|
328
|
+
attempts: delivered
|
|
329
|
+
}).dead_letter;
|
|
330
|
+
const data = message.getData();
|
|
331
|
+
const body = typeof data === "string" ? Buffer.from(data, "utf8") : Buffer.from(data);
|
|
332
|
+
const record = buildPoisonRecord(body, JSON.stringify(deadLetter));
|
|
333
|
+
return keepRoute({ data: Buffer.from(record, "utf8"), properties: poisonProperties(record) }, message);
|
|
334
|
+
}
|
|
335
|
+
const original = envelope.meta?.queue || this.options.queue || stripDlqSuffix(topicName(producer.getTopic()));
|
|
336
|
+
const annotated = annotate({ ...envelope, attempts: counted }, kind, original, details);
|
|
337
|
+
return keepRoute(toPulsarMessage(annotated), message);
|
|
184
338
|
}
|
|
185
339
|
notifyRoutingMismatch(propertyUrn, bodyUrn, envelope) {
|
|
186
340
|
const hook = this.options.onRoutingMismatch;
|
|
@@ -200,7 +354,41 @@ function safeProperties(message) {
|
|
|
200
354
|
return {};
|
|
201
355
|
}
|
|
202
356
|
}
|
|
357
|
+
function keepRoute(outgoing, incoming) {
|
|
358
|
+
const route = safeProperties(incoming)["bq-job"];
|
|
359
|
+
if (typeof route === "string" && route.trim() !== "") {
|
|
360
|
+
outgoing.properties = { ...outgoing.properties, "bq-job": route };
|
|
361
|
+
}
|
|
362
|
+
return outgoing;
|
|
363
|
+
}
|
|
364
|
+
function errorText(error) {
|
|
365
|
+
return error instanceof Error ? error.message : String(error);
|
|
366
|
+
}
|
|
367
|
+
function stripDlqSuffix(topic) {
|
|
368
|
+
return topic.endsWith(".dlq") && topic.length > 4 ? topic.slice(0, -4) : topic;
|
|
369
|
+
}
|
|
370
|
+
function poisonProperties(recordJson) {
|
|
371
|
+
const properties = {};
|
|
372
|
+
let record;
|
|
373
|
+
try {
|
|
374
|
+
record = JSON.parse(recordJson);
|
|
375
|
+
} catch {
|
|
376
|
+
return properties;
|
|
377
|
+
}
|
|
378
|
+
if (record === null || typeof record !== "object") return properties;
|
|
379
|
+
const top = record;
|
|
380
|
+
const str = (value) => typeof value === "string" && value !== "" ? value : void 0;
|
|
381
|
+
const job = str(top["job"]) ?? str(top["urn"]);
|
|
382
|
+
if (job) properties["bq-job"] = job;
|
|
383
|
+
const trace = str(top["trace_id"]);
|
|
384
|
+
if (trace) properties["bq-trace-id"] = trace;
|
|
385
|
+
const meta = top["meta"] !== null && typeof top["meta"] === "object" ? top["meta"] : {};
|
|
386
|
+
const id = str(meta["id"]);
|
|
387
|
+
if (id) properties["bq-message-id"] = id;
|
|
388
|
+
return properties;
|
|
389
|
+
}
|
|
203
390
|
export {
|
|
391
|
+
DEFAULT_MAX_ATTEMPTS,
|
|
204
392
|
PulsarConsumer,
|
|
205
393
|
PulsarPublisher,
|
|
206
394
|
headersOf,
|