@voltro/plugin-queue 0.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9987 -0
- package/LICENSE +61 -0
- package/README.md +26 -0
- package/SECURITY.md +56 -0
- package/THIRD-PARTY-NOTICES.md +378 -0
- package/dist/index.d.ts +284 -0
- package/dist/index.js +303 -0
- package/package.json +46 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import { Context } from 'effect';
|
|
2
|
+
import { DataStore } from '@voltro/database';
|
|
3
|
+
import { Schema } from 'effect';
|
|
4
|
+
import { VoltroPlugin } from '@voltro/protocol';
|
|
5
|
+
|
|
6
|
+
/** The provider the running `queuePlugin` connected — for the outbox handler
|
|
7
|
+
* and the cdc sink, which are constructed in app files where the plugin
|
|
8
|
+
* instance is not in scope. */
|
|
9
|
+
export declare const activeQueueProvider: () => QueueProvider;
|
|
10
|
+
|
|
11
|
+
declare interface CdcSinkRecord {
|
|
12
|
+
readonly table: string;
|
|
13
|
+
readonly op: 'insert' | 'update' | 'delete';
|
|
14
|
+
readonly key: string;
|
|
15
|
+
readonly data: Record<string, unknown> | null;
|
|
16
|
+
readonly deliveryKey: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Declare one queue consumer. One per `*.consumer.ts` file, exported —
|
|
21
|
+
* loading the module registers it; the queue plugin starts it.
|
|
22
|
+
*/
|
|
23
|
+
export declare const defineQueueConsumer: <A>(definition: QueueConsumerDefinition<A>) => QueueConsumerDefinition<A>;
|
|
24
|
+
|
|
25
|
+
export declare const kafkaProvider: (config: KafkaProviderConfig) => QueueProvider;
|
|
26
|
+
|
|
27
|
+
export declare interface KafkaProviderConfig {
|
|
28
|
+
readonly brokers: ReadonlyArray<string>;
|
|
29
|
+
readonly clientId?: string;
|
|
30
|
+
readonly ssl?: boolean;
|
|
31
|
+
readonly sasl?: {
|
|
32
|
+
readonly mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
|
|
33
|
+
readonly username: string;
|
|
34
|
+
readonly password: string;
|
|
35
|
+
};
|
|
36
|
+
/** Partitions consumed in parallel per consumer (default 3). Within one
|
|
37
|
+
* partition processing is ALWAYS serial — see the header. */
|
|
38
|
+
readonly partitionsConsumedConcurrently?: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* A `CdcSink` producing each record to `topic`: message key = the source
|
|
43
|
+
* row's id (so one row's changes stay in one partition, ordered), value =
|
|
44
|
+
* the record JSON, `x-voltro-delivery-key` header = cdc-out's at-least-once
|
|
45
|
+
* dedupe handle (a consumer that upserts on it collapses retries).
|
|
46
|
+
*/
|
|
47
|
+
export declare const kafkaSink: (options: KafkaSinkOptions) => {
|
|
48
|
+
readonly name: string;
|
|
49
|
+
readonly deliver: (batch: ReadonlyArray<CdcSinkRecord>) => Promise<void>;
|
|
50
|
+
readonly outboundHost?: string;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
declare interface KafkaSinkOptions {
|
|
54
|
+
readonly topic: string;
|
|
55
|
+
/** Explicit provider — defaults to the running queuePlugin's. */
|
|
56
|
+
readonly provider?: QueueProvider;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export declare interface QueueConsumeHandle {
|
|
60
|
+
/** Settles after the consumer left the group and in-flight work stopped. */
|
|
61
|
+
readonly stop: () => Promise<void>;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export declare interface QueueConsumeOptions {
|
|
65
|
+
readonly topic: string;
|
|
66
|
+
readonly groupId: string;
|
|
67
|
+
readonly fromBeginning?: boolean;
|
|
68
|
+
/**
|
|
69
|
+
* Called once per message, SERIALLY PER PARTITION — parallelism exists only
|
|
70
|
+
* ACROSS partitions. The provider commits the message's offset after this
|
|
71
|
+
* resolves; a rejection stops the batch without committing, so the message
|
|
72
|
+
* redelivers (at-least-once). Ordering + commit semantics both depend on
|
|
73
|
+
* this seriality; concurrency inside a partition would destroy both.
|
|
74
|
+
*/
|
|
75
|
+
readonly onMessage: (message: QueueIncomingMessage, ctx: {
|
|
76
|
+
/** Keep the group session alive during a long handler. */
|
|
77
|
+
readonly heartbeat: () => Promise<void>;
|
|
78
|
+
/** `true` when the partition was revoked mid-batch (a rebalance): the
|
|
79
|
+
* remaining messages belong to their NEW owner — stop without treating
|
|
80
|
+
* it as a handler failure. */
|
|
81
|
+
readonly isStale: () => boolean;
|
|
82
|
+
}) => Promise<void>;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export declare interface QueueConsumerContext {
|
|
86
|
+
readonly topic: string;
|
|
87
|
+
readonly partition: number;
|
|
88
|
+
readonly offset: string;
|
|
89
|
+
readonly key: string | null;
|
|
90
|
+
readonly headers: Readonly<Record<string, string>>;
|
|
91
|
+
/** Delivery attempt within THIS process, 1-based. Redelivery after a crash
|
|
92
|
+
* starts at 1 again — the at-least-once contract. */
|
|
93
|
+
readonly attempt: number;
|
|
94
|
+
/** W3C `traceparent` from the message headers, when the producer sent one —
|
|
95
|
+
* pass it on for cross-system trace continuity. */
|
|
96
|
+
readonly traceparent?: string;
|
|
97
|
+
/** The app's data store — what a consumer writes its results through.
|
|
98
|
+
* Reads and writes are NOT wrapped in a transaction by the framework
|
|
99
|
+
* (the documented HTTP-handler boundary); a handler needing atomicity
|
|
100
|
+
* opens `store.transactional` itself, and MUST be idempotent either way
|
|
101
|
+
* (at-least-once delivery). Throws if the plugin has no bound store yet
|
|
102
|
+
* (a message arriving before boot finished — the retry absorbs it). */
|
|
103
|
+
readonly store: DataStore;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export declare interface QueueConsumerDefinition<A = unknown> {
|
|
107
|
+
/** The topic to consume. */
|
|
108
|
+
readonly topic: string;
|
|
109
|
+
/** Consumer group id. Default: `<appName>.<topic>` at runtime — every
|
|
110
|
+
* replica of one app shares the group (Kafka coordinates the partitions;
|
|
111
|
+
* no advisory lock needed, unlike schedules). */
|
|
112
|
+
readonly groupId?: string;
|
|
113
|
+
/** Decode of the message VALUE (JSON-parsed first). A message that fails
|
|
114
|
+
* to parse or decode goes STRAIGHT to the dead-letter topic with the
|
|
115
|
+
* reason — never into the retry loop (a decode failure is deterministic;
|
|
116
|
+
* retrying it is an infinite loop with extra steps). */
|
|
117
|
+
readonly schema: Schema.Schema<A, any, never>;
|
|
118
|
+
/**
|
|
119
|
+
* The handler. Runs SERIALLY PER PARTITION; its offset commits only after
|
|
120
|
+
* it resolves. It MUST be idempotent: delivery is at-least-once, and a
|
|
121
|
+
* crash between "handled" and "committed" redelivers.
|
|
122
|
+
*
|
|
123
|
+
* NOT wrapped in a store transaction by the framework (same documented
|
|
124
|
+
* boundary as HTTP route handlers): a handler doing multiple writes that
|
|
125
|
+
* must be atomic opens its own `store.transactional`.
|
|
126
|
+
*/
|
|
127
|
+
readonly handler: (message: A, ctx: QueueConsumerContext) => Promise<void>;
|
|
128
|
+
/** In-process retry attempts before dead-lettering (default 3). Backoff
|
|
129
|
+
* BLOCKS the partition on purpose — ordering is the contract. */
|
|
130
|
+
readonly maxAttempts?: number;
|
|
131
|
+
/** Dead-letter topic (default `<topic>.dlq`). */
|
|
132
|
+
readonly dlqTopic?: string;
|
|
133
|
+
/** Read the topic from the beginning on first group join (default false). */
|
|
134
|
+
readonly fromBeginning?: boolean;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export declare interface QueueConsumersHandle {
|
|
138
|
+
readonly stop: () => Promise<void>;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** One message as the transport hands it to the framework. */
|
|
142
|
+
export declare interface QueueIncomingMessage {
|
|
143
|
+
readonly topic: string;
|
|
144
|
+
readonly partition: number;
|
|
145
|
+
readonly offset: string;
|
|
146
|
+
readonly key: string | null;
|
|
147
|
+
/** Raw value bytes as a UTF-8 string (Kafka values are bytes; the framework
|
|
148
|
+
* layer owns JSON parsing + Schema decoding). */
|
|
149
|
+
readonly value: string | null;
|
|
150
|
+
readonly headers: Readonly<Record<string, string>>;
|
|
151
|
+
readonly timestamp: string;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Per-process counters, keyed by topic — the inspect endpoint reads them. */
|
|
155
|
+
export declare const queueMetrics: () => QueueMetricsSlot;
|
|
156
|
+
|
|
157
|
+
declare interface QueueMetricsSlot {
|
|
158
|
+
consumed: Record<string, number>;
|
|
159
|
+
dlq: Record<string, number>;
|
|
160
|
+
retried: Record<string, number>;
|
|
161
|
+
lastError: Record<string, string>;
|
|
162
|
+
produced: Record<string, number>;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The `*.outbox.ts` handler that makes producing TRANSACTIONAL: a mutation
|
|
167
|
+
* calls `ctx.outbox.enqueue('queue.produce', { topic, messages })` inside its
|
|
168
|
+
* transaction — the intent commits or rolls back WITH the domain write, and
|
|
169
|
+
* the outbox runner delivers it (batched `produce`, at-least-once, retried,
|
|
170
|
+
* dead-lettered) after commit. One durability path: the existing outbox, not
|
|
171
|
+
* a second one.
|
|
172
|
+
*
|
|
173
|
+
* // src/queue.outbox.ts
|
|
174
|
+
* import { queueOutboxHandler } from '@voltro/plugin-queue'
|
|
175
|
+
* export default queueOutboxHandler()
|
|
176
|
+
*/
|
|
177
|
+
export declare const queueOutboxHandler: () => {
|
|
178
|
+
readonly effect: "queue.produce";
|
|
179
|
+
readonly handler: (ctx: {
|
|
180
|
+
readonly payload: Record<string, unknown>;
|
|
181
|
+
}) => Promise<void>;
|
|
182
|
+
readonly maxAttempts: number;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
/** One message to produce. */
|
|
186
|
+
export declare interface QueueOutgoingMessage {
|
|
187
|
+
readonly key?: string;
|
|
188
|
+
readonly value: string;
|
|
189
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Queue interop against an existing Kafka. Consumers come from
|
|
194
|
+
* `*.consumer.ts` files (`defineQueueConsumer`) — discovered by both boot
|
|
195
|
+
* paths, started at activation, stopped at deactivation (the runner hands
|
|
196
|
+
* its teardown over at construction; a consumer that outlives
|
|
197
|
+
* `store.close()` is the rolling-deploy leak).
|
|
198
|
+
*
|
|
199
|
+
* Replica coordination is Kafka's own: every replica joins the same consumer
|
|
200
|
+
* GROUP and the broker assigns partitions — no advisory lock, unlike
|
|
201
|
+
* schedules (which coordinate through the claim table because there is no
|
|
202
|
+
* broker to do it).
|
|
203
|
+
*/
|
|
204
|
+
export declare const queuePlugin: (config: QueuePluginConfig) => VoltroPlugin;
|
|
205
|
+
|
|
206
|
+
export declare interface QueuePluginConfig extends KafkaProviderConfig {
|
|
207
|
+
/** Instance alias (default `queue`). */
|
|
208
|
+
readonly name?: string;
|
|
209
|
+
/** Consumer-group prefix — defaults to the app name at activation. */
|
|
210
|
+
readonly groupPrefix?: string;
|
|
211
|
+
/** Injectable for tests. */
|
|
212
|
+
readonly retryBaseMs?: number;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* A queue transport. `kafkaProvider` is the shipped implementation; the
|
|
217
|
+
* methods are exactly what the framework layer (consumer runner, outbox
|
|
218
|
+
* bridge, cdc sink) needs — connect, produce (batched), consume, and a
|
|
219
|
+
* DLQ publish that is just `produce` to the dead-letter topic.
|
|
220
|
+
*/
|
|
221
|
+
export declare interface QueueProvider {
|
|
222
|
+
readonly name: string;
|
|
223
|
+
/** Hosts the provider talks to — surfaced as network permissions. */
|
|
224
|
+
readonly hosts: ReadonlyArray<string>;
|
|
225
|
+
readonly connect: () => Promise<void>;
|
|
226
|
+
readonly disconnect: () => Promise<void>;
|
|
227
|
+
/** Produce a BATCH to one topic (one transport round-trip — the outbox
|
|
228
|
+
* drain and the cdc sink both deliver batches). */
|
|
229
|
+
readonly produce: (topic: string, messages: ReadonlyArray<QueueOutgoingMessage>) => Promise<void>;
|
|
230
|
+
readonly consume: (options: QueueConsumeOptions) => Promise<QueueConsumeHandle>;
|
|
231
|
+
/** Create topics that don't exist yet (no-op for existing ones). NOT called
|
|
232
|
+
* implicitly — an adopter's Kafka is foreign infrastructure, and whether a
|
|
233
|
+
* client may create topics is their policy. Tests and bootstrap scripts
|
|
234
|
+
* call it explicitly. */
|
|
235
|
+
readonly ensureTopics: (topics: ReadonlyArray<string>) => Promise<void>;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
export declare interface QueueRunnerLogger {
|
|
239
|
+
info: (message: string, fields?: Record<string, unknown>) => void;
|
|
240
|
+
warn: (message: string, fields?: Record<string, unknown>) => void;
|
|
241
|
+
error: (message: string, fields?: Record<string, unknown>) => void;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** `yield* QueueService` in a handler to produce directly. */
|
|
245
|
+
export declare class QueueService extends QueueService_base {
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
declare const QueueService_base: Context.TagClass<QueueService, "@voltro/plugin-queue/QueueService", QueueServiceApi>;
|
|
249
|
+
|
|
250
|
+
export declare interface QueueServiceApi {
|
|
251
|
+
/** Produce a batch to a topic (one transport round-trip). NOT transactional
|
|
252
|
+
* with the surrounding mutation — for atomic produce-with-write, enqueue
|
|
253
|
+
* through the outbox (`queueOutboxHandler`). */
|
|
254
|
+
readonly produce: (topic: string, messages: ReadonlyArray<QueueOutgoingMessage>) => Promise<void>;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** All registered consumers — read by the plugin's activation. */
|
|
258
|
+
export declare const registeredQueueConsumers: () => ReadonlyArray<QueueConsumerDefinition>;
|
|
259
|
+
|
|
260
|
+
/** Test seam. */
|
|
261
|
+
export declare const resetQueueConsumersForTest: () => void;
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Start one provider consumer per registered definition. Returns the handle
|
|
265
|
+
* whose `stop()` leaves every group — the plugin takes this at CONSTRUCTION
|
|
266
|
+
* and registers it for shutdown (the `startOutboxRunner` onShutdown pattern:
|
|
267
|
+
* a runner that outlives `store.close()` is the rolling-deploy leak).
|
|
268
|
+
*/
|
|
269
|
+
export declare const startQueueConsumers: (args: StartQueueConsumersArgs) => Promise<QueueConsumersHandle>;
|
|
270
|
+
|
|
271
|
+
export declare interface StartQueueConsumersArgs {
|
|
272
|
+
readonly provider: QueueProvider;
|
|
273
|
+
readonly consumers: ReadonlyArray<QueueConsumerDefinition<any>>;
|
|
274
|
+
readonly appName: string;
|
|
275
|
+
readonly log: QueueRunnerLogger;
|
|
276
|
+
/** The bound store, read LAZILY per message — binding may complete after
|
|
277
|
+
* the consumers started. */
|
|
278
|
+
readonly store: () => DataStore | null;
|
|
279
|
+
/** Base backoff (ms) between handler retries; attempt N waits base·2^(N-1).
|
|
280
|
+
* Injectable so tests don't wait wall-clock seconds. */
|
|
281
|
+
readonly retryBaseMs?: number;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
export { }
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
import { Context as e, Effect as t, Layer as n, Schema as r } from "effect";
|
|
2
|
+
import { definePlugin as i } from "@voltro/protocol";
|
|
3
|
+
import { Kafka as a, logLevel as o } from "kafkajs";
|
|
4
|
+
//#region src/consumer.ts
|
|
5
|
+
var s = Symbol.for("@voltro/plugin-queue:consumers"), c = () => {
|
|
6
|
+
let e = globalThis;
|
|
7
|
+
return e[s] ??= /* @__PURE__ */ new Map();
|
|
8
|
+
}, l = (e) => {
|
|
9
|
+
if (e.topic.trim() === "") throw Error("defineQueueConsumer: `topic` must be a non-empty name");
|
|
10
|
+
if (e.maxAttempts !== void 0 && e.maxAttempts < 1) throw Error(`defineQueueConsumer("${e.topic}"): maxAttempts must be >= 1`);
|
|
11
|
+
let t = `${e.topic}|${e.groupId ?? ""}`, n = c().get(t);
|
|
12
|
+
if (n !== void 0 && n !== e) throw Error(`defineQueueConsumer: topic '${e.topic}'${e.groupId ? ` group '${e.groupId}'` : ""} is declared twice — two consumers in one group on one topic would split its partitions unpredictably. Use a distinct groupId for an independent consumer.`);
|
|
13
|
+
return c().set(t, e), e;
|
|
14
|
+
}, u = () => [...c().values()], d = () => {
|
|
15
|
+
c().clear();
|
|
16
|
+
}, f = (e) => {
|
|
17
|
+
let t = {};
|
|
18
|
+
for (let [n, r] of Object.entries(e ?? {})) r != null && (t[n] = Buffer.isBuffer(r) ? r.toString("utf8") : String(r));
|
|
19
|
+
return t;
|
|
20
|
+
}, p = (e) => {
|
|
21
|
+
let t = new a({
|
|
22
|
+
clientId: e.clientId ?? "voltro",
|
|
23
|
+
brokers: [...e.brokers],
|
|
24
|
+
logLevel: o.ERROR,
|
|
25
|
+
...e.ssl === void 0 ? {} : { ssl: e.ssl },
|
|
26
|
+
...e.sasl === void 0 ? {} : { sasl: e.sasl }
|
|
27
|
+
}), n = null, r = /* @__PURE__ */ new Set();
|
|
28
|
+
return {
|
|
29
|
+
name: "kafka",
|
|
30
|
+
hosts: e.brokers.map((e) => e.split("/").pop() ?? e),
|
|
31
|
+
connect: async () => {
|
|
32
|
+
n === null && (n = t.producer({ allowAutoTopicCreation: !0 }), await n.connect());
|
|
33
|
+
},
|
|
34
|
+
disconnect: async () => {
|
|
35
|
+
let e = n;
|
|
36
|
+
n = null, await Promise.all([...r].map((e) => e.disconnect().catch(() => {}))), r.clear(), e !== null && await e.disconnect().catch(() => {});
|
|
37
|
+
},
|
|
38
|
+
produce: async (e, t) => {
|
|
39
|
+
if (n === null) throw Error("kafkaProvider: produce before connect()");
|
|
40
|
+
await n.send({
|
|
41
|
+
topic: e,
|
|
42
|
+
messages: t.map((e) => ({
|
|
43
|
+
...e.key === void 0 ? {} : { key: e.key },
|
|
44
|
+
value: e.value,
|
|
45
|
+
...e.headers === void 0 ? {} : { headers: { ...e.headers } }
|
|
46
|
+
}))
|
|
47
|
+
});
|
|
48
|
+
},
|
|
49
|
+
ensureTopics: async (e) => {
|
|
50
|
+
let n = t.admin();
|
|
51
|
+
await n.connect();
|
|
52
|
+
try {
|
|
53
|
+
await n.createTopics({
|
|
54
|
+
topics: e.map((e) => ({
|
|
55
|
+
topic: e,
|
|
56
|
+
numPartitions: 1
|
|
57
|
+
})),
|
|
58
|
+
waitForLeaders: !0
|
|
59
|
+
});
|
|
60
|
+
} finally {
|
|
61
|
+
await n.disconnect();
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
consume: async (n) => {
|
|
65
|
+
let i = t.consumer({ groupId: n.groupId });
|
|
66
|
+
return r.add(i), await i.connect(), await i.subscribe({
|
|
67
|
+
topic: n.topic,
|
|
68
|
+
fromBeginning: n.fromBeginning ?? !1
|
|
69
|
+
}), await i.run({
|
|
70
|
+
eachBatchAutoResolve: !1,
|
|
71
|
+
partitionsConsumedConcurrently: e.partitionsConsumedConcurrently ?? 3,
|
|
72
|
+
eachBatch: async ({ batch: e, resolveOffset: t, commitOffsetsIfNecessary: r, heartbeat: i, isRunning: a, isStale: o }) => {
|
|
73
|
+
for (let s of e.messages) {
|
|
74
|
+
if (!a() || o()) return;
|
|
75
|
+
let c = {
|
|
76
|
+
topic: e.topic,
|
|
77
|
+
partition: e.partition,
|
|
78
|
+
offset: s.offset,
|
|
79
|
+
key: s.key === null ? null : s.key.toString("utf8"),
|
|
80
|
+
value: s.value === null ? null : s.value.toString("utf8"),
|
|
81
|
+
headers: f(s.headers),
|
|
82
|
+
timestamp: s.timestamp
|
|
83
|
+
};
|
|
84
|
+
await n.onMessage(c, {
|
|
85
|
+
heartbeat: i,
|
|
86
|
+
isStale: o
|
|
87
|
+
}), t(s.offset), await r();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}), { stop: async () => {
|
|
91
|
+
r.delete(i), await i.disconnect().catch(() => {});
|
|
92
|
+
} };
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
}, m = Symbol.for("@voltro/plugin-queue:metrics"), h = () => {
|
|
96
|
+
let e = globalThis;
|
|
97
|
+
return e[m] ??= {
|
|
98
|
+
consumed: {},
|
|
99
|
+
dlq: {},
|
|
100
|
+
retried: {},
|
|
101
|
+
lastError: {},
|
|
102
|
+
produced: {}
|
|
103
|
+
};
|
|
104
|
+
}, g = (e, t) => {
|
|
105
|
+
e[t] = (e[t] ?? 0) + 1;
|
|
106
|
+
}, _ = (e) => new Promise((t) => setTimeout(t, e)), v = async (e) => {
|
|
107
|
+
let { provider: t, log: n } = e, i = h(), a = { current: !1 }, o = /* @__PURE__ */ new Set(), s = async (a) => {
|
|
108
|
+
let o = a.groupId ?? `${e.appName}.${a.topic}`, s = a.dlqTopic ?? `${a.topic}.dlq`, c = a.maxAttempts ?? 3, l = r.decodeUnknownEither(a.schema), u = async (e, r, o) => {
|
|
109
|
+
g(i.dlq, a.topic), i.lastError[a.topic] = `${e}: ${r}`.slice(0, 500), n.warn("queue: dead-lettered", {
|
|
110
|
+
topic: a.topic,
|
|
111
|
+
dlqTopic: s,
|
|
112
|
+
reason: e,
|
|
113
|
+
partition: o.partition,
|
|
114
|
+
offset: o.offset
|
|
115
|
+
}), await t.produce(s, [{
|
|
116
|
+
...o.key === null ? {} : { key: o.key },
|
|
117
|
+
value: o.value ?? "",
|
|
118
|
+
headers: {
|
|
119
|
+
...o.headers,
|
|
120
|
+
"x-voltro-dlq-reason": e,
|
|
121
|
+
"x-voltro-dlq-detail": r.slice(0, 500),
|
|
122
|
+
"x-voltro-dlq-source-topic": o.topic,
|
|
123
|
+
"x-voltro-dlq-source-partition": String(o.partition),
|
|
124
|
+
"x-voltro-dlq-source-offset": o.offset
|
|
125
|
+
}
|
|
126
|
+
}]);
|
|
127
|
+
}, d = await t.consume({
|
|
128
|
+
topic: a.topic,
|
|
129
|
+
groupId: o,
|
|
130
|
+
...a.fromBeginning === void 0 ? {} : { fromBeginning: a.fromBeginning },
|
|
131
|
+
onMessage: async (t, r) => {
|
|
132
|
+
let o;
|
|
133
|
+
try {
|
|
134
|
+
o = t.value === null ? null : JSON.parse(t.value);
|
|
135
|
+
} catch (e) {
|
|
136
|
+
await u("decode-failed", `invalid JSON: ${e instanceof Error ? e.message : String(e)}`, t);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
let s = l(o);
|
|
140
|
+
if (s._tag === "Left") {
|
|
141
|
+
await u("decode-failed", String(s.left), t);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
for (let o = 1; o <= c; o++) {
|
|
145
|
+
if (r.isStale()) return;
|
|
146
|
+
try {
|
|
147
|
+
await a.handler(s.right, {
|
|
148
|
+
topic: t.topic,
|
|
149
|
+
partition: t.partition,
|
|
150
|
+
offset: t.offset,
|
|
151
|
+
key: t.key,
|
|
152
|
+
headers: t.headers,
|
|
153
|
+
attempt: o,
|
|
154
|
+
...t.headers.traceparent === void 0 ? {} : { traceparent: t.headers.traceparent },
|
|
155
|
+
get store() {
|
|
156
|
+
let t = e.store();
|
|
157
|
+
if (t === null) throw Error("queue: store not bound yet — the retry loop absorbs a message arriving before boot finished");
|
|
158
|
+
return t;
|
|
159
|
+
}
|
|
160
|
+
}), g(i.consumed, a.topic);
|
|
161
|
+
return;
|
|
162
|
+
} catch (s) {
|
|
163
|
+
let l = s instanceof Error ? s.message : String(s);
|
|
164
|
+
if (o >= c) {
|
|
165
|
+
await u("handler-failed", l, t);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
g(i.retried, a.topic), n.warn("queue: handler failed, retrying", {
|
|
169
|
+
topic: a.topic,
|
|
170
|
+
attempt: o,
|
|
171
|
+
maxAttempts: c,
|
|
172
|
+
detail: l
|
|
173
|
+
}), await r.heartbeat().catch(() => {}), await _((e.retryBaseMs ?? 1e3) * 2 ** (o - 1));
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
return n.info("queue: consumer started", {
|
|
179
|
+
topic: a.topic,
|
|
180
|
+
groupId: o,
|
|
181
|
+
dlqTopic: s
|
|
182
|
+
}), d;
|
|
183
|
+
}, c = e.consumers.map((e) => {
|
|
184
|
+
let t = { handle: null }, r = () => {
|
|
185
|
+
s(e).then((e) => {
|
|
186
|
+
if (a.current) {
|
|
187
|
+
e.stop();
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
t.handle = e;
|
|
191
|
+
}).catch((t) => {
|
|
192
|
+
if (a.current) return;
|
|
193
|
+
n.warn("queue: consumer start failed — retrying in 5s", {
|
|
194
|
+
topic: e.topic,
|
|
195
|
+
detail: t instanceof Error ? t.message : String(t)
|
|
196
|
+
});
|
|
197
|
+
let i = setTimeout(() => {
|
|
198
|
+
o.delete(i), r();
|
|
199
|
+
}, 5e3);
|
|
200
|
+
o.add(i);
|
|
201
|
+
});
|
|
202
|
+
};
|
|
203
|
+
return r(), t;
|
|
204
|
+
});
|
|
205
|
+
return { stop: async () => {
|
|
206
|
+
a.current = !0;
|
|
207
|
+
for (let e of o) clearTimeout(e);
|
|
208
|
+
o.clear(), await Promise.all(c.map((e) => e.handle?.stop() ?? Promise.resolve()));
|
|
209
|
+
} };
|
|
210
|
+
}, y = (e) => {
|
|
211
|
+
let t = () => e.provider ?? C();
|
|
212
|
+
return {
|
|
213
|
+
name: `kafka(${e.topic})`,
|
|
214
|
+
...e.provider !== void 0 && e.provider.hosts[0] !== void 0 ? { outboundHost: e.provider.hosts[0] } : {},
|
|
215
|
+
deliver: async (n) => {
|
|
216
|
+
await t().produce(e.topic, n.map((e) => ({
|
|
217
|
+
key: e.key,
|
|
218
|
+
value: JSON.stringify({
|
|
219
|
+
table: e.table,
|
|
220
|
+
op: e.op,
|
|
221
|
+
key: e.key,
|
|
222
|
+
data: e.data
|
|
223
|
+
}),
|
|
224
|
+
headers: { "x-voltro-delivery-key": e.deliveryKey }
|
|
225
|
+
})));
|
|
226
|
+
}
|
|
227
|
+
};
|
|
228
|
+
}, b = class extends e.Tag("@voltro/plugin-queue/QueueService")() {}, x = Symbol.for("@voltro/plugin-queue:active-provider"), S = () => {
|
|
229
|
+
let e = globalThis;
|
|
230
|
+
return e[x] ??= { current: null };
|
|
231
|
+
}, C = () => {
|
|
232
|
+
let e = S().current;
|
|
233
|
+
if (e === null) throw Error("[voltro] no active queue provider — is queuePlugin({...}) in this app's plugins? The outbox handler and kafkaSink need the plugin's connected provider.");
|
|
234
|
+
return e;
|
|
235
|
+
}, w = () => ({
|
|
236
|
+
effect: "queue.produce",
|
|
237
|
+
maxAttempts: 8,
|
|
238
|
+
handler: async ({ payload: e }) => {
|
|
239
|
+
let t = e.topic, n = e.messages;
|
|
240
|
+
if (typeof t != "string" || !Array.isArray(n)) throw Error("queue.produce payload must be { topic: string, messages: QueueOutgoingMessage[] }");
|
|
241
|
+
await C().produce(t, n);
|
|
242
|
+
let r = h();
|
|
243
|
+
r.produced[t] = (r.produced[t] ?? 0) + n.length;
|
|
244
|
+
}
|
|
245
|
+
}), T = (e) => {
|
|
246
|
+
let r = e.name ?? "queue", a = p(e), o = ["inspect:read", ...a.hosts.map((e) => `network:outbound:${e}`)], s = null, c = { current: null };
|
|
247
|
+
return i({
|
|
248
|
+
name: r,
|
|
249
|
+
baseName: "queue",
|
|
250
|
+
description: "Queue interop — consume and produce against an existing Kafka: Schema-decoded consumers (at-least-once, serial per partition, retry + dead-letter), batched producing via a handler service or transactionally through the outbox, and a kafkaSink for plugin-cdc-out.",
|
|
251
|
+
permissions: o,
|
|
252
|
+
services: n.succeed(b, { produce: (e, t) => a.produce(e, t) }),
|
|
253
|
+
bindDataStore: (e) => {
|
|
254
|
+
c.current = e;
|
|
255
|
+
},
|
|
256
|
+
onActivate: (n) => t.tryPromise(async () => {
|
|
257
|
+
await a.connect(), S().current = a;
|
|
258
|
+
let t = u();
|
|
259
|
+
s = await v({
|
|
260
|
+
provider: a,
|
|
261
|
+
consumers: t,
|
|
262
|
+
appName: e.groupPrefix ?? n.app.name,
|
|
263
|
+
log: n.logger,
|
|
264
|
+
store: () => c.current,
|
|
265
|
+
...e.retryBaseMs === void 0 ? {} : { retryBaseMs: e.retryBaseMs }
|
|
266
|
+
}), n.logger.info("queue: connected", {
|
|
267
|
+
brokers: a.hosts,
|
|
268
|
+
consumers: t.length
|
|
269
|
+
});
|
|
270
|
+
}),
|
|
271
|
+
onDeactivate: () => t.tryPromise(async () => {
|
|
272
|
+
await s?.stop(), s = null, S().current === a && (S().current = null), await a.disconnect();
|
|
273
|
+
}),
|
|
274
|
+
inspectEndpoints: [{
|
|
275
|
+
method: "GET",
|
|
276
|
+
path: "/consumers",
|
|
277
|
+
description: "Registered consumers + per-topic consumed/retried/dead-lettered/produced counters (this replica).",
|
|
278
|
+
handler: () => t.sync(() => {
|
|
279
|
+
let e = h();
|
|
280
|
+
return {
|
|
281
|
+
kind: "json",
|
|
282
|
+
data: {
|
|
283
|
+
connected: S().current !== null,
|
|
284
|
+
brokers: a.hosts,
|
|
285
|
+
consumers: u().map((t) => ({
|
|
286
|
+
topic: t.topic,
|
|
287
|
+
groupId: t.groupId ?? null,
|
|
288
|
+
dlqTopic: t.dlqTopic ?? `${t.topic}.dlq`,
|
|
289
|
+
maxAttempts: t.maxAttempts ?? 3,
|
|
290
|
+
consumed: e.consumed[t.topic] ?? 0,
|
|
291
|
+
retried: e.retried[t.topic] ?? 0,
|
|
292
|
+
deadLettered: e.dlq[t.topic] ?? 0,
|
|
293
|
+
lastError: e.lastError[t.topic] ?? null
|
|
294
|
+
})),
|
|
295
|
+
produced: e.produced
|
|
296
|
+
}
|
|
297
|
+
};
|
|
298
|
+
})
|
|
299
|
+
}]
|
|
300
|
+
});
|
|
301
|
+
};
|
|
302
|
+
//#endregion
|
|
303
|
+
export { b as QueueService, C as activeQueueProvider, l as defineQueueConsumer, p as kafkaProvider, y as kafkaSink, h as queueMetrics, w as queueOutboxHandler, T as queuePlugin, u as registeredQueueConsumers, d as resetQueueConsumersForTest, v as startQueueConsumers };
|
package/package.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@voltro/plugin-queue",
|
|
3
|
+
"version": "0.53.0",
|
|
4
|
+
"description": "Queue interop — consume and produce against an existing Kafka: Schema-decoded consumers via defineQueueConsumer (*.consumer.ts; at-least-once, serial per partition, retry + dead-letter), batched producing via QueueService or transactionally through the outbox (queueOutboxHandler), and a kafkaSink for plugin-cdc-out.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"voltro",
|
|
7
|
+
"typescript",
|
|
8
|
+
"framework"
|
|
9
|
+
],
|
|
10
|
+
"license": "SEE LICENSE IN LICENSE",
|
|
11
|
+
"homepage": "https://voltro.dev",
|
|
12
|
+
"bugs": {
|
|
13
|
+
"email": "support@voltro.dev"
|
|
14
|
+
},
|
|
15
|
+
"author": {
|
|
16
|
+
"name": "Voltro UG",
|
|
17
|
+
"url": "https://voltro.dev"
|
|
18
|
+
},
|
|
19
|
+
"type": "module",
|
|
20
|
+
"exports": {
|
|
21
|
+
".": {
|
|
22
|
+
"types": "./dist/index.d.ts",
|
|
23
|
+
"import": "./dist/index.js",
|
|
24
|
+
"default": "./dist/index.js"
|
|
25
|
+
},
|
|
26
|
+
"./package.json": "./package.json"
|
|
27
|
+
},
|
|
28
|
+
"main": "./dist/index.js",
|
|
29
|
+
"module": "./dist/index.js",
|
|
30
|
+
"types": "./dist/index.d.ts",
|
|
31
|
+
"sideEffects": false,
|
|
32
|
+
"engines": {
|
|
33
|
+
"node": ">=24.0.0"
|
|
34
|
+
},
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"@voltro/protocol": "0.53.0",
|
|
37
|
+
"kafkajs": "^2.2.4",
|
|
38
|
+
"@voltro/database": "0.53.0"
|
|
39
|
+
},
|
|
40
|
+
"peerDependencies": {
|
|
41
|
+
"effect": "^3.22.0"
|
|
42
|
+
},
|
|
43
|
+
"publishConfig": {
|
|
44
|
+
"access": "public"
|
|
45
|
+
}
|
|
46
|
+
}
|