@voltro/plugin-queue 0.85.0 → 0.87.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ import { activeQueueProvider as e } from "./index.js";
2
+ //#region src/cdcSink.ts
3
+ var t = (t) => {
4
+ let n = () => t.provider ?? e();
5
+ return {
6
+ name: `kafka(${t.topic})`,
7
+ ...t.provider !== void 0 && t.provider.hosts[0] !== void 0 ? { outboundHost: t.provider.hosts[0] } : {},
8
+ deliver: async (e) => {
9
+ await n().produce(t.topic, e.map((e) => ({
10
+ key: e.key,
11
+ value: JSON.stringify({
12
+ table: e.table,
13
+ op: e.op,
14
+ key: e.key,
15
+ data: e.data
16
+ }),
17
+ headers: { "x-voltro-delivery-key": e.deliveryKey }
18
+ })));
19
+ }
20
+ };
21
+ };
22
+ //#endregion
23
+ export { t as kafkaSink };
@@ -0,0 +1,15 @@
1
+ //#region src/consumer.ts
2
+ var e = Symbol.for("@voltro/plugin-queue:consumers"), t = () => {
3
+ let t = globalThis;
4
+ return t[e] ??= /* @__PURE__ */ new Map();
5
+ }, n = (e) => {
6
+ if (e.topic.trim() === "") throw Error("defineQueueConsumer: `topic` must be a non-empty name");
7
+ if (e.maxAttempts !== void 0 && e.maxAttempts < 1) throw Error(`defineQueueConsumer("${e.topic}"): maxAttempts must be >= 1`);
8
+ let n = `${e.topic}|${e.groupId ?? ""}`, r = t().get(n);
9
+ if (r !== void 0 && r !== e) throw Error(`defineQueueConsumer: topic '${e.topic}'${e.groupId ? ` group '${e.groupId}'` : ""} is declared twice — two consumers in one group on one topic would split its partitions unpredictably. Use a distinct groupId for an independent consumer.`);
10
+ return t().set(n, e), e;
11
+ }, r = () => [...t().values()], i = () => {
12
+ t().clear();
13
+ };
14
+ //#endregion
15
+ export { n as defineQueueConsumer, r as registeredQueueConsumers, i as resetQueueConsumersForTest };
package/dist/index.js CHANGED
@@ -1,335 +1,70 @@
1
- import { Cause as e, Context as t, Effect as n, Exit as r, Layer as i, Metric as a, Result as o, Schema as s } from "effect";
2
- import { definePlugin as c } from "@voltro/protocol";
3
- import { Kafka as l, logLevel as u } from "kafkajs";
4
- import { externalSpanFromTraceparent as d, withServerTracer as f } from "@voltro/runtime";
5
- //#region src/consumer.ts
6
- var p = Symbol.for("@voltro/plugin-queue:consumers"), m = () => {
1
+ import { defineQueueConsumer as e, registeredQueueConsumers as t, resetQueueConsumersForTest as n } from "./consumer.js";
2
+ import { queueMetrics as r, recordQueueProduced as i } from "./queueMetrics.js";
3
+ import { kafkaProvider as a } from "./kafka.js";
4
+ import { startQueueConsumers as o } from "./runner.js";
5
+ import { kafkaSink as s } from "./cdcSink.js";
6
+ import { Context as c, Effect as l, Layer as u } from "effect";
7
+ import { definePlugin as d } from "@voltro/protocol";
8
+ //#region src/index.ts
9
+ var f = class extends c.Service()("@voltro/plugin-queue/QueueService") {}, p = Symbol.for("@voltro/plugin-queue:active-provider"), m = () => {
7
10
  let e = globalThis;
8
- return e[p] ??= /* @__PURE__ */ new Map();
9
- }, h = (e) => {
10
- if (e.topic.trim() === "") throw Error("defineQueueConsumer: `topic` must be a non-empty name");
11
- if (e.maxAttempts !== void 0 && e.maxAttempts < 1) throw Error(`defineQueueConsumer("${e.topic}"): maxAttempts must be >= 1`);
12
- let t = `${e.topic}|${e.groupId ?? ""}`, n = m().get(t);
13
- if (n !== void 0 && n !== e) throw Error(`defineQueueConsumer: topic '${e.topic}'${e.groupId ? ` group '${e.groupId}'` : ""} is declared twice — two consumers in one group on one topic would split its partitions unpredictably. Use a distinct groupId for an independent consumer.`);
14
- return m().set(t, e), e;
15
- }, g = () => [...m().values()], _ = () => {
16
- m().clear();
17
- }, v = () => Object.create(null), y = Symbol.for("@voltro/plugin-queue:metrics"), b = () => {
18
- let e = globalThis;
19
- return e[y] ??= {
20
- consumed: v(),
21
- dlq: v(),
22
- retried: v(),
23
- lastError: v(),
24
- produced: v()
25
- };
26
- }, x = (e, t, n = 1) => {
27
- e[t] = (e[t] ?? 0) + n;
28
- }, S = a.counter("voltro_queue_consumed_total", { description: "Messages a consumer finished with, by topic and outcome (ok | dead-lettered)." }), C = a.counter("voltro_queue_retries_total", { description: "In-process handler retries, by topic. A retry BLOCKS its partition." }), w = a.counter("voltro_queue_produced_total", { description: "Messages produced through the outbox bridge and QueueService, by topic." }), T = a.gauge("voltro_queue_lag_messages", { description: "Messages between the offset just picked up and the partition high watermark, by topic and partition. Sampled at pickup; holds its last value for an idle or revoked partition." }), E = t.empty(), D = /* @__PURE__ */ new Map(), O = /* @__PURE__ */ new Map(), k = /* @__PURE__ */ new Map(), A = /* @__PURE__ */ new Map(), j = (e, t, n) => {
29
- let r = e.get(t);
30
- if (r !== void 0) return r;
31
- let i = n();
32
- return e.set(t, i), i;
33
- }, M = (e) => {
34
- x(b().consumed, e), j(D, `ok\u0000${e}`, () => a.withAttributes(a.withAttributes(S, { topic: e }), { outcome: "ok" })).updateUnsafe(1, E);
35
- }, N = (e, t) => {
36
- let n = b();
37
- x(n.dlq, e), n.lastError[e] = t.slice(0, 500), j(D, `dlq\u0000${e}`, () => a.withAttributes(a.withAttributes(S, { topic: e }), { outcome: "dead-lettered" })).updateUnsafe(1, E);
38
- }, P = (e) => {
39
- x(b().retried, e), j(O, e, () => a.withAttributes(C, { topic: e })).updateUnsafe(1, E);
40
- }, F = (e, t) => {
41
- t <= 0 || (x(b().produced, e, t), j(k, e, () => a.withAttributes(w, { topic: e })).updateUnsafe(t, E));
42
- }, I = (e, t, n, r) => {
43
- let i;
44
- try {
45
- let e = BigInt(n) - BigInt(r) - 1n;
46
- i = e < 0n ? 0 : Number(e);
47
- } catch {
48
- return;
49
- }
50
- j(A, `${e}\u0000${t}`, () => a.withAttributes(a.withAttributes(T, { topic: e }), { partition: String(t) })).updateUnsafe(i, E);
51
- }, L = (e) => {
52
- let t = {};
53
- for (let [n, r] of Object.entries(e ?? {})) r != null && (t[n] = Buffer.isBuffer(r) ? r.toString("utf8") : String(r));
54
- return t;
55
- }, R = (e) => {
56
- let t = new l({
57
- clientId: e.clientId ?? "voltro",
58
- brokers: [...e.brokers],
59
- logLevel: u.ERROR,
60
- ...e.ssl === void 0 ? {} : { ssl: e.ssl },
61
- ...e.sasl === void 0 ? {} : { sasl: e.sasl }
62
- }), n = null, r = /* @__PURE__ */ new Set();
63
- return {
64
- name: "kafka",
65
- hosts: e.brokers.map((e) => e.split("/").pop() ?? e),
66
- connect: async () => {
67
- n === null && (n = t.producer({ allowAutoTopicCreation: !0 }), await n.connect());
68
- },
69
- disconnect: async () => {
70
- let e = n;
71
- n = null, await Promise.all([...r].map((e) => e.disconnect().catch(() => {}))), r.clear(), e !== null && await e.disconnect().catch(() => {});
72
- },
73
- produce: async (e, t) => {
74
- if (n === null) throw Error("kafkaProvider: produce before connect()");
75
- await n.send({
76
- topic: e,
77
- messages: t.map((e) => ({
78
- ...e.key === void 0 ? {} : { key: e.key },
79
- value: e.value,
80
- ...e.headers === void 0 ? {} : { headers: { ...e.headers } }
81
- }))
82
- });
83
- },
84
- ensureTopics: async (e) => {
85
- let n = t.admin();
86
- await n.connect();
87
- try {
88
- await n.createTopics({
89
- topics: e.map((e) => ({
90
- topic: e,
91
- numPartitions: 1
92
- })),
93
- waitForLeaders: !0
94
- });
95
- } finally {
96
- await n.disconnect();
97
- }
98
- },
99
- consume: async (n) => {
100
- let i = t.consumer({ groupId: n.groupId });
101
- return r.add(i), await i.connect(), await i.subscribe({
102
- topic: n.topic,
103
- fromBeginning: n.fromBeginning ?? !1
104
- }), await i.run({
105
- eachBatchAutoResolve: !1,
106
- partitionsConsumedConcurrently: e.partitionsConsumedConcurrently ?? 3,
107
- eachBatch: async ({ batch: e, resolveOffset: t, commitOffsetsIfNecessary: r, heartbeat: i, isRunning: a, isStale: o }) => {
108
- for (let s of e.messages) {
109
- if (!a() || o()) return;
110
- let c = {
111
- topic: e.topic,
112
- partition: e.partition,
113
- offset: s.offset,
114
- key: s.key === null ? null : s.key.toString("utf8"),
115
- value: s.value === null ? null : s.value.toString("utf8"),
116
- headers: L(s.headers),
117
- timestamp: s.timestamp
118
- };
119
- I(e.topic, e.partition, e.highWatermark, s.offset), await n.onMessage(c, {
120
- heartbeat: i,
121
- isStale: o
122
- }), t(s.offset), await r();
123
- }
124
- }
125
- }), { stop: async () => {
126
- r.delete(i), await i.disconnect().catch(() => {});
127
- } };
128
- }
129
- };
130
- }, z = "queue.consume", B = (e, t) => (r) => {
131
- let i = d(e.traceparent);
132
- return r.pipe(n.tap((e) => t === void 0 ? n.void : n.annotateCurrentSpan(t(e))), n.withSpan(z, {
133
- attributes: {
134
- "messaging.system": e.system,
135
- "messaging.operation": "process",
136
- "messaging.destination.name": e.topic,
137
- "messaging.consumer.group.name": e.groupId,
138
- "messaging.destination.partition.id": e.partition,
139
- "messaging.message.offset": e.offset
140
- },
141
- ...i ? { parent: i } : {}
142
- }));
143
- }, V = async (t, i, a) => {
144
- let o, s = await n.runPromiseExit(n.tryPromise({
145
- try: i,
146
- catch: (e) => (o = { value: e }, e)
147
- }).pipe(B(t, a), f));
148
- if (r.isSuccess(s)) return s.value;
149
- throw o === void 0 ? e.squash(s.cause) : o.value;
150
- }, H = (e) => new Promise((t) => setTimeout(t, e)), U = async (e) => {
151
- let { provider: t, log: n } = e, r = { current: !1 }, i = /* @__PURE__ */ new Set(), a = async (r) => {
152
- let i = r.groupId ?? `${e.appName}.${r.topic}`, a = r.dlqTopic ?? `${r.topic}.dlq`, c = r.maxAttempts ?? 3, l = s.decodeUnknownResult(r.schema), u = async (e, i, o) => {
153
- N(r.topic, `${e}: ${i}`), n.warn("queue: dead-lettered", {
154
- topic: r.topic,
155
- dlqTopic: a,
156
- reason: e,
157
- partition: o.partition,
158
- offset: o.offset
159
- }), await t.produce(a, [{
160
- ...o.key === null ? {} : { key: o.key },
161
- value: o.value ?? "",
162
- headers: {
163
- ...o.headers,
164
- "x-voltro-dlq-reason": e,
165
- "x-voltro-dlq-detail": i.slice(0, 500),
166
- "x-voltro-dlq-source-topic": o.topic,
167
- "x-voltro-dlq-source-partition": String(o.partition),
168
- "x-voltro-dlq-source-offset": o.offset
169
- }
170
- }]);
171
- }, d = await t.consume({
172
- topic: r.topic,
173
- groupId: i,
174
- ...r.fromBeginning === void 0 ? {} : { fromBeginning: r.fromBeginning },
175
- onMessage: async (a, s) => {
176
- await V({
177
- system: t.name,
178
- topic: r.topic,
179
- groupId: i,
180
- partition: a.partition,
181
- offset: a.offset,
182
- traceparent: a.headers.traceparent
183
- }, async () => {
184
- let t;
185
- try {
186
- t = a.value === null ? null : JSON.parse(a.value);
187
- } catch (e) {
188
- return await u("decode-failed", `invalid JSON: ${e instanceof Error ? e.message : String(e)}`, a), "dead-lettered";
189
- }
190
- let i = l(t);
191
- if (o.isFailure(i)) return await u("decode-failed", String(i.failure), a), "dead-lettered";
192
- for (let t = 1; t <= c; t++) {
193
- if (s.isStale()) return "stale";
194
- try {
195
- return await r.handler(i.success, {
196
- topic: a.topic,
197
- partition: a.partition,
198
- offset: a.offset,
199
- key: a.key,
200
- headers: a.headers,
201
- attempt: t,
202
- ...a.headers.traceparent === void 0 ? {} : { traceparent: a.headers.traceparent },
203
- get store() {
204
- let t = e.store();
205
- if (t === null) throw Error("queue: store not bound yet — the retry loop absorbs a message arriving before boot finished");
206
- return t;
207
- }
208
- }), M(r.topic), "ok";
209
- } catch (i) {
210
- let o = i instanceof Error ? i.message : String(i);
211
- if (t >= c) return await u("handler-failed", o, a), "dead-lettered";
212
- P(r.topic), n.warn("queue: handler failed, retrying", {
213
- topic: r.topic,
214
- attempt: t,
215
- maxAttempts: c,
216
- detail: o
217
- }), await s.heartbeat().catch(() => {}), await H((e.retryBaseMs ?? 1e3) * 2 ** (t - 1));
218
- }
219
- }
220
- return "stale";
221
- }, (e) => ({ "voltro.queue.outcome": e }));
222
- }
223
- });
224
- return n.info("queue: consumer started", {
225
- topic: r.topic,
226
- groupId: i,
227
- dlqTopic: a
228
- }), d;
229
- }, c = e.consumers.map((e) => {
230
- let t = { handle: null }, o = () => {
231
- a(e).then((e) => {
232
- if (r.current) {
233
- e.stop();
234
- return;
235
- }
236
- t.handle = e;
237
- }).catch((t) => {
238
- if (r.current) return;
239
- n.warn("queue: consumer start failed — retrying in 5s", {
240
- topic: e.topic,
241
- detail: t instanceof Error ? t.message : String(t)
242
- });
243
- let a = setTimeout(() => {
244
- i.delete(a), o();
245
- }, 5e3);
246
- i.add(a);
247
- });
248
- };
249
- return o(), t;
250
- });
251
- return { stop: async () => {
252
- r.current = !0;
253
- for (let e of i) clearTimeout(e);
254
- i.clear(), await Promise.all(c.map((e) => e.handle?.stop() ?? Promise.resolve()));
255
- } };
256
- }, W = (e) => {
257
- let t = () => e.provider ?? J();
258
- return {
259
- name: `kafka(${e.topic})`,
260
- ...e.provider !== void 0 && e.provider.hosts[0] !== void 0 ? { outboundHost: e.provider.hosts[0] } : {},
261
- deliver: async (n) => {
262
- await t().produce(e.topic, n.map((e) => ({
263
- key: e.key,
264
- value: JSON.stringify({
265
- table: e.table,
266
- op: e.op,
267
- key: e.key,
268
- data: e.data
269
- }),
270
- headers: { "x-voltro-delivery-key": e.deliveryKey }
271
- })));
272
- }
273
- };
274
- }, G = class extends t.Service()("@voltro/plugin-queue/QueueService") {}, K = Symbol.for("@voltro/plugin-queue:active-provider"), q = () => {
275
- let e = globalThis;
276
- return e[K] ??= { current: null };
277
- }, J = () => {
278
- let e = q().current;
11
+ return e[p] ??= { current: null };
12
+ }, h = () => {
13
+ let e = m().current;
279
14
  if (e === null) throw Error("[voltro] no active queue provider — is queuePlugin({...}) in this app's plugins? The outbox handler and kafkaSink need the plugin's connected provider.");
280
15
  return e;
281
- }, Y = () => ({
16
+ }, g = () => ({
282
17
  effect: "queue.produce",
283
18
  maxAttempts: 8,
284
19
  handler: async ({ payload: e }) => {
285
20
  let t = e.topic, n = e.messages;
286
21
  if (typeof t != "string" || !Array.isArray(n)) throw Error("queue.produce payload must be { topic: string, messages: QueueOutgoingMessage[] }");
287
- await J().produce(t, n), F(t, n.length);
22
+ await h().produce(t, n), i(t, n.length);
288
23
  }
289
- }), X = (e) => {
290
- let t = e.name ?? "queue", r = R(e), a = ["inspect:read", ...r.hosts.map((e) => `network:outbound:${e}`)], o = null, s = { current: null };
291
- return c({
292
- name: t,
24
+ }), _ = (e) => {
25
+ let n = e.name ?? "queue", s = a(e), c = ["inspect:read", ...s.hosts.map((e) => `network:outbound:${e}`)], p = null, h = { current: null };
26
+ return d({
27
+ name: n,
293
28
  baseName: "queue",
294
29
  description: "Queue interop — consume and produce against an existing Kafka: Schema-decoded consumers (at-least-once, serial per partition, retry + dead-letter), batched producing via a handler service or transactionally through the outbox, and a kafkaSink for plugin-cdc-out.",
295
- permissions: a,
296
- services: i.succeed(G, { produce: async (e, t) => {
297
- await r.produce(e, t), F(e, t.length);
30
+ permissions: c,
31
+ services: u.succeed(f, { produce: async (e, t) => {
32
+ await s.produce(e, t), i(e, t.length);
298
33
  } }),
299
34
  bindDataStore: (e) => {
300
- s.current = e;
35
+ h.current = e;
301
36
  },
302
- onActivate: (t) => n.tryPromise(async () => {
303
- await r.connect(), q().current = r;
304
- let n = g();
305
- o = await U({
306
- provider: r,
307
- consumers: n,
308
- appName: e.groupPrefix ?? t.app.name,
309
- log: t.logger,
310
- store: () => s.current,
37
+ onActivate: (n) => l.tryPromise(async () => {
38
+ await s.connect(), m().current = s;
39
+ let r = t();
40
+ p = await o({
41
+ provider: s,
42
+ consumers: r,
43
+ appName: e.groupPrefix ?? n.app.name,
44
+ log: n.logger,
45
+ store: () => h.current,
311
46
  ...e.retryBaseMs === void 0 ? {} : { retryBaseMs: e.retryBaseMs }
312
- }), t.logger.info("queue: connected", {
313
- brokers: r.hosts,
314
- consumers: n.length
47
+ }), n.logger.info("queue: connected", {
48
+ brokers: s.hosts,
49
+ consumers: r.length
315
50
  });
316
51
  }),
317
- onDeactivate: () => n.tryPromise(async () => {
318
- await o?.stop(), o = null, q().current === r && (q().current = null), await r.disconnect();
52
+ onDeactivate: () => l.tryPromise(async () => {
53
+ await p?.stop(), p = null, m().current === s && (m().current = null), await s.disconnect();
319
54
  }),
320
55
  inspectEndpoints: [{
321
56
  method: "GET",
322
57
  path: "/consumers",
323
58
  scope: "process",
324
59
  description: "Registered consumers + per-topic consumed/retried/dead-lettered/produced counters (this replica).",
325
- handler: () => n.sync(() => {
326
- let e = b();
60
+ handler: () => l.sync(() => {
61
+ let e = r();
327
62
  return {
328
63
  kind: "json",
329
64
  data: {
330
- producerSelected: q().current === r,
331
- brokers: r.hosts,
332
- consumers: g().map((t) => ({
65
+ producerSelected: m().current === s,
66
+ brokers: s.hosts,
67
+ consumers: t().map((t) => ({
333
68
  topic: t.topic,
334
69
  groupId: t.groupId ?? null,
335
70
  dlqTopic: t.dlqTopic ?? `${t.topic}.dlq`,
@@ -347,4 +82,4 @@ var p = Symbol.for("@voltro/plugin-queue:consumers"), m = () => {
347
82
  });
348
83
  };
349
84
  //#endregion
350
- export { G as QueueService, J as activeQueueProvider, h as defineQueueConsumer, R as kafkaProvider, W as kafkaSink, b as queueMetrics, Y as queueOutboxHandler, X as queuePlugin, g as registeredQueueConsumers, _ as resetQueueConsumersForTest, U as startQueueConsumers };
85
+ export { f as QueueService, h as activeQueueProvider, e as defineQueueConsumer, a as kafkaProvider, s as kafkaSink, r as queueMetrics, g as queueOutboxHandler, _ as queuePlugin, t as registeredQueueConsumers, n as resetQueueConsumersForTest, o as startQueueConsumers };
package/dist/kafka.js ADDED
@@ -0,0 +1,85 @@
1
+ import { recordQueueLag as e } from "./queueMetrics.js";
2
+ import { Kafka as t, logLevel as n } from "kafkajs";
3
+ //#region src/kafka.ts
4
+ var r = (e) => {
5
+ let t = {};
6
+ for (let [n, r] of Object.entries(e ?? {})) r != null && (t[n] = Buffer.isBuffer(r) ? r.toString("utf8") : String(r));
7
+ return t;
8
+ }, i = (i) => {
9
+ let a = new t({
10
+ clientId: i.clientId ?? "voltro",
11
+ brokers: [...i.brokers],
12
+ logLevel: n.ERROR,
13
+ ...i.ssl === void 0 ? {} : { ssl: i.ssl },
14
+ ...i.sasl === void 0 ? {} : { sasl: i.sasl }
15
+ }), o = null, s = /* @__PURE__ */ new Set();
16
+ return {
17
+ name: "kafka",
18
+ hosts: i.brokers.map((e) => e.split("/").pop() ?? e),
19
+ connect: async () => {
20
+ o === null && (o = a.producer({ allowAutoTopicCreation: !0 }), await o.connect());
21
+ },
22
+ disconnect: async () => {
23
+ let e = o;
24
+ o = null, await Promise.all([...s].map((e) => e.disconnect().catch(() => {}))), s.clear(), e !== null && await e.disconnect().catch(() => {});
25
+ },
26
+ produce: async (e, t) => {
27
+ if (o === null) throw Error("kafkaProvider: produce before connect()");
28
+ await o.send({
29
+ topic: e,
30
+ messages: t.map((e) => ({
31
+ ...e.key === void 0 ? {} : { key: e.key },
32
+ value: e.value,
33
+ ...e.headers === void 0 ? {} : { headers: { ...e.headers } }
34
+ }))
35
+ });
36
+ },
37
+ ensureTopics: async (e) => {
38
+ let t = a.admin();
39
+ await t.connect();
40
+ try {
41
+ await t.createTopics({
42
+ topics: e.map((e) => ({
43
+ topic: e,
44
+ numPartitions: 1
45
+ })),
46
+ waitForLeaders: !0
47
+ });
48
+ } finally {
49
+ await t.disconnect();
50
+ }
51
+ },
52
+ consume: async (t) => {
53
+ let n = a.consumer({ groupId: t.groupId });
54
+ return s.add(n), await n.connect(), await n.subscribe({
55
+ topic: t.topic,
56
+ fromBeginning: t.fromBeginning ?? !1
57
+ }), await n.run({
58
+ eachBatchAutoResolve: !1,
59
+ partitionsConsumedConcurrently: i.partitionsConsumedConcurrently ?? 3,
60
+ eachBatch: async ({ batch: n, resolveOffset: i, commitOffsetsIfNecessary: a, heartbeat: o, isRunning: s, isStale: c }) => {
61
+ for (let l of n.messages) {
62
+ if (!s() || c()) return;
63
+ let u = {
64
+ topic: n.topic,
65
+ partition: n.partition,
66
+ offset: l.offset,
67
+ key: l.key === null ? null : l.key.toString("utf8"),
68
+ value: l.value === null ? null : l.value.toString("utf8"),
69
+ headers: r(l.headers),
70
+ timestamp: l.timestamp
71
+ };
72
+ e(n.topic, n.partition, n.highWatermark, l.offset), await t.onMessage(u, {
73
+ heartbeat: o,
74
+ isStale: c
75
+ }), i(l.offset), await a();
76
+ }
77
+ }
78
+ }), { stop: async () => {
79
+ s.delete(n), await n.disconnect().catch(() => {});
80
+ } };
81
+ }
82
+ };
83
+ };
84
+ //#endregion
85
+ export { i as kafkaProvider };
@@ -0,0 +1,39 @@
1
+ import { Context as e, Metric as t } from "effect";
2
+ //#region src/queueMetrics.ts
3
+ var n = () => Object.create(null), r = Symbol.for("@voltro/plugin-queue:metrics"), i = () => {
4
+ let e = globalThis;
5
+ return e[r] ??= {
6
+ consumed: n(),
7
+ dlq: n(),
8
+ retried: n(),
9
+ lastError: n(),
10
+ produced: n()
11
+ };
12
+ }, a = (e, t, n = 1) => {
13
+ e[t] = (e[t] ?? 0) + n;
14
+ }, o = t.counter("voltro_queue_consumed_total", { description: "Messages a consumer finished with, by topic and outcome (ok | dead-lettered)." }), s = t.counter("voltro_queue_retries_total", { description: "In-process handler retries, by topic. A retry BLOCKS its partition." }), c = t.counter("voltro_queue_produced_total", { description: "Messages produced through the outbox bridge and QueueService, by topic." }), l = t.gauge("voltro_queue_lag_messages", { description: "Messages between the offset just picked up and the partition high watermark, by topic and partition. Sampled at pickup; holds its last value for an idle or revoked partition." }), u = e.empty(), d = /* @__PURE__ */ new Map(), f = /* @__PURE__ */ new Map(), p = /* @__PURE__ */ new Map(), m = /* @__PURE__ */ new Map(), h = (e, t, n) => {
15
+ let r = e.get(t);
16
+ if (r !== void 0) return r;
17
+ let i = n();
18
+ return e.set(t, i), i;
19
+ }, g = (e) => {
20
+ a(i().consumed, e), h(d, `ok\u0000${e}`, () => t.withAttributes(t.withAttributes(o, { topic: e }), { outcome: "ok" })).updateUnsafe(1, u);
21
+ }, _ = (e, n) => {
22
+ let r = i();
23
+ a(r.dlq, e), r.lastError[e] = n.slice(0, 500), h(d, `dlq\u0000${e}`, () => t.withAttributes(t.withAttributes(o, { topic: e }), { outcome: "dead-lettered" })).updateUnsafe(1, u);
24
+ }, v = (e) => {
25
+ a(i().retried, e), h(f, e, () => t.withAttributes(s, { topic: e })).updateUnsafe(1, u);
26
+ }, y = (e, n) => {
27
+ n <= 0 || (a(i().produced, e, n), h(p, e, () => t.withAttributes(c, { topic: e })).updateUnsafe(n, u));
28
+ }, b = (e, n, r, i) => {
29
+ let a;
30
+ try {
31
+ let e = BigInt(r) - BigInt(i) - 1n;
32
+ a = e < 0n ? 0 : Number(e);
33
+ } catch {
34
+ return;
35
+ }
36
+ h(m, `${e}\u0000${n}`, () => t.withAttributes(t.withAttributes(l, { topic: e }), { partition: String(n) })).updateUnsafe(a, u);
37
+ };
38
+ //#endregion
39
+ export { i as queueMetrics, g as recordQueueConsumed, _ as recordQueueDeadLettered, b as recordQueueLag, y as recordQueueProduced, v as recordQueueRetry };
@@ -0,0 +1,26 @@
1
+ import { Cause as e, Effect as t, Exit as n } from "effect";
2
+ import { externalSpanFromTraceparent as r, withServerTracer as i } from "@voltro/runtime";
3
+ //#region src/queueTracing.ts
4
+ var a = "queue.consume", o = (e, n) => (i) => {
5
+ let o = r(e.traceparent);
6
+ return i.pipe(t.tap((e) => n === void 0 ? t.void : t.annotateCurrentSpan(n(e))), t.withSpan(a, {
7
+ attributes: {
8
+ "messaging.system": e.system,
9
+ "messaging.operation": "process",
10
+ "messaging.destination.name": e.topic,
11
+ "messaging.consumer.group.name": e.groupId,
12
+ "messaging.destination.partition.id": e.partition,
13
+ "messaging.message.offset": e.offset
14
+ },
15
+ ...o ? { parent: o } : {}
16
+ }));
17
+ }, s = async (r, a, s) => {
18
+ let c, l = await t.runPromiseExit(t.tryPromise({
19
+ try: a,
20
+ catch: (e) => (c = { value: e }, e)
21
+ }).pipe(o(r, s), i));
22
+ if (n.isSuccess(l)) return l.value;
23
+ throw c === void 0 ? e.squash(l.cause) : c.value;
24
+ };
25
+ //#endregion
26
+ export { a as QUEUE_CONSUME_SPAN, s as runInQueueMessageSpan, o as withQueueMessageSpan };