@voltro/plugin-queue 0.53.0 → 0.54.0

package/dist/index.d.ts

@@ -154,6 +154,8 @@ export declare interface QueueIncomingMessage {
 /** Per-process counters, keyed by topic — the inspect endpoint reads them. */
 export declare const queueMetrics: () => QueueMetricsSlot;
 
+/** Per-topic counters the inspect endpoint renders. Cumulative for the life of
+ *  THIS process — the dashboard's live view, not the alerting source. */
 declare interface QueueMetricsSlot {
     consumed: Record<string, number>;
     dlq: Record<string, number>;

package/dist/index.js

@@ -1,27 +1,62 @@
-import { Context as e, Effect as t, Layer as n, Schema as r } from "effect";
-import { definePlugin as i } from "@voltro/protocol";
-import { Kafka as a, logLevel as o } from "kafkajs";
+import { Cause as e, Context as t, Effect as n, Exit as r, Layer as i, Metric as a, Schema as o } from "effect";
+import { definePlugin as s } from "@voltro/protocol";
+import { Kafka as c, logLevel as l } from "kafkajs";
+import { externalSpanFromTraceparent as u, withServerTracer as d } from "@voltro/runtime";
 //#region src/consumer.ts
-var s = Symbol.for("@voltro/plugin-queue:consumers"), c = () => {
+var f = Symbol.for("@voltro/plugin-queue:consumers"), p = () => {
 	let e = globalThis;
-	return e[s] ??= /* @__PURE__ */ new Map();
-}, l = (e) => {
+	return e[f] ??= /* @__PURE__ */ new Map();
+}, m = (e) => {
 	if (e.topic.trim() === "") throw Error("defineQueueConsumer: `topic` must be a non-empty name");
 	if (e.maxAttempts !== void 0 && e.maxAttempts < 1) throw Error(`defineQueueConsumer("${e.topic}"): maxAttempts must be >= 1`);
-	let t = `${e.topic}|${e.groupId ?? ""}`, n = c().get(t);
+	let t = `${e.topic}|${e.groupId ?? ""}`, n = p().get(t);
 	if (n !== void 0 && n !== e) throw Error(`defineQueueConsumer: topic '${e.topic}'${e.groupId ? ` group '${e.groupId}'` : ""} is declared twice — two consumers in one group on one topic would split its partitions unpredictably. Use a distinct groupId for an independent consumer.`);
-	return c().set(t, e), e;
-}, u = () => [...c().values()], d = () => {
-	c().clear();
-}, f = (e) => {
+	return p().set(t, e), e;
+}, h = () => [...p().values()], g = () => {
+	p().clear();
+}, _ = Symbol.for("@voltro/plugin-queue:metrics"), v = () => {
+	let e = globalThis;
+	return e[_] ??= {
+		consumed: {},
+		dlq: {},
+		retried: {},
+		lastError: {},
+		produced: {}
+	};
+}, y = (e, t, n = 1) => {
+	e[t] = (e[t] ?? 0) + n;
+}, b = a.counter("voltro_queue_consumed_total", { description: "Messages a consumer finished with, by topic and outcome (ok | dead-lettered)." }), x = a.counter("voltro_queue_retries_total", { description: "In-process handler retries, by topic. A retry BLOCKS its partition." }), S = a.counter("voltro_queue_produced_total", { description: "Messages produced through the outbox bridge, by topic." }), C = a.gauge("voltro_queue_lag_messages", { description: "Messages between the offset just picked up and the partition high watermark, by topic and partition. Sampled at pickup; holds its last value for an idle or revoked partition." }), w = [], T = /* @__PURE__ */ new Map(), E = /* @__PURE__ */ new Map(), D = /* @__PURE__ */ new Map(), O = /* @__PURE__ */ new Map(), k = (e, t, n) => {
+	let r = e.get(t);
+	if (r !== void 0) return r;
+	let i = n();
+	return e.set(t, i), i;
+}, A = (e) => {
+	y(v().consumed, e), k(T, `ok\u0000${e}`, () => a.tagged(a.tagged(b, "topic", e), "outcome", "ok")).unsafeUpdate(1, w);
+}, j = (e, t) => {
+	let n = v();
+	y(n.dlq, e), n.lastError[e] = t.slice(0, 500), k(T, `dlq\u0000${e}`, () => a.tagged(a.tagged(b, "topic", e), "outcome", "dead-lettered")).unsafeUpdate(1, w);
+}, M = (e) => {
+	y(v().retried, e), k(E, e, () => a.tagged(x, "topic", e)).unsafeUpdate(1, w);
+}, N = (e, t) => {
+	t <= 0 || (y(v().produced, e, t), k(D, e, () => a.tagged(S, "topic", e)).unsafeUpdate(t, w));
+}, P = (e, t, n, r) => {
+	let i;
+	try {
+		let e = BigInt(n) - BigInt(r) - 1n;
+		i = e < 0n ? 0 : Number(e);
+	} catch {
+		return;
+	}
+	k(O, `${e}\u0000${t}`, () => a.tagged(a.tagged(C, "topic", e), "partition", String(t))).unsafeUpdate(i, w);
+}, F = (e) => {
 	let t = {};
 	for (let [n, r] of Object.entries(e ?? {})) r != null && (t[n] = Buffer.isBuffer(r) ? r.toString("utf8") : String(r));
 	return t;
-}, p = (e) => {
-	let t = new a({
+}, I = (e) => {
+	let t = new c({
 		clientId: e.clientId ?? "voltro",
 		brokers: [...e.brokers],
-		logLevel: o.ERROR,
+		logLevel: l.ERROR,
 		...e.ssl === void 0 ? {} : { ssl: e.ssl },
 		...e.sasl === void 0 ? {} : { sasl: e.sasl }
 	}), n = null, r = /* @__PURE__ */ new Set();
@@ -78,10 +113,10 @@ var s = Symbol.for("@voltro/plugin-queue:consumers"), c = () => {
 							offset: s.offset,
 							key: s.key === null ? null : s.key.toString("utf8"),
 							value: s.value === null ? null : s.value.toString("utf8"),
-							headers: f(s.headers),
+							headers: F(s.headers),
 							timestamp: s.timestamp
 						};
-						await n.onMessage(c, {
+						P(e.topic, e.partition, e.highWatermark, s.offset), await n.onMessage(c, {
 							heartbeat: i,
 							isStale: o
 						}), t(s.offset), await r();
@@ -92,123 +127,134 @@ var s = Symbol.for("@voltro/plugin-queue:consumers"), c = () => {
 			} };
 		}
 	};
-}, m = Symbol.for("@voltro/plugin-queue:metrics"), h = () => {
-	let e = globalThis;
-	return e[m] ??= {
-		consumed: {},
-		dlq: {},
-		retried: {},
-		lastError: {},
-		produced: {}
-	};
-}, g = (e, t) => {
-	e[t] = (e[t] ?? 0) + 1;
-}, _ = (e) => new Promise((t) => setTimeout(t, e)), v = async (e) => {
-	let { provider: t, log: n } = e, i = h(), a = { current: !1 }, o = /* @__PURE__ */ new Set(), s = async (a) => {
-		let o = a.groupId ?? `${e.appName}.${a.topic}`, s = a.dlqTopic ?? `${a.topic}.dlq`, c = a.maxAttempts ?? 3, l = r.decodeUnknownEither(a.schema), u = async (e, r, o) => {
-			g(i.dlq, a.topic), i.lastError[a.topic] = `${e}: ${r}`.slice(0, 500), n.warn("queue: dead-lettered", {
-				topic: a.topic,
-				dlqTopic: s,
+}, L = "queue.consume", R = (e, t) => (r) => {
+	let i = u(e.traceparent);
+	return r.pipe(n.tap((e) => t === void 0 ? n.void : n.annotateCurrentSpan(t(e))), n.withSpan(L, {
+		attributes: {
+			"messaging.system": e.system,
+			"messaging.operation": "process",
+			"messaging.destination.name": e.topic,
+			"messaging.consumer.group.name": e.groupId,
+			"messaging.destination.partition.id": e.partition,
+			"messaging.message.offset": e.offset
+		},
+		...i ? { parent: i } : {}
+	}));
+}, z = async (t, i, a) => {
+	let o, s = await n.runPromiseExit(n.tryPromise({
+		try: i,
+		catch: (e) => (o = { value: e }, e)
+	}).pipe(R(t, a), d));
+	if (r.isSuccess(s)) return s.value;
+	throw o === void 0 ? e.squash(s.cause) : o.value;
+}, B = (e) => new Promise((t) => setTimeout(t, e)), V = async (e) => {
+	let { provider: t, log: n } = e, r = { current: !1 }, i = /* @__PURE__ */ new Set(), a = async (r) => {
+		let i = r.groupId ?? `${e.appName}.${r.topic}`, a = r.dlqTopic ?? `${r.topic}.dlq`, s = r.maxAttempts ?? 3, c = o.decodeUnknownEither(r.schema), l = async (e, i, o) => {
+			j(r.topic, `${e}: ${i}`), n.warn("queue: dead-lettered", {
+				topic: r.topic,
+				dlqTopic: a,
 				reason: e,
 				partition: o.partition,
 				offset: o.offset
-			}), await t.produce(s, [{
+			}), await t.produce(a, [{
 				...o.key === null ? {} : { key: o.key },
 				value: o.value ?? "",
 				headers: {
 					...o.headers,
 					"x-voltro-dlq-reason": e,
-					"x-voltro-dlq-detail": r.slice(0, 500),
+					"x-voltro-dlq-detail": i.slice(0, 500),
 					"x-voltro-dlq-source-topic": o.topic,
 					"x-voltro-dlq-source-partition": String(o.partition),
 					"x-voltro-dlq-source-offset": o.offset
 				}
 			}]);
-		}, d = await t.consume({
-			topic: a.topic,
-			groupId: o,
-			...a.fromBeginning === void 0 ? {} : { fromBeginning: a.fromBeginning },
-			onMessage: async (t, r) => {
-				let o;
-				try {
-					o = t.value === null ? null : JSON.parse(t.value);
-				} catch (e) {
-					await u("decode-failed", `invalid JSON: ${e instanceof Error ? e.message : String(e)}`, t);
-					return;
-				}
-				let s = l(o);
-				if (s._tag === "Left") {
-					await u("decode-failed", String(s.left), t);
-					return;
-				}
-				for (let o = 1; o <= c; o++) {
-					if (r.isStale()) return;
+		}, u = await t.consume({
+			topic: r.topic,
+			groupId: i,
+			...r.fromBeginning === void 0 ? {} : { fromBeginning: r.fromBeginning },
+			onMessage: async (a, o) => {
+				await z({
+					system: t.name,
+					topic: r.topic,
+					groupId: i,
+					partition: a.partition,
+					offset: a.offset,
+					traceparent: a.headers.traceparent
+				}, async () => {
+					let t;
 					try {
-						await a.handler(s.right, {
-							topic: t.topic,
-							partition: t.partition,
-							offset: t.offset,
-							key: t.key,
-							headers: t.headers,
-							attempt: o,
-							...t.headers.traceparent === void 0 ? {} : { traceparent: t.headers.traceparent },
-							get store() {
-								let t = e.store();
-								if (t === null) throw Error("queue: store not bound yet — the retry loop absorbs a message arriving before boot finished");
-								return t;
-							}
-						}), g(i.consumed, a.topic);
-						return;
-					} catch (s) {
-						let l = s instanceof Error ? s.message : String(s);
-						if (o >= c) {
-							await u("handler-failed", l, t);
-							return;
+						t = a.value === null ? null : JSON.parse(a.value);
+					} catch (e) {
+						return await l("decode-failed", `invalid JSON: ${e instanceof Error ? e.message : String(e)}`, a), "dead-lettered";
+					}
+					let i = c(t);
+					if (i._tag === "Left") return await l("decode-failed", String(i.left), a), "dead-lettered";
+					for (let t = 1; t <= s; t++) {
+						if (o.isStale()) return "stale";
+						try {
+							return await r.handler(i.right, {
+								topic: a.topic,
+								partition: a.partition,
+								offset: a.offset,
+								key: a.key,
+								headers: a.headers,
+								attempt: t,
+								...a.headers.traceparent === void 0 ? {} : { traceparent: a.headers.traceparent },
+								get store() {
+									let t = e.store();
+									if (t === null) throw Error("queue: store not bound yet — the retry loop absorbs a message arriving before boot finished");
+									return t;
+								}
+							}), A(r.topic), "ok";
+						} catch (i) {
+							let c = i instanceof Error ? i.message : String(i);
+							if (t >= s) return await l("handler-failed", c, a), "dead-lettered";
+							M(r.topic), n.warn("queue: handler failed, retrying", {
+								topic: r.topic,
+								attempt: t,
+								maxAttempts: s,
+								detail: c
+							}), await o.heartbeat().catch(() => {}), await B((e.retryBaseMs ?? 1e3) * 2 ** (t - 1));
 						}
-						g(i.retried, a.topic), n.warn("queue: handler failed, retrying", {
-							topic: a.topic,
-							attempt: o,
-							maxAttempts: c,
-							detail: l
-						}), await r.heartbeat().catch(() => {}), await _((e.retryBaseMs ?? 1e3) * 2 ** (o - 1));
 					}
-				}
+					return "stale";
+				}, (e) => ({ "voltro.queue.outcome": e }));
 			}
 		});
 		return n.info("queue: consumer started", {
-			topic: a.topic,
-			groupId: o,
-			dlqTopic: s
-		}), d;
-	}, c = e.consumers.map((e) => {
-		let t = { handle: null }, r = () => {
-			s(e).then((e) => {
-				if (a.current) {
+			topic: r.topic,
+			groupId: i,
+			dlqTopic: a
+		}), u;
+	}, s = e.consumers.map((e) => {
+		let t = { handle: null }, o = () => {
+			a(e).then((e) => {
+				if (r.current) {
 					e.stop();
 					return;
 				}
 				t.handle = e;
 			}).catch((t) => {
-				if (a.current) return;
+				if (r.current) return;
 				n.warn("queue: consumer start failed — retrying in 5s", {
 					topic: e.topic,
 					detail: t instanceof Error ? t.message : String(t)
 				});
-				let i = setTimeout(() => {
-					o.delete(i), r();
+				let a = setTimeout(() => {
+					i.delete(a), o();
 				}, 5e3);
-				o.add(i);
+				i.add(a);
 			});
 		};
-		return r(), t;
+		return o(), t;
 	});
 	return { stop: async () => {
-		a.current = !0;
-		for (let e of o) clearTimeout(e);
-		o.clear(), await Promise.all(c.map((e) => e.handle?.stop() ?? Promise.resolve()));
+		r.current = !0;
+		for (let e of i) clearTimeout(e);
+		i.clear(), await Promise.all(s.map((e) => e.handle?.stop() ?? Promise.resolve()));
 	} };
-}, y = (e) => {
-	let t = () => e.provider ?? C();
+}, H = (e) => {
+	let t = () => e.provider ?? K();
 	return {
 		name: `kafka(${e.topic})`,
 		...e.provider !== void 0 && e.provider.hosts[0] !== void 0 ? { outboundHost: e.provider.hosts[0] } : {},
@@ -225,64 +271,62 @@ var s = Symbol.for("@voltro/plugin-queue:consumers"), c = () => {
 			})));
 		}
 	};
-}, b = class extends e.Tag("@voltro/plugin-queue/QueueService")() {}, x = Symbol.for("@voltro/plugin-queue:active-provider"), S = () => {
+}, U = class extends t.Tag("@voltro/plugin-queue/QueueService")() {}, W = Symbol.for("@voltro/plugin-queue:active-provider"), G = () => {
 	let e = globalThis;
-	return e[x] ??= { current: null };
-}, C = () => {
-	let e = S().current;
+	return e[W] ??= { current: null };
+}, K = () => {
+	let e = G().current;
 	if (e === null) throw Error("[voltro] no active queue provider — is queuePlugin({...}) in this app's plugins? The outbox handler and kafkaSink need the plugin's connected provider.");
 	return e;
-}, w = () => ({
+}, q = () => ({
 	effect: "queue.produce",
 	maxAttempts: 8,
 	handler: async ({ payload: e }) => {
 		let t = e.topic, n = e.messages;
 		if (typeof t != "string" || !Array.isArray(n)) throw Error("queue.produce payload must be { topic: string, messages: QueueOutgoingMessage[] }");
-		await C().produce(t, n);
-		let r = h();
-		r.produced[t] = (r.produced[t] ?? 0) + n.length;
+		await K().produce(t, n), N(t, n.length);
 	}
-}), T = (e) => {
-	let r = e.name ?? "queue", a = p(e), o = ["inspect:read", ...a.hosts.map((e) => `network:outbound:${e}`)], s = null, c = { current: null };
-	return i({
-		name: r,
+}), J = (e) => {
+	let t = e.name ?? "queue", r = I(e), a = ["inspect:read", ...r.hosts.map((e) => `network:outbound:${e}`)], o = null, c = { current: null };
+	return s({
+		name: t,
 		baseName: "queue",
 		description: "Queue interop — consume and produce against an existing Kafka: Schema-decoded consumers (at-least-once, serial per partition, retry + dead-letter), batched producing via a handler service or transactionally through the outbox, and a kafkaSink for plugin-cdc-out.",
-		permissions: o,
-		services: n.succeed(b, { produce: (e, t) => a.produce(e, t) }),
+		permissions: a,
+		services: i.succeed(U, { produce: (e, t) => r.produce(e, t) }),
 		bindDataStore: (e) => {
 			c.current = e;
 		},
-		onActivate: (n) => t.tryPromise(async () => {
-			await a.connect(), S().current = a;
-			let t = u();
-			s = await v({
-				provider: a,
-				consumers: t,
-				appName: e.groupPrefix ?? n.app.name,
-				log: n.logger,
+		onActivate: (t) => n.tryPromise(async () => {
+			await r.connect(), G().current = r;
+			let n = h();
+			o = await V({
+				provider: r,
+				consumers: n,
+				appName: e.groupPrefix ?? t.app.name,
+				log: t.logger,
 				store: () => c.current,
 				...e.retryBaseMs === void 0 ? {} : { retryBaseMs: e.retryBaseMs }
-			}), n.logger.info("queue: connected", {
-				brokers: a.hosts,
-				consumers: t.length
+			}), t.logger.info("queue: connected", {
+				brokers: r.hosts,
+				consumers: n.length
 			});
 		}),
-		onDeactivate: () => t.tryPromise(async () => {
-			await s?.stop(), s = null, S().current === a && (S().current = null), await a.disconnect();
+		onDeactivate: () => n.tryPromise(async () => {
+			await o?.stop(), o = null, G().current === r && (G().current = null), await r.disconnect();
 		}),
 		inspectEndpoints: [{
 			method: "GET",
 			path: "/consumers",
 			description: "Registered consumers + per-topic consumed/retried/dead-lettered/produced counters (this replica).",
-			handler: () => t.sync(() => {
-				let e = h();
+			handler: () => n.sync(() => {
+				let e = v();
 				return {
 					kind: "json",
 					data: {
-						connected: S().current !== null,
-						brokers: a.hosts,
-						consumers: u().map((t) => ({
+						connected: G().current !== null,
+						brokers: r.hosts,
+						consumers: h().map((t) => ({
 							topic: t.topic,
 							groupId: t.groupId ?? null,
 							dlqTopic: t.dlqTopic ?? `${t.topic}.dlq`,
@@ -300,4 +344,4 @@ var s = Symbol.for("@voltro/plugin-queue:consumers"), c = () => {
 	});
 };
 //#endregion
-export { b as QueueService, C as activeQueueProvider, l as defineQueueConsumer, p as kafkaProvider, y as kafkaSink, h as queueMetrics, w as queueOutboxHandler, T as queuePlugin, u as registeredQueueConsumers, d as resetQueueConsumersForTest, v as startQueueConsumers };
+export { U as QueueService, K as activeQueueProvider, m as defineQueueConsumer, I as kafkaProvider, H as kafkaSink, v as queueMetrics, q as queueOutboxHandler, J as queuePlugin, h as registeredQueueConsumers, g as resetQueueConsumersForTest, V as startQueueConsumers };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voltro/plugin-queue",
-  "version": "0.53.0",
+  "version": "0.54.0",
   "description": "Queue interop — consume and produce against an existing Kafka: Schema-decoded consumers via defineQueueConsumer (*.consumer.ts; at-least-once, serial per partition, retry + dead-letter), batched producing via QueueService or transactionally through the outbox (queueOutboxHandler), and a kafkaSink for plugin-cdc-out.",
   "keywords": [
     "voltro",
@@ -33,9 +33,10 @@
     "node": ">=24.0.0"
   },
   "dependencies": {
-    "@voltro/protocol": "0.53.0",
+    "@voltro/protocol": "0.54.0",
     "kafkajs": "^2.2.4",
-    "@voltro/database": "0.53.0"
+    "@voltro/database": "0.54.0",
+    "@voltro/runtime": "0.54.0"
   },
   "peerDependencies": {
     "effect": "^3.22.0"