@voltro/plugin-cdc-out 0.1.0

package/dist/index.js

@@ -0,0 +1,507 @@
+import { Duration as e, Effect as t, Fiber as n, Schedule as r, Schema as i } from "effect";
+import { definePlugin as a } from "@voltro/protocol";
+import { and as o, count as s, eq as c, generateId as l, id as u, inSet as d, integer as f, json as p, lt as m, registerRetention as h, retentionTtlMsFromEnv as g, table as _, text as v, timestamp as y } from "@voltro/database";
+//#region src/errors.ts
+var b = class extends i.TaggedError()("CdcConfigError", { message: i.String }) {}, x = class extends i.TaggedError()("CdcDeliveryError", {
+	sink: i.String,
+	reason: i.String
+}) {
+	get message() {
+		return `cdc-out delivery to '${this.sink}' failed: ${this.reason}`;
+	}
+}, S = "_voltro_cdcout_outbox", C = "_voltro_cdcout_leases", w = {
+	kind: "typeid",
+	prefix: "cdcout"
+}, T = () => l(w, S), E = _(S, {
+	id: u({ prefix: "cdcout" }),
+	pipe: v(),
+	sourceTable: v(),
+	op: v(),
+	key: v(),
+	payload: p().nullable(),
+	status: v(),
+	attempts: f().default(0),
+	nextAttemptAt: y(),
+	claimedBy: v().nullable(),
+	claimedAt: y().nullable(),
+	createdAt: y(),
+	deliveredAt: y().nullable(),
+	lastError: v().nullable()
+}).index(["pipe", "status"]), D = _(C, {
+	id: u({ scheme: "numeric" }),
+	key: v(),
+	holder: v(),
+	expiresAt: y(),
+	renewedAt: y()
+}).unique("cdcoutLeaseKey", ["key"]), O = (e) => e instanceof Date ? e.getTime() : new Date(String(e)).getTime(), k = class {
+	opts;
+	held = !1;
+	constructor(e) {
+		this.opts = e;
+	}
+	get isLeader() {
+		return this.held;
+	}
+	async heartbeat(e) {
+		let t = (this.opts.now ?? Date.now)(), n = {
+			holder: this.opts.holder,
+			expiresAt: new Date(t + this.opts.ttlMs),
+			renewedAt: new Date(t)
+		};
+		try {
+			if (await e.updateMany("_voltro_cdcout_leases", n, { where: o(c("key", this.opts.key), c("holder", this.opts.holder)) }) === 1) return this.held = !0, !0;
+			let r = await e.insertIgnore(C, {
+				key: this.opts.key,
+				...n
+			}, { conflictColumns: ["key"] });
+			if (r.holder === this.opts.holder) return this.held = !0, !0;
+			let i = await e.updateMany(C, n, { where: o(c("key", this.opts.key), c("holder", String(r.holder)), m("expiresAt", new Date(t))) });
+			return this.held = i === 1, this.held;
+		} catch {
+			return this.held = !1, !1;
+		}
+	}
+	async release(e) {
+		this.held = !1;
+		try {
+			await e.updateMany(C, { expiresAt: /* @__PURE__ */ new Date(0) }, { where: o(c("key", this.opts.key), c("holder", this.opts.holder)) });
+		} catch {}
+	}
+}, A = (n, r, i) => t.suspend(() => {
+	let a = new AbortController();
+	return t.try(() => n.deliver(r, { signal: a.signal })).pipe(t.flatMap((e) => t.isEffect(e) ? e : t.tryPromise(() => e))).pipe(t.timeoutFail({
+		duration: e.millis(i.timeoutMs),
+		onTimeout: () => new x({
+			sink: n.name,
+			reason: `delivery timed out after ${i.timeoutMs}ms`
+		})
+	}), t.onError(() => t.sync(() => a.abort())), t.mapError((e) => e instanceof x ? e : j(n.name, e)), t.withSpan("cdcOut.deliver", { attributes: {
+		pipe: i.pipe,
+		sink: n.name,
+		records: r.length
+	} }));
+}), j = (e, t) => {
+	let n = typeof t == "object" && t && "error" in t ? t.error : t;
+	return new x({
+		sink: e,
+		reason: n instanceof Error ? n.message : String(n)
+	});
+}, M = (e, t) => {
+	let n = Math.min(t.baseMs * 2 ** Math.max(0, e - 1), t.maxMs);
+	return Math.round(n * (.5 + .5 * (t.random ?? Math.random)()));
+}, N = class {
+	opts;
+	now;
+	stuckClaimMs;
+	constructor(e) {
+		this.opts = e, this.now = e.now ?? Date.now, this.stuckClaimMs = Math.max(2 * e.deliveryTimeoutMs, 1e4);
+	}
+	async tick(e) {
+		this.opts.isLeader() && await Promise.all(this.opts.pipes.map(async (t) => {
+			if (!t.busy) {
+				t.busy = !0;
+				try {
+					await this.pipeTick(e, t);
+				} finally {
+					t.busy = !1;
+				}
+			}
+		}));
+	}
+	async pipeTick(e, n) {
+		let r = this.now();
+		if (await e.updateMany(S, {
+			status: "pending",
+			claimedBy: null,
+			claimedAt: null
+		}, { where: o(c("pipe", n.key), c("status", "delivering"), m("claimedAt", new Date(r - this.stuckClaimMs))) }), (await e.query({
+			table: "_voltro_cdcout_outbox",
+			predicate: o(c("pipe", n.key), c("status", "delivering")),
+			order: [],
+			take: 1,
+			skip: void 0,
+			projection: ["id"]
+		})).length > 0) return;
+		let i = await e.query({
+			table: S,
+			predicate: o(c("pipe", n.key), c("status", "pending")),
+			order: [{
+				column: "id",
+				direction: "asc"
+			}],
+			take: n.batchSize,
+			skip: void 0,
+			projection: void 0
+		});
+		if (i.length === 0) return;
+		let a = [];
+		for (let e of i) {
+			if (O(e.nextAttemptAt) > r) break;
+			a.push(e);
+		}
+		if (a.length === 0) return;
+		let s = a.map((e) => e.id);
+		if (await e.updateMany("_voltro_cdcout_outbox", {
+			status: "delivering",
+			claimedBy: this.opts.replicaId,
+			claimedAt: new Date(r)
+		}, { where: o(c("pipe", n.key), d("id", s), c("status", "pending")) }) === 0) return;
+		let l = await e.query({
+			table: S,
+			predicate: o(c("pipe", n.key), d("id", s), c("status", "delivering"), c("claimedBy", this.opts.replicaId)),
+			order: [{
+				column: "id",
+				direction: "asc"
+			}],
+			take: void 0,
+			skip: void 0,
+			projection: void 0
+		});
+		if (l.length === 0) return;
+		let u = l.map((e) => ({
+			table: e.sourceTable,
+			op: e.op,
+			key: e.key,
+			data: e.payload,
+			deliveryKey: e.id
+		})), f = await t.runPromise(t.either(A(n.sink, u, {
+			timeoutMs: this.opts.deliveryTimeoutMs,
+			pipe: n.key
+		}))), p = this.now();
+		if (f._tag === "Right") {
+			await e.updateMany(S, {
+				status: "delivered",
+				deliveredAt: new Date(p),
+				claimedBy: null,
+				claimedAt: null,
+				lastError: null
+			}, { where: o(c("pipe", n.key), d("id", l.map((e) => e.id)), c("status", "delivering"), c("claimedBy", this.opts.replicaId)) }), n.stats.delivered += l.length, n.stats.lastDeliveryAtMs = p;
+			return;
+		}
+		let h = f.left.message;
+		for (let t of l) {
+			let r = t.attempts + 1;
+			if (r >= this.opts.maxAttempts) await e.update(S, t.id, {
+				status: "dead",
+				attempts: r,
+				claimedBy: null,
+				claimedAt: null,
+				lastError: h
+			}), n.stats.dead += 1;
+			else {
+				let n = M(r, {
+					baseMs: this.opts.backoffBaseMs,
+					maxMs: this.opts.backoffMaxMs,
+					...this.opts.random === void 0 ? {} : { random: this.opts.random }
+				});
+				await e.update(S, t.id, {
+					status: "pending",
+					attempts: r,
+					nextAttemptAt: new Date(p + n),
+					claimedBy: null,
+					claimedAt: null,
+					lastError: h
+				});
+			}
+			n.stats.failed += 1;
+		}
+		this.opts.warn?.("cdc-out delivery failed (durable retry scheduled)", {
+			pipe: n.key,
+			sink: n.sink.name,
+			records: l.length,
+			attempts: l[0] === void 0 ? 1 : l[0].attempts + 1,
+			maxAttempts: this.opts.maxAttempts,
+			reason: h
+		});
+	}
+}, P = (e = "memory") => {
+	let t = [];
+	return {
+		name: e,
+		deliver: async (e) => {
+			t.push([...e]);
+		},
+		get batches() {
+			return t;
+		},
+		get delivered() {
+			return t.flat();
+		}
+	};
+}, F = (e, t) => {
+	let n;
+	try {
+		n = new URL(e).host;
+	} catch {
+		throw new b({ message: `webhookSink: invalid url '${e}'` });
+	}
+	let r = `webhook(${n})`;
+	return {
+		name: r,
+		outboundHost: n,
+		deliver: async (n, i) => {
+			let a = await fetch(e, {
+				method: "POST",
+				headers: {
+					"content-type": "application/json",
+					...t?.headers ?? {}
+				},
+				body: JSON.stringify({ records: n }),
+				signal: i.signal
+			});
+			if (!a.ok) throw new x({
+				sink: r,
+				reason: `${a.status} ${a.statusText}`
+			});
+		}
+	};
+}, I = 100, L = 5, R = 200, z = 3e4, B = 250, V = 1e4, H = 15e3, U = 72, W = (e) => {
+	throw new b({ message: e });
+}, G = (e) => {
+	(e.sinks === void 0 || e.sinks.length === 0) && W("cdcOutPlugin: at least one sink config is required");
+	let t = /* @__PURE__ */ new Set();
+	e.sinks.forEach((e, n) => {
+		(typeof e.table != "string" || e.table.length === 0) && W(`cdcOutPlugin: sinks[${n}].table is required`), e.table.startsWith("_voltro_cdcout") && W(`cdcOutPlugin: sinks[${n}].table '${e.table}' is the plugin's own delivery state — it cannot be mirrored`), t.has(e.table) && W(`cdcOutPlugin: duplicate sink config for table '${e.table}' — one sink config per table per instance; for multiple sinks on one table wire a second cdcOutPlugin({ name: '…' }) instance`), t.add(e.table), (e.sink === void 0 || typeof e.sink.deliver != "function") && W(`cdcOutPlugin: sinks[${n}] (table '${e.table}') needs a sink with a deliver(batch) function`), e.batchSize !== void 0 && e.batchSize <= 0 && W(`cdcOutPlugin: sinks[${n}] (table '${e.table}') batchSize must be > 0`);
+	}), e.name !== void 0 && !/^[a-zA-Z0-9-]+$/.test(e.name) && W(`cdcOutPlugin: name '${e.name}' must match [a-zA-Z0-9-]+ (it lands in the inspect URL)`);
+	let n = [
+		["maxAttempts", e.maxAttempts],
+		["backoffBaseMs", e.backoffBaseMs],
+		["backoffMaxMs", e.backoffMaxMs],
+		["sweepIntervalMs", e.sweepIntervalMs],
+		["deliveryTimeoutMs", e.deliveryTimeoutMs],
+		["leaseTtlMs", e.leaseTtlMs],
+		["retentionHours", e.retentionHours]
+	];
+	for (let [e, t] of n) t !== void 0 && t < 1 && W(`cdcOutPlugin: ${e} must be >= 1`);
+}, K = (e, t) => {
+	if (t.filter !== void 0 && !t.filter(e)) return null;
+	if (e.op === "delete") {
+		let t = String(e.old?.id ?? "");
+		return t === "" ? null : {
+			table: e.table,
+			op: "delete",
+			key: t,
+			data: null
+		};
+	}
+	let n = e.new;
+	if (n === null) return null;
+	let r = String(n.id ?? "");
+	return r === "" ? null : {
+		table: e.table,
+		op: e.op,
+		key: r,
+		data: t.map ? t.map(n) : n
+	};
+}, q = (e, t) => `${e}:${t.table}:${t.sink.name}`, J = (e, t, n) => ({
+	id: T(),
+	pipe: t,
+	sourceTable: e.table,
+	op: e.op,
+	key: e.key,
+	payload: e.data,
+	status: "pending",
+	attempts: 0,
+	nextAttemptAt: new Date(n),
+	claimedBy: null,
+	claimedAt: null,
+	createdAt: new Date(n),
+	deliveredAt: null,
+	lastError: null
+}), Y = async (e, t, n, r) => {
+	let i = q(r?.instanceName ?? "default", t), a = (r?.now ?? Date.now)(), o = n.flatMap((e) => {
+		let n = String(e.id ?? "");
+		return n === "" ? [] : [J({
+			table: t.table,
+			op: "insert",
+			key: n,
+			data: t.map ? t.map(e) : e
+		}, i, a)];
+	});
+	return o.length > 0 && await e.insertMany(S, o), o.length;
+}, X = (i) => {
+	G(i);
+	let l = i.name ?? "default", u = i.name ? `@voltro/plugin-cdc-out#${i.name}` : "@voltro/plugin-cdc-out", f = `cdcout-${Math.random().toString(36).slice(2, 10)}`, p = i.maxAttempts ?? L, m = i.sweepIntervalMs ?? B, _ = i.leaseTtlMs ?? H, v = /* @__PURE__ */ new Map();
+	for (let e of i.sinks) v.set(e.table, {
+		config: e,
+		enginePipe: {
+			key: q(l, e),
+			sink: e.sink,
+			batchSize: e.batchSize ?? I,
+			busy: !1,
+			stats: {
+				delivered: 0,
+				failed: 0,
+				dead: 0,
+				lastDeliveryAtMs: null
+			}
+		}
+	});
+	let y = new k({
+		key: l,
+		holder: f,
+		ttlMs: _
+	}), b = new N({
+		pipes: [...v.values()].map((e) => e.enginePipe),
+		replicaId: f,
+		maxAttempts: p,
+		backoffBaseMs: i.backoffBaseMs ?? R,
+		backoffMaxMs: i.backoffMaxMs ?? z,
+		deliveryTimeoutMs: i.deliveryTimeoutMs ?? V,
+		isLeader: () => y.isLeader,
+		warn: (e, t) => C(e, t)
+	}), x = { current: null }, C = () => void 0, w = 0, T, O;
+	h({
+		table: S,
+		timeColumn: "createdAt",
+		ttlMs: i.retentionHours === void 0 ? g(process.env.CDCOUT_RETENTION_HOURS, U) : i.retentionHours * 36e5,
+		where: d("status", ["delivered", "dead"]),
+		label: "cdc-out outbox (delivered/dead)"
+	});
+	let A = /* @__PURE__ */ new Set();
+	for (let e of i.sinks) e.sink.outboundHost && A.add(e.sink.outboundHost);
+	let j = [
+		"store:changes:read",
+		"store:write",
+		"inspect:read",
+		...[...A].map((e) => `network:outbound:${e}`)
+	], M = [
+		"pending",
+		"delivering",
+		"delivered",
+		"dead"
+	], P = async (e, t) => {
+		let n = {
+			pending: 0,
+			delivering: 0,
+			delivered: 0,
+			dead: 0
+		};
+		return await Promise.all(M.map(async (r) => {
+			let i = await e.query({
+				table: S,
+				predicate: o(c("pipe", t), c("status", r)),
+				order: [],
+				take: void 0,
+				skip: void 0,
+				projection: void 0,
+				aggregations: [s("n")]
+			});
+			n[r] = Number(i[0]?.n ?? 0);
+		})), n;
+	};
+	return a({
+		name: u,
+		description: "Declarative reverse-ETL: mirror table changes outward to external sinks (webhook/Kafka/Snowflake/BigQuery) through a durable outbox — ordered per pipe, at-least-once from enqueue, retried with backoff, dead-lettered.",
+		permissions: j,
+		declaredEnv: [{
+			name: "CDCOUT_RETENTION_HOURS",
+			required: !1,
+			secret: !1,
+			description: "Hours delivered/dead outbox rows are kept before the retention sweep purges them (default 72).",
+			example: "72"
+		}],
+		extendSchema: { tables: [E, D] },
+		bindDataStore: (e) => {
+			x.current = e;
+		},
+		inspectEndpoints: [{
+			method: "GET",
+			path: "/sinks",
+			description: "Configured sinks + per-pipe outbox counts and delivery stats.",
+			handler: () => t.tryPromise(async () => {
+				let e = x.current;
+				return {
+					kind: "json",
+					data: { sinks: await Promise.all([...v.values()].map(async ({ config: t, enginePipe: n }) => {
+						let r = e === null ? {
+							pending: 0,
+							delivering: 0,
+							delivered: 0,
+							dead: 0
+						} : await P(e, n.key);
+						return {
+							table: t.table,
+							sink: t.sink.name,
+							pipe: n.key,
+							...r,
+							failed: r.dead,
+							lastDeliveryAtMs: n.stats.lastDeliveryAtMs
+						};
+					})) }
+				};
+			})
+		}, {
+			method: "GET",
+			path: "/dead-letter",
+			description: "Outbox rows that exhausted maxAttempts (newest first, capped at 100).",
+			handler: () => t.tryPromise(async () => {
+				let e = x.current;
+				return e === null ? {
+					kind: "json",
+					data: { rows: [] }
+				} : {
+					kind: "json",
+					data: { rows: (await e.query({
+						table: S,
+						predicate: c("status", "dead"),
+						order: [{
+							column: "id",
+							direction: "desc"
+						}],
+						take: 100,
+						skip: void 0,
+						projection: void 0
+					})).map((e) => ({
+						deliveryKey: e.id,
+						pipe: e.pipe,
+						table: e.sourceTable,
+						op: e.op,
+						key: e.key,
+						attempts: e.attempts,
+						lastError: e.lastError,
+						createdAt: e.createdAt
+					})) }
+				};
+			})
+		}],
+		onChangeEvent: (e) => {
+			let n = v.get(e.table);
+			if (n === void 0) return t.void;
+			if (e.changeScope === "fleet") {
+				if (!y.isLeader) return t.void;
+			} else if (e.origin === "injected") return t.void;
+			let r = K(e, n.config);
+			if (r === null) return t.void;
+			let i = x.current;
+			if (i === null) {
+				let n = Date.now();
+				return n - w > 6e4 && (w = n, C("cdc-out: no DataStore bound — change events are NOT being mirrored", { table: e.table })), t.void;
+			}
+			let a = J(r, n.enginePipe.key, Date.now());
+			return t.tryPromise(() => i.insert(S, a).then(() => void 0));
+		},
+		onActivate: (n) => t.sync(() => {
+			C = (e, t) => n.logger.warn(e, t);
+			let i = t.suspend(() => {
+				let e = x.current;
+				return e === null ? t.void : t.promise(() => y.heartbeat(e).then(() => void 0));
+			}).pipe(t.repeat(r.spaced(e.millis(Math.max(1, Math.floor(_ / 3)))))), a = t.suspend(() => {
+				let e = x.current;
+				return e === null ? t.void : t.tryPromise(() => b.tick(e)).pipe(t.catchAll((e) => t.sync(() => C("cdc-out worker sweep failed", { reason: String(e) }))));
+			}).pipe(t.repeat(r.spaced(e.millis(m))));
+			O = t.runFork(i), T = t.runFork(a), n.logger.info("cdc-out active", {
+				tables: [...v.keys()],
+				replicaId: f,
+				outbox: S
+			});
+		}),
+		onDeactivate: () => t.gen(function* () {
+			T !== void 0 && (yield* n.interrupt(T)), O !== void 0 && (yield* n.interrupt(O)), T = void 0, O = void 0;
+			let e = x.current;
+			e !== null && (yield* t.promise(() => y.release(e)));
+		})
+	});
+};
+//#endregion
+export { C as CDCOUT_LEASES_TABLE, S as CDCOUT_OUTBOX_TABLE, b as CdcConfigError, x as CdcDeliveryError, k as CdcLeaseManager, N as CdcOutEngine, D as cdcOutLeasesTable, E as cdcOutOutboxTable, X as cdcOutPlugin, A as deliverOnce, Y as enqueueBackfill, K as mapChange, P as memorySink, T as nextOutboxId, J as outboxRowFor, q as pipeKey, M as retryDelayMs, G as validateCdcOutConfig, F as webhookSink };

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@voltro/plugin-cdc-out",
+  "version": "0.1.0",
+  "description": "Declarative reverse-ETL: mirror framework table changes outward to external sinks (webhook / Kafka / Snowflake / BigQuery) through a durable outbox — ordered per pipe, at-least-once from enqueue, retried with backoff, dead-lettered.",
+  "keywords": [
+    "voltro",
+    "typescript",
+    "framework"
+  ],
+  "license": "SEE LICENSE IN LICENSE",
+  "homepage": "https://voltro.dev",
+  "bugs": {
+    "email": "support@voltro.dev"
+  },
+  "author": {
+    "name": "Voltro UG",
+    "url": "https://voltro.dev"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "sideEffects": false,
+  "engines": {
+    "node": ">=24.0.0"
+  },
+  "dependencies": {
+    "@voltro/database": "0.1.0",
+    "@voltro/protocol": "0.1.0"
+  },
+  "peerDependencies": {
+    "effect": "^3.21.4"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}