@voltro/workflow 0.85.0 → 0.86.0

package/dist/runRecording.js

@@ -0,0 +1,615 @@
+import { failureMessage as e } from "./failureMessage.js";
+import { STEP_JOURNAL_BINDING as t, anyWorkflowEncrypted as n, isWorkflowEncrypted as r, loggableCause as i, openWorkflowRow as a, sealEventPayload as o, sealJournalValue as s } from "./journalEncryption.js";
+import { CurrentWorkflowExecutionId as c, CurrentWorkflowGeneration as l, CurrentWorkflowPatches as u, CurrentWorkflowRunId as d, CurrentWorkflowWakeTenant as f, WorkflowRunRecorder as p } from "./recorder.js";
+import { CurrentWorkflowReplayShape as ee, makeReplayShapeTracker as m, nondeterminismEventPayload as te } from "./nondeterminism.js";
+import { CurrentWorkflowDelivery as ne } from "./workflowDelivery.js";
+import { completeWorkflowRunGenerationInTransaction as h, enterWorkflowRunGenerationInTransaction as re, suspendWorkflowRunGenerationInTransaction as g } from "./runGeneration.js";
+import { awaitWorkflowCancellation as ie } from "./controlMonitor.js";
+import { Workflow as _ } from "effect/workflow";
+import { Cause as v, Effect as y, Exit as b, Fiber as x } from "effect";
+import { publishServerError as S } from "@voltro/protocol";
+import { eq as C } from "@voltro/database";
+import { WorkflowInstance as w } from "effect/workflow/WorkflowEngine";
+//#region src/runRecording.ts
+var T = 8192, E = 3, D = 2e3, O = () => {
+	let e = (e) => {
+		if (e === void 0) return;
+		let t = Number(e);
+		return Number.isFinite(t) && t > 0 ? Math.floor(t) : void 0;
+	}, t = e(process.env.VOLTRO_WORKFLOW_MAX_RECLAIMS), n = e(process.env.VOLTRO_WORKFLOW_REPLAY_SHAPE_LIMIT);
+	return {
+		...t === void 0 ? {} : { maxRunReclaims: t },
+		...n === void 0 ? {} : { replayShapeLimit: n }
+	};
+}, k = (e) => {
+	let t = typeof e == "string" ? (() => {
+		try {
+			return JSON.parse(e);
+		} catch {
+			return;
+		}
+	})() : e;
+	if (Array.isArray(t)) return t.filter((e) => typeof e == "string");
+}, A = () => {
+	let e = globalThis.crypto;
+	return typeof e?.randomUUID == "function" ? e.randomUUID() : `${Date.now().toString(36)}${Math.random().toString(36).slice(2)}`;
+}, j = (e) => `${e}_${A()}`, M = (e, t = T) => {
+	if (e == null) return e;
+	try {
+		let n = JSON.stringify(e);
+		return n.length <= t ? e : {
+			__truncated: !0,
+			bytes: n.length,
+			sample: n.slice(0, t)
+		};
+	} catch {
+		return { __unserialisable: String(e) };
+	}
+}, N = async (e, t, r) => {
+	if (r === null || !n() || !("value" in r || "result" in r || "errorMessage" in r) || e.query === void 0) return r;
+	let [i] = await e.query({
+		table: "_voltro_workflow_runs",
+		predicate: C("id", t),
+		order: [],
+		take: 1,
+		skip: void 0,
+		projection: void 0
+	});
+	return typeof i?.tag == "string" ? o(i.tag, r) : r;
+}, P = (e) => {
+	let t = {};
+	for (let [n, r] of Object.entries(e)) t[n] = r instanceof Date ? r.toISOString() : r;
+	return a(t);
+}, F = (n) => {
+	let r = n.truncateBytes ?? T, i = n.makeId ?? j, a = (e, t) => n.log?.warn(e, t), o = (e, n) => n === !0 ? s(e, "workflow step value", t) : e, c = n.emit, l = n.recording === "coarse";
+	return {
+		startStep: async ({ runId: t, stepName: s, attempt: u, stepInput: d, retryPolicy: f, sealed: p }) => {
+			if (!l) try {
+				let e = await n.store.insert("_voltro_workflow_run_steps", {
+					id: i("ws"),
+					runId: t,
+					stepName: s,
+					attempt: u,
+					status: "running",
+					input: d === void 0 ? null : o(M(d, r), p),
+					retryPolicy: f ?? null,
+					output: null,
+					errorTag: null,
+					errorMessage: null,
+					startedAt: /* @__PURE__ */ new Date(),
+					completedAt: null,
+					durationMs: null
+				});
+				return c?.("workflowSteps", {
+					op: "insert",
+					row: P(e)
+				}), e.id;
+			} catch (n) {
+				a("failed to record step start (continuing)", {
+					runId: t,
+					stepName: s,
+					attempt: u,
+					reason: e(n)
+				});
+				return;
+			}
+		},
+		endStepSuccess: async ({ stepRecordId: t, output: i, durationMs: s, sealed: u }) => {
+			if (!l) try {
+				let e = await n.store.update("_voltro_workflow_run_steps", t, {
+					status: "succeeded",
+					output: o(M(i, r), u),
+					completedAt: /* @__PURE__ */ new Date(),
+					durationMs: s
+				});
+				e && c?.("workflowSteps", {
+					op: "update",
+					row: P(e)
+				});
+			} catch (n) {
+				a("failed to record step success", {
+					stepRecordId: t,
+					reason: e(n)
+				});
+			}
+		},
+		endStepFailure: async ({ stepRecordId: t, errorTag: i, errorMessage: s, errorCause: u, durationMs: d, sealed: f }) => {
+			if (!l) try {
+				let e = await n.store.update("_voltro_workflow_run_steps", t, {
+					status: "failed",
+					errorTag: i,
+					errorMessage: s == null ? s : o(s, f),
+					errorCause: u === void 0 ? null : o(M(u, r), f),
+					completedAt: /* @__PURE__ */ new Date(),
+					durationMs: d
+				});
+				e && c?.("workflowSteps", {
+					op: "update",
+					row: P(e)
+				});
+			} catch (n) {
+				a("failed to record step failure", {
+					stepRecordId: t,
+					reason: e(n)
+				});
+			}
+		},
+		recordEvent: async ({ runId: t, eventType: r, payload: o, stepName: s, attempt: l }) => {
+			let u = i("wfev"), d = !0;
+			try {
+				let e = await n.store.insert("_voltro_workflow_run_events", {
+					id: u,
+					runId: t,
+					eventType: r,
+					payload: await N(n.store, t, o ?? null),
+					occurredAt: /* @__PURE__ */ new Date(),
+					stepName: s ?? null,
+					attempt: l ?? null
+				});
+				c?.("workflowEvents", {
+					op: "insert",
+					row: P(e)
+				});
+			} catch (n) {
+				d = !1, a("failed to record workflow event (continuing)", {
+					runId: t,
+					eventType: r,
+					reason: e(n)
+				});
+			}
+			if (n.wakeups !== void 0) {
+				let e = o?.scheduledWakeAt, i = typeof e == "string" ? new Date(e) : void 0, a = o?.tenantId, s = typeof a == "string" ? a : void 0;
+				r === "timer-set" && i !== void 0 && !Number.isNaN(i.getTime()) ? await n.wakeups.register(t, i, s).catch(() => {}) : (r === "timer-fired" || r === "run-succeeded" || r === "run-failed" || r === "run-cancelled") && await n.wakeups.cancel(t, s).catch(() => {});
+			}
+			return {
+				id: u,
+				recorded: d
+			};
+		}
+	};
+}, ae = (t, n, r) => y.gen(function* () {
+	let i = t.store.query;
+	if (i === void 0) return;
+	let a = t.replayShapeLimit ?? 2e3, o = yield* y.promise(() => i.call(t.store, {
+		table: "_voltro_workflow_run_steps",
+		order: [{
+			column: "startedAt",
+			direction: "asc"
+		}],
+		predicate: C("runId", n),
+		projection: ["stepName"],
+		skip: void 0,
+		take: a + 1
+	}).catch((t) => (r("failed to read the replay shape (nondeterminism tripwire disabled for this attempt)", {
+		runId: n,
+		reason: e(t)
+	}), [])));
+	if (o.length === 0) return;
+	if (o.length > a) {
+		r("replay shape exceeds the step limit — nondeterminism tripwire disabled for this run", {
+			runId: n,
+			limit: a
+		});
+		return;
+	}
+	let s = o.flatMap((e) => {
+		let t = e.stepName;
+		return typeof t == "string" ? [{ stepName: t }] : [];
+	});
+	return s.length === 0 ? void 0 : m(s);
+}), I = (t, n) => {
+	let a = t.truncateBytes ?? T, o = t.makeId ?? j, s = t.recorder ?? F(t), m = (e, n) => t.log?.warn(e, n), C = t.emit, E = (e) => y.gen(function* () {
+		yield* y.sync(() => C?.("workflowEvents", {
+			op: "insert",
+			row: P(e)
+		}));
+		let n = t.wakeups, r = e.runId;
+		if (n !== void 0 && typeof r == "string") {
+			let e = t.subject?.tenantId;
+			yield* y.promise(() => n.cancel(r, typeof e == "string" ? e : void 0).catch(() => {
+				m("failed to clear terminal workflow wakeup", { runId: r });
+			}));
+		}
+	}), D = (n, r, i) => y.promise(async () => {
+		if (t.onParentClose) try {
+			await t.onParentClose({
+				workflowName: t.name,
+				executionId: n,
+				...i === void 0 ? {} : { runId: i },
+				status: r
+			});
+		} catch (i) {
+			m("failed to apply workflow parent-close policy", {
+				tag: t.name,
+				executionId: n,
+				status: r,
+				reason: e(i)
+			});
+		}
+	}), O = (n, r, i, a) => y.promise(async () => {
+		if (t.onTerminal) try {
+			await t.onTerminal({
+				workflowName: t.name,
+				executionId: n,
+				...i === void 0 ? {} : { runId: i },
+				status: r,
+				...a === void 0 ? {} : a
+			});
+		} catch (i) {
+			m("workflow terminal observer failed", {
+				tag: t.name,
+				executionId: n,
+				status: r,
+				reason: e(i)
+			});
+		}
+	});
+	return (T, j) => y.uninterruptibleMask((N) => y.gen(function* () {
+		let F = Date.now(), I = yield* c, oe = yield* l, L = t.parentExecutionId ?? I ?? null, se = t.parentGeneration ?? (t.parentExecutionId == null ? oe : void 0) ?? null, R = t.parentClosePolicy ?? null, z, B = (n, r) => r.pipe(y.catchCause((r) => v.hasInterrupts(r) ? y.failCause(r) : y.sync(() => {
+			let i = v.squash(r);
+			m(n, {
+				tag: t.name,
+				executionId: j,
+				reason: e(i)
+			});
+		}))), V = `wfrg_${A()}`, H = yield* y.serviceOption(ne), ce = o("wr"), le = yield* y.sync(() => t.encodePayload === void 0 ? T : t.encodePayload(T)), U = yield* y.promise(() => t.transaction((e) => re(e, {
+			workflowName: t.name,
+			executionId: j,
+			generation: V,
+			now: F,
+			delivery: H._tag === "Some" ? H.value : null,
+			initialRow: {
+				id: ce,
+				payload: le,
+				workflowVersion: t.workflowVersion ?? null,
+				workflowPatches: t.workflowPatches ?? null,
+				output: null,
+				subject: t.subject ?? null,
+				source: t.source ?? null,
+				errorTag: null,
+				errorMessage: null,
+				cancelled: !1,
+				startedAt: new Date(F),
+				completedAt: null,
+				durationMs: null,
+				traceId: t.traceId ?? null,
+				parentExecutionId: L,
+				parentGeneration: se,
+				parentClosePolicy: R
+			}
+		})));
+		if (typeof U.row.id != "string") return yield* y.die(/* @__PURE__ */ Error("Recorded workflow entry has no run identity"));
+		if (z = U.row.id, U.restartEvent !== void 0) {
+			let e = U.restartEvent;
+			yield* B("failed to observe recorded workflow restart", y.sync(() => {
+				C?.("workflowEvents", {
+					op: "insert",
+					row: P(e)
+				});
+			}));
+		}
+		if (U.kind === "cancelled" || U.kind === "already-cancelled") {
+			let e = yield* w;
+			return e.interrupted = !0, U.kind === "cancelled" && (yield* B("failed to observe suspended workflow cancellation", y.gen(function* () {
+				if (C?.("workflowRuns", {
+					op: "update",
+					row: P(U.row)
+				}), U.terminalEvent === void 0) return yield* y.die(/* @__PURE__ */ Error("Cancelled entry has no terminal event"));
+				yield* E(U.terminalEvent), yield* D(j, "cancelled", z);
+			})), yield* O(j, "cancelled", z), yield* y.sync(() => t.recordRun?.({
+				name: t.name,
+				status: "cancelled",
+				durationMs: 0
+			}))), yield* y.interrupt;
+		}
+		let W = U.previous, G = W?.workflowVersion ?? null;
+		if (W?.id !== void 0 && G !== null && t.workflowCompatibleVersions !== void 0 && t.workflowCompatibleVersions !== null && !t.workflowCompatibleVersions.includes(G)) {
+			let e = `Workflow '${t.name}' run '${j}' was started on version ${G}, but current code is compatible with ${t.workflowCompatibleVersions.join(", ")}`, n = t.failureContext ?? {
+				handler: null,
+				callerContext: null,
+				tenantId: null
+			}, r = yield* y.promise(() => t.transaction((r) => h(r, {
+				owner: {
+					runId: z,
+					workflowName: t.name,
+					executionId: j,
+					generation: V
+				},
+				now: Date.now(),
+				durationMs: 0,
+				outcome: {
+					status: "failed",
+					errorTag: "WorkflowVersionIncompatible",
+					errorMessage: e,
+					...n
+				}
+			})));
+			return r.kind === "committed" && (yield* B("failed to observe workflow version rejection", y.gen(function* () {
+				C?.("workflowRuns", {
+					op: "update",
+					row: P(r.row)
+				}), yield* E(r.terminalEvent), yield* D(j, "failed", z);
+			})), yield* O(j, "failed", z, {
+				payload: U.row.payload,
+				errorTag: "WorkflowVersionIncompatible",
+				errorMessage: e
+			}), yield* y.sync(() => t.recordRun?.({
+				name: t.name,
+				status: "failed",
+				durationMs: 0
+			})), m("workflow run failed", {
+				workflow: t.name,
+				executionId: j,
+				runId: z,
+				errorTag: "WorkflowVersionIncompatible",
+				cause: e
+			})), yield* y.fail(Error(e));
+		}
+		let K = t.workflowPatches ?? [], q;
+		if (W?.id !== void 0) {
+			z = W.id, K = k(W.workflowPatches) ?? K;
+			let e = U.row.reclaimCount;
+			if (typeof e != "number" || !Number.isSafeInteger(e) || e < 0) return yield* y.die(/* @__PURE__ */ Error("Recorded workflow entry has an invalid reclaim counter"));
+			let n = e > 0, r = t.maxRunReclaims ?? 3;
+			if (n && e >= r) {
+				let n = `Workflow '${t.name}' run '${j}' killed its runner ${e} times in a row; parked instead of reclaimed again (maxRunReclaims=${r}).`, i = yield* y.promise(() => t.transaction((e) => g(e, {
+					runId: z,
+					workflowName: t.name,
+					executionId: j,
+					generation: V
+				}, {
+					errorTag: "WorkflowCrashLooped",
+					errorMessage: n
+				})));
+				i.kind === "committed" && (yield* B("failed to observe workflow crash-loop park", y.gen(function* () {
+					C?.("workflowRuns", {
+						op: "update",
+						row: P(i.row)
+					}), yield* y.promise(() => s.recordEvent({
+						runId: z,
+						eventType: "run-crashlooped",
+						payload: {
+							reclaimCount: e,
+							maxRunReclaims: r,
+							executionId: j,
+							tag: t.name
+						}
+					}));
+				})), m("workflow run parked after repeated runner deaths", {
+					tag: t.name,
+					executionId: j,
+					runId: z,
+					reclaimCount: e,
+					maxRunReclaims: r
+				}));
+				let a = yield* w;
+				return yield* _.suspend(a);
+			}
+			q = yield* ae(t, z, m), yield* B("failed to record workflow re-entry", y.gen(function* () {
+				C?.("workflowRuns", {
+					op: "update",
+					row: P(U.row)
+				}), n && (yield* y.promise(() => s.recordEvent({
+					runId: z,
+					eventType: "run-resumed",
+					payload: {
+						reason: "runner-reclaim",
+						reclaimCount: e,
+						maxRunReclaims: r
+					}
+				})));
+			}));
+		} else yield* B("failed to emit recorded workflow start", y.gen(function* () {
+			if (C?.("workflowRuns", {
+				op: "insert",
+				row: P(U.row)
+			}), z) {
+				let e = z;
+				yield* y.promise(() => s.recordEvent({
+					runId: e,
+					eventType: "run-started",
+					payload: {
+						tag: t.name,
+						executionId: j,
+						source: t.source ?? null,
+						workflowVersion: t.workflowVersion ?? null,
+						workflowPatches: t.workflowPatches ?? null,
+						parentExecutionId: L,
+						parentClosePolicy: R
+					}
+				}));
+			}
+		}));
+		let J = t.subject?.tenantId, Y = y.suspend(() => n(T, j, V)).pipe(y.provideService(d, z), y.provideService(c, j), y.provideService(l, V), y.provideService(f, typeof J == "string" ? J : void 0), y.provideService(u, K), y.provideService(ee, q), y.provideService(p, s)), ue = t.pluginWorkflowStepLayer ? Y.pipe(y.provide(t.pluginWorkflowStepLayer)) : Y, X = yield* _.intoResult(y.gen(function* () {
+			let e = yield* y.forkChild(N(ue)), n = yield* y.forkChild(N(ie({
+				runId: z,
+				workflowName: t.name,
+				executionId: j,
+				generation: V
+			}, async (e) => t.store.query(e), () => m("workflow control read failed; retrying without claiming cancellation", {
+				executionId: j,
+				runId: z
+			})))), r = yield* y.exit(N(y.raceAll([y.map(x.await(e), (e) => ({
+				other: n,
+				exit: e
+			})), y.map(x.await(n), (t) => ({
+				other: e,
+				exit: t
+			}))])));
+			if (b.isSuccess(r)) {
+				yield* x.interrupt(r.value.other);
+				let e = r.value.exit;
+				if (b.isFailure(e) && v.hasInterruptsOnly(e.cause)) {
+					let e = yield* w;
+					e.suspended || (e.interrupted = !0);
+				}
+				return yield* e;
+			}
+			yield* x.interrupt(n), yield* x.interrupt(e);
+			let i = yield* w;
+			return i.interrupted = !0, yield* x.await(e);
+		}));
+		if (_.isResult(X) && X._tag === "Suspended") {
+			let n = X.cause, i = n !== void 0 && n.reasons.length > 0, a = i ? v.squash(n) : void 0, o = a && typeof a == "object" && "_tag" in a ? String(a._tag) : null, c = i ? e(a) : null, l = yield* y.promise(() => t.transaction((e) => g(e, {
+				runId: z,
+				workflowName: t.name,
+				executionId: j,
+				generation: V
+			}, {
+				errorTag: o,
+				errorMessage: c
+			})));
+			l.kind === "committed" && i && (yield* y.sync(() => S({
+				error: a,
+				source: "workflow",
+				name: t.name,
+				contentSensitive: r(t.name),
+				...t.traceId ? { traceId: t.traceId } : {},
+				fields: {
+					executionId: j,
+					suspendedOnFailure: !0,
+					...o ? { errorTag: o } : {},
+					...z ? { runId: z } : {}
+				}
+			}))), l.kind === "committed" && (yield* B("failed to observe workflow suspend", y.gen(function* () {
+				C?.("workflowRuns", {
+					op: "update",
+					row: P(l.row)
+				}), yield* y.promise(() => s.recordEvent({
+					runId: z,
+					eventType: "run-suspended",
+					payload: i ? {
+						reason: "suspend-on-failure",
+						errorTag: o,
+						errorMessage: c
+					} : { reason: "workflow-suspend" }
+				}));
+			})));
+			let u = yield* w;
+			return yield* _.suspend(u);
+		}
+		let Z = X;
+		if (q !== void 0 && z !== void 0) {
+			let e = q.settle();
+			if (e.length > 0) {
+				let n = z;
+				m("workflow replay reached a different step shape than this run recorded", {
+					tag: t.name,
+					executionId: j,
+					runId: n,
+					findings: e.map((e) => `${e.kind}:${e.stepName}`)
+				}), yield* B("failed to record nondeterminism findings", y.promise(async () => {
+					for (let t of e) await s.recordEvent({
+						runId: n,
+						eventType: "nondeterminism-suspected",
+						payload: te(t)
+					}).catch(() => void 0);
+				}));
+			}
+		}
+		if (b.isFailure(Z.exit)) {
+			let n = Z.exit.cause;
+			if ((yield* w).interrupted && v.hasInterruptsOnly(n)) {
+				let e = Date.now(), r = yield* y.promise(() => t.transaction((n) => h(n, {
+					owner: {
+						runId: z,
+						workflowName: t.name,
+						executionId: j,
+						generation: V
+					},
+					now: e,
+					durationMs: e - F,
+					outcome: { status: "cancelled" }
+				})));
+				return r.kind === "committed" && (yield* B("failed to observe workflow cancellation", y.gen(function* () {
+					C?.("workflowRuns", {
+						op: "update",
+						row: P(r.row)
+					}), yield* E(r.terminalEvent), yield* D(j, "cancelled", z);
+				})), yield* O(j, "cancelled", z), yield* y.sync(() => t.recordRun?.({
+					name: t.name,
+					status: "cancelled",
+					durationMs: e - F
+				}))), yield* y.failCause(n);
+			}
+			let a = v.findErrorOption(n), o = a._tag === "Some" ? a.value : v.squash(n), s = o && typeof o == "object" && "_tag" in o ? String(o._tag) : null, c = e(o), l = Date.now(), u = t.failureContext ?? {
+				handler: null,
+				callerContext: null,
+				tenantId: null
+			}, d = yield* y.promise(() => t.transaction((e) => h(e, {
+				owner: {
+					runId: z,
+					workflowName: t.name,
+					executionId: j,
+					generation: V
+				},
+				now: l,
+				durationMs: l - F,
+				outcome: {
+					status: "failed",
+					errorTag: s,
+					errorMessage: c,
+					...u
+				}
+			})));
+			return d.kind === "committed" && (m("workflow run failed", {
+				workflow: t.name,
+				executionId: j,
+				runId: z,
+				...s ? { errorTag: s } : {},
+				cause: i(t.name, o)
+			}), yield* y.sync(() => S({
+				error: o,
+				source: "workflow",
+				name: t.name,
+				contentSensitive: r(t.name),
+				...t.traceId ? { traceId: t.traceId } : {},
+				fields: {
+					executionId: j,
+					...s ? { errorTag: s } : {},
+					...z ? { runId: z } : {}
+				}
+			})), z && (yield* B("failed to record workflow failure", y.gen(function* () {
+				C?.("workflowRuns", {
+					op: "update",
+					row: P(d.row)
+				}), yield* E(d.terminalEvent), yield* D(j, "failed", z);
+			}))), yield* O(j, "failed", z, {
+				payload: T,
+				errorTag: s,
+				errorMessage: c
+			}), yield* y.sync(() => t.recordRun?.({
+				name: t.name,
+				status: "failed",
+				durationMs: Date.now() - F
+			}))), yield* y.failCause(n);
+		}
+		let Q = Z.exit.value, de = Date.now(), $ = yield* y.promise(() => t.transaction((e) => h(e, {
+			owner: {
+				runId: z,
+				workflowName: t.name,
+				executionId: j,
+				generation: V
+			},
+			now: de,
+			durationMs: de - F,
+			outcome: {
+				status: "succeeded",
+				output: M(Q, a)
+			}
+		})));
+		return $.kind === "committed" && (z && (yield* B("failed to record workflow success", y.gen(function* () {
+			C?.("workflowRuns", {
+				op: "update",
+				row: P($.row)
+			}), yield* E($.terminalEvent), yield* D(j, "succeeded", z);
+		}))), yield* O(j, "succeeded", z), yield* y.sync(() => t.recordRun?.({
+			name: t.name,
+			status: "succeeded",
+			durationMs: Date.now() - F,
+			completedAtSec: Date.now() / 1e3
+		}))), Q;
+	}));
+};
+//#endregion
+export { E as DEFAULT_MAX_RUN_RECLAIMS, D as DEFAULT_REPLAY_SHAPE_LIMIT, F as makeWorkflowRunRecorder, k as normaliseWorkflowPatches, O as resolveRunGuardTuning, P as serialiseWorkflowRowForWire, M as truncateWorkflowValue, I as wrapWorkflowExecuteWithRunRecording };

package/dist/runStepsTable.js

@@ -0,0 +1,28 @@
+import { _voltroWorkflowRunsTable as e } from "./runsTable.js";
+import { id as t, integer as n, json as r, reference as i, table as a, text as o, timestamp as s } from "@voltro/database";
+//#region src/runStepsTable.ts
+var c = a("_voltro_workflow_run_steps", {
+	id: t({ prefix: "wfstep" }),
+	runId: i(() => e, {
+		index: !1,
+		onDelete: "cascade"
+	}),
+	stepName: o(),
+	attempt: n().default(1),
+	status: o().oneOf([
+		"running",
+		"succeeded",
+		"failed"
+	]).default("running"),
+	input: r().nullable(),
+	retryPolicy: r().nullable(),
+	output: r().nullable(),
+	errorTag: o().nullable(),
+	errorMessage: o().nullable(),
+	errorCause: r().nullable(),
+	startedAt: s().default("now"),
+	completedAt: s().nullable(),
+	durationMs: n().nullable()
+}).index(["runId", "stepName"]).index(["startedAt"]);
+//#endregion
+export { c as _voltroWorkflowRunStepsTable };

package/dist/runnerStorageGuard.js

@@ -0,0 +1,29 @@
+import { Duration as e, Effect as t, Schedule as n } from "effect";
+import { RunnerStorage as r } from "effect/cluster";
+import { PersistenceError as i } from "effect/cluster/ClusterError";
+//#region src/runnerStorageGuard.ts
+var a = (a, o) => t.gen(function* () {
+	let s = e.fromInputUnsafe(o.interval), c = e.toMillis(o.giveUpAfter), l, u, d = !1, f = !1;
+	yield* o.probe.pipe(t.timeout(s), t.isSuccess, t.flatMap((e) => t.sync(() => {
+		let t = Date.now();
+		if (e) {
+			if (l = void 0, !d || (u ??= t, t - u < c)) return;
+			o.log.info("workflow runner: its storage answers again — taking part in shard assignment again"), u = void 0, d = !1, f = !1;
+		} else u = void 0, l ??= t, !d && t - l >= c && (d = !0, o.log.warn(`workflow runner: its connection pool has not served a query for ${Math.round((t - l) / 1e3)}s — released its shards and stopped its heartbeat, so another runner takes them`, { fix: "look for what holds this process's database connections (`voltro_db_operations_in_flight` against DB_MAX_CONNECTIONS)" }));
+	}))).pipe(t.repeat(n.spaced(s)), t.forkScoped);
+	let p = () => new i({ cause: /* @__PURE__ */ Error("workflow runner gave its shards up: its connection pool does not serve queries") });
+	return r.RunnerStorage.of({
+		...a,
+		register: (e, n) => t.suspend(() => d ? t.fail(p()) : a.register(e, n)),
+		acquire: (e, n) => t.suspend(() => d ? t.succeed([]) : a.acquire(e, n)),
+		refresh: (e, n) => t.suspend(() => {
+			if (!d) return a.refresh(e, n);
+			let r = f ? t.void : a.releaseAll(e).pipe(t.tap(() => t.sync(() => {
+				f = !0;
+			})), t.ignore);
+			return t.andThen(r, t.fail(p()));
+		})
+	});
+});
+//#endregion
+export { a as guardRunnerStorage };