@voltro/workflow 0.83.0 → 0.83.1

package/dist/{cluster-NnChVZoK.js → cluster--WvQZFwk.js}

@@ -1,13 +1,14 @@
 import { _ as e, a as t, b as n, c as r, l as i, m as a, n as o, p as s, s as c, t as l, x as u } from "./clusterDiagnostics-CGUtnJkX.js";
-import { Context as d, Duration as f, Effect as p, Layer as m, Option as h, Runtime as g, Schema as _ } from "effect";
-import { eq as v } from "@voltro/database";
-import { SqlClient as y } from "@effect/sql";
-import { ClusterCron as ee, ClusterSchema as te, ClusterWorkflowEngine as b, EntityAddress as ne, EntityId as re, EntityType as ie, MessageStorage as x, RunnerAddress as S, ShardId as ae, Sharding as C, ShardingConfig as oe, SingleRunner as se, SqlMessageStorage as ce, SqlRunnerStorage as le } from "@effect/cluster";
-import { applyUndeclaredCharsetWork as ue, fnv1a64 as w, readUndeclaredCharsetColumns as de, resolveMigrationLockSchema as T, sqlPoolCapacity as fe, undeclaredCharsetWork as pe } from "@voltro/database/sql";
+import { Context as d, Duration as f, Effect as p, Layer as m, Option as h, Runtime as g, Schedule as _, Schema as v } from "effect";
+import { eq as y } from "@voltro/database";
+import { SqlClient as b } from "@effect/sql";
+import { ClusterCron as ee, ClusterSchema as te, ClusterWorkflowEngine as x, EntityAddress as ne, EntityId as re, EntityType as ie, MessageStorage as S, RunnerAddress as C, RunnerHealth as ae, RunnerStorage as w, Runners as oe, ShardId as se, Sharding as T, ShardingConfig as ce, SingleRunner as le, SqlMessageStorage as E, SqlRunnerStorage as D } from "@effect/cluster";
+import { applyUndeclaredCharsetWork as ue, fnv1a64 as O, readUndeclaredCharsetColumns as de, resolveMigrationLockSchema as k, sqlPoolCapacity as fe, undeclaredCharsetWork as pe } from "@voltro/database/sql";
 import { realpath as me } from "node:fs/promises";
 import { createLogger as he } from "@voltro/logger";
+import { PersistenceError as ge } from "@effect/cluster/ClusterError";
 //#region src/workflowResumeDispatch.ts
-var ge = (e) => p.gen(function* () {
+var _e = (e) => p.gen(function* () {
 	let t = yield* p.runtime(), r = yield* p.promise(() => e.transaction((n) => u(n, e.requestId, (e) => g.runPromise(t)(s(e)))));
 	if (r.kind !== "bound") return r;
 	let i = yield* p.promise(() => e.transaction((t) => n(t, e.requestId)));
@@ -19,15 +20,15 @@ var ge = (e) => p.gen(function* () {
 		executionId: o.executionId,
 		...o.resumeJournal
 	});
-	return yield* (yield* C.Sharding).pollStorage, {
+	return yield* (yield* T.Sharding).pollStorage, {
 		kind: "attempted",
 		request: o,
 		journal: c
 	};
-}), E = class extends d.Tag("@voltro/workflow/WorkflowResumeDispatcher")() {}, _e = p.gen(function* () {
-	let e = yield* y.SqlClient, t = yield* C.Sharding;
-	return E.of({ dispatch: (n) => ge(n).pipe(p.provideService(y.SqlClient, e), p.provideService(C.Sharding, t)) });
-}), ve = async (e) => {
+}), A = class extends d.Tag("@voltro/workflow/WorkflowResumeDispatcher")() {}, ve = p.gen(function* () {
+	let e = yield* b.SqlClient, t = yield* T.Sharding;
+	return A.of({ dispatch: (n) => _e(n).pipe(p.provideService(b.SqlClient, e), p.provideService(T.Sharding, t)) });
+}), ye = async (e) => {
 	if (e.workflow.name !== e.command.workflowName || e.control.workflowName !== e.command.workflowName) throw Error("Restart preparation requires the matching workflow");
 	let n = await e.transaction(async (n) => {
 		let r = await t(n, e.command);
@@ -37,7 +38,7 @@ var ge = (e) => p.gen(function* () {
 		};
 		let [i] = await n.query({
 			table: "_voltro_workflow_runs",
-			predicate: v("id", e.command.runId),
+			predicate: y("id", e.command.runId),
 			lock: "update",
 			order: [],
 			take: 1,
@@ -46,24 +47,24 @@ var ge = (e) => p.gen(function* () {
 		});
 		return {
 			kind: "captured",
-			generation: _.decodeUnknownSync(_.NonEmptyString)(i?.runnerGeneration)
+			generation: v.decodeUnknownSync(v.NonEmptyString)(i?.runnerGeneration)
 		};
 	});
 	return n.kind === "accepted" ? {
 		kind: "reserved",
 		request: n.request,
 		reused: !0
-	} : be({
+	} : xe({
 		...e,
 		command: {
 			...e.command,
 			generation: n.generation
 		}
 	});
-}, ye = _.Struct({
-	requestId: _.NonEmptyString,
-	restartToken: _.NullOr(_.NonEmptyString)
-}), be = async (e) => {
+}, be = v.Struct({
+	requestId: v.NonEmptyString,
+	restartToken: v.NullOr(v.NonEmptyString)
+}), xe = async (e) => {
 	let { command: t } = e;
 	if (e.workflow.name !== t.workflowName || e.control.workflowName !== t.workflowName) throw Error("Restart preparation requires the matching workflow");
 	let n = await e.transaction(async (n) => {
@@ -74,7 +75,7 @@ var ge = (e) => p.gen(function* () {
 		};
 		let [i] = await n.query({
 			table: "_voltro_workflow_runs",
-			predicate: v("id", t.runId),
+			predicate: y("id", t.runId),
 			lock: "update",
 			order: [],
 			take: 1,
@@ -82,11 +83,11 @@ var ge = (e) => p.gen(function* () {
 			projection: void 0
 		});
 		if (i === void 0 || i.tag !== t.workflowName || i.runnerGeneration !== t.generation || i.status !== "failed" || i.discardedAt != null || typeof i.executionId != "string" || i.executionId.length === 0) throw Error("Restart preparation requires its failed resident generation");
-		if ((_.decodeUnknownSync(_.NullOr(_.Struct({ tenantId: _.NullOr(_.NonEmptyString) })))(i.subject ?? null)?.tenantId ?? null) !== t.tenantId) throw Error("Restart preparation belongs to another tenant");
+		if ((v.decodeUnknownSync(v.NullOr(v.Struct({ tenantId: v.NullOr(v.NonEmptyString) })))(i.subject ?? null)?.tenantId ?? null) !== t.tenantId) throw Error("Restart preparation belongs to another tenant");
 		return {
 			kind: "observe",
 			executionId: i.executionId,
-			delivery: _.decodeUnknownSync(ye)(i.engineDelivery),
+			delivery: v.decodeUnknownSync(be)(i.engineDelivery),
 			activities: await e.readActivities(n)
 		};
 	});
@@ -111,7 +112,7 @@ var ge = (e) => p.gen(function* () {
 		if (i.kind === "unavailable") return i;
 		let [s] = await a.query({
 			table: "_voltro_workflow_runs",
-			predicate: v("id", t.runId),
+			predicate: y("id", t.runId),
 			lock: "update",
 			order: [],
 			take: 1,
@@ -129,16 +130,16 @@ var ge = (e) => p.gen(function* () {
 			now: e.now
 		});
 	});
-}, D = (e, t) => `${e}/${t}`, O = (e, t, n) => {
+}, j = (e, t) => `${e}/${t}`, M = (e, t, n) => {
 	let r = re.make(t), i = d.get(e.annotations, te.ShardGroup)(r);
 	return new ne.EntityAddress({
 		entityType: ie.EntityType.make(`Workflow/${e.name}`),
 		entityId: r,
 		shardId: n.getShardId(r, i)
 	});
-}, xe = (e, t) => p.gen(function* () {
-	let n = yield* C.Sharding, r = yield* x.MessageStorage, i = yield* r.requestIdForPrimaryKey({
-		address: O(e, t, n),
+}, Se = (e, t) => p.gen(function* () {
+	let n = yield* T.Sharding, r = yield* S.MessageStorage, i = yield* r.requestIdForPrimaryKey({
+		address: M(e, t, n),
 		tag: "run",
 		id: ""
 	});
@@ -153,23 +154,23 @@ var ge = (e) => p.gen(function* () {
 		replyId: String(o.id),
 		exit: o.exit
 	};
-}), k = _.decodeUnknownOption(_.Exit({
-	success: _.Unknown,
-	failure: _.Unknown,
-	defect: _.Unknown
-})), A = (e) => {
-	let t = (e) => e && typeof e == "object" && "_tag" in e && typeof e._tag == "string" ? e._tag : void 0, n = k(e);
+}), N = v.decodeUnknownOption(v.Exit({
+	success: v.Unknown,
+	failure: v.Unknown,
+	defect: v.Unknown
+})), P = (e) => {
+	let t = (e) => e && typeof e == "object" && "_tag" in e && typeof e._tag == "string" ? e._tag : void 0, n = N(e);
 	if (h.isNone(n)) return "unrecognized-reply";
 	if (n.value._tag === "Failure") return "failed";
 	let r = n.value.value, i = t(r);
 	if (i === "Suspended") return "suspended";
 	if (i === "Complete") {
-		let e = r.exit, t = k(e);
+		let e = r.exit, t = N(e);
 		return h.isNone(t) ? "unrecognized-reply" : t.value._tag === "Success" ? "succeeded" : "failed";
 	}
 	return "unrecognized-reply";
-}, Se = (e) => M(e.workflow, e.executionId, e.failedActivities), j = (e) => M(e.workflow, e.executionId, e.resetActivities), M = (e, t, n) => p.gen(function* () {
-	let r = yield* C.Sharding, i = yield* x.MessageStorage, a = O(e, t, r), o = yield* i.requestIdForPrimaryKey({
+}, Ce = (e) => F(e.workflow, e.executionId, e.failedActivities), we = (e) => F(e.workflow, e.executionId, e.resetActivities), F = (e, t, n) => p.gen(function* () {
+	let r = yield* T.Sharding, i = yield* S.MessageStorage, a = M(e, t, r), o = yield* i.requestIdForPrimaryKey({
 		address: a,
 		tag: "run",
 		id: ""
@@ -187,7 +188,7 @@ var ge = (e) => p.gen(function* () {
 		activitiesReset: 0,
 		reason: "no-reply"
 	};
-	let c = A(s.exit);
+	let c = P(s.exit);
 	if (c !== "failed") return {
 		redriven: !1,
 		runReplyCleared: !1,
@@ -199,7 +200,7 @@ var ge = (e) => p.gen(function* () {
 		let t = yield* i.requestIdForPrimaryKey({
 			address: a,
 			tag: "activity",
-			id: D(e.name, e.attempt)
+			id: j(e.name, e.attempt)
 		}).pipe(p.orDie);
 		if (h.isNone(t)) return {
 			redriven: !1,
@@ -231,12 +232,12 @@ var ge = (e) => p.gen(function* () {
 		activitiesReset: u,
 		reason: "run-reset-unconfirmed"
 	};
-}), Ce = _.parseJson(_.Record({
-	key: _.String,
-	value: _.String
-})), we = (e) => p.gen(function* () {
+}), Te = v.parseJson(v.Record({
+	key: v.String,
+	value: v.String
+})), Ee = (e) => p.gen(function* () {
 	if (!/^[1-9]\d*$/.test(e.delivery.requestId) || e.executionId.length === 0 || e.activities.some((e) => e.name.length === 0 || !Number.isSafeInteger(e.attempt) || e.attempt < 1)) return yield* p.die(/* @__PURE__ */ Error("Restart observation requires an exact delivery and valid activity attempts"));
-	let t = (yield* y.SqlClient).withoutTransforms(), n = yield* x.MessageStorage, r = yield* C.Sharding, i = O(e.workflow, e.executionId, r), a = (n, r) => p.gen(function* () {
+	let t = (yield* b.SqlClient).withoutTransforms(), n = yield* S.MessageStorage, r = yield* T.Sharding, i = M(e.workflow, e.executionId, r), a = (n, r) => p.gen(function* () {
 		let [i] = yield* t`SELECT m.entity_type, m.entity_id, m.tag, m.headers, r.id AS reply_id, r.payload
        FROM cluster_messages m
        LEFT JOIN cluster_replies r ON r.id = m.last_reply_id AND r.request_id = m.id AND r.kind = 0
@@ -247,7 +248,7 @@ var ge = (e) => p.gen(function* () {
 		kind: "unavailable",
 		reason: "no-journal"
 	};
-	if (((yield* p.sync(() => _.decodeUnknownSync(Ce)(o.headers)))["voltro-workflow-restart"] ?? null) !== e.delivery.restartToken) return {
+	if (((yield* p.sync(() => v.decodeUnknownSync(Te)(o.headers)))["voltro-workflow-restart"] ?? null) !== e.delivery.restartToken) return {
 		kind: "unavailable",
 		reason: "delivery-changed"
 	};
@@ -255,7 +256,7 @@ var ge = (e) => p.gen(function* () {
 		kind: "unavailable",
 		reason: "no-reply"
 	};
-	let s = A(yield* p.sync(() => JSON.parse(o.payload)));
+	let s = P(yield* p.sync(() => JSON.parse(o.payload)));
 	if (s !== "failed") return {
 		kind: "unavailable",
 		reason: s
@@ -265,7 +266,7 @@ var ge = (e) => p.gen(function* () {
 		let e = yield* n.requestIdForPrimaryKey({
 			address: i,
 			tag: "activity",
-			id: D(t.name, t.attempt)
+			id: j(t.name, t.attempt)
 		});
 		if (h.isNone(e)) return {
 			kind: "unavailable",
@@ -285,59 +286,59 @@ var ge = (e) => p.gen(function* () {
 			replyId: t
 		}))
 	};
-}).pipe(p.provideService(y.SafeIntegers, !0)), Te = (e) => p.gen(function* () {
+}).pipe(p.provideService(b.SafeIntegers, !0)), De = (e) => p.gen(function* () {
 	let t = yield* i(e);
-	return t.kind === "waiting-capacity" ? t : (yield* (yield* C.Sharding).pollStorage, {
+	return t.kind === "waiting-capacity" ? t : (yield* (yield* T.Sharding).pollStorage, {
 		kind: "attempted",
 		journal: t
 	});
-}), N = class extends d.Tag("@voltro/workflow/WorkflowRestartDispatcher")() {}, Ee = p.gen(function* () {
-	let e = yield* y.SqlClient, t = yield* C.Sharding, n = yield* x.MessageStorage, r = (r) => we(r).pipe(p.provideService(y.SqlClient, e), p.provideService(C.Sharding, t), p.provideService(x.MessageStorage, n));
-	return N.of({
+}), I = class extends d.Tag("@voltro/workflow/WorkflowRestartDispatcher")() {}, Oe = p.gen(function* () {
+	let e = yield* b.SqlClient, t = yield* T.Sharding, n = yield* S.MessageStorage, r = (r) => Ee(r).pipe(p.provideService(b.SqlClient, e), p.provideService(T.Sharding, t), p.provideService(S.MessageStorage, n));
+	return I.of({
 		prepare: (e) => p.tryPromise({
-			try: (t) => ve({
+			try: (t) => ye({
 				...e,
 				observe: (e) => p.runPromise(r(e), { signal: t })
 			}),
 			catch: (e) => e
 		}),
 		observe: r,
-		dispatch: (n) => Te(n).pipe(p.provideService(y.SqlClient, e), p.provideService(C.Sharding, t))
+		dispatch: (n) => De(n).pipe(p.provideService(b.SqlClient, e), p.provideService(T.Sharding, t))
 	});
-}), P = 6322741009312601n, F = () => {
-	let e = T();
-	return e === void 0 ? P : w(`voltro_cluster_migration_lock:${e}`) & 9223372036854775807n;
-}, I = () => {
-	let e = `voltro_cluster_migration_${P.toString(36)}`, t = T();
+}), L = 6322741009312601n, R = () => {
+	let e = k();
+	return e === void 0 ? L : O(`voltro_cluster_migration_lock:${e}`) & 9223372036854775807n;
+}, z = () => {
+	let e = `voltro_cluster_migration_${L.toString(36)}`, t = k();
 	if (t === void 0) return e;
 	let n = `${e}_${t}`;
-	return /^[A-Za-z0-9_]+$/.test(t) && n.length <= 64 ? n : `${e}_h${w(`voltro_cluster_migration_lock:${t}`).toString(16)}`;
-}, L = () => {
+	return /^[A-Za-z0-9_]+$/.test(t) && n.length <= 64 ? n : `${e}_h${O(`voltro_cluster_migration_lock:${t}`).toString(16)}`;
+}, B = () => {
 	let e = Number(process.env.VOLTRO_CLUSTER_MIGRATION_LOCK_TIMEOUT_MS);
 	return Number.isFinite(e) && e > 0 ? e : 6e4;
-}, De = 250, R = (e) => e[0]?.[0], Oe = (e, t) => e.onDialectOrElse({
+}, ke = 250, V = (e) => e[0]?.[0], Ae = (e, t) => e.onDialectOrElse({
 	sqlite: () => p.void,
-	mysql: () => t.executeValues(`SELECT GET_LOCK('${I()}', ${Math.ceil(L() / 1e3)})`, []).pipe(p.flatMap((e) => R(e) === 1 || R(e) === "1" ? p.void : p.die(/* @__PURE__ */ Error(`could not acquire cluster migration lock '${I()}' (GET_LOCK timed out)`)))),
-	mssql: () => t.executeValues(`DECLARE @res int; EXEC @res = sp_getapplock @Resource = '${I()}', @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = ${L()}; SELECT @res`, []).pipe(p.flatMap((e) => Number(R(e) ?? -999) >= 0 ? p.void : p.die(/* @__PURE__ */ Error(`could not acquire cluster migration lock '${I()}' (sp_getapplock ${String(R(e))})`)))),
+	mysql: () => t.executeValues(`SELECT GET_LOCK('${z()}', ${Math.ceil(B() / 1e3)})`, []).pipe(p.flatMap((e) => V(e) === 1 || V(e) === "1" ? p.void : p.die(/* @__PURE__ */ Error(`could not acquire cluster migration lock '${z()}' (GET_LOCK timed out)`)))),
+	mssql: () => t.executeValues(`DECLARE @res int; EXEC @res = sp_getapplock @Resource = '${z()}', @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = ${B()}; SELECT @res`, []).pipe(p.flatMap((e) => Number(V(e) ?? -999) >= 0 ? p.void : p.die(/* @__PURE__ */ Error(`could not acquire cluster migration lock '${z()}' (sp_getapplock ${String(V(e))})`)))),
 	orElse: () => {
-		let e = F().toString(), n = Date.now(), r = t.executeValues(`SELECT pg_try_advisory_lock(${e}::bigint)`, []).pipe(p.flatMap((e) => {
-			let t = R(e);
-			return t === !0 || t === "t" || t === 1 ? p.void : Date.now() - n >= L() ? p.die(/* @__PURE__ */ Error(`could not acquire the cluster migration advisory lock within ${Math.round(L() / 1e3)}s. Another runner is migrating the cluster schema, or a prior boot wedged while holding it. (Raise VOLTRO_CLUSTER_MIGRATION_LOCK_TIMEOUT_MS, or inspect pg_locks for the holder.)`)) : p.sleep(`${De} millis`).pipe(p.zipRight(r));
+		let e = R().toString(), n = Date.now(), r = t.executeValues(`SELECT pg_try_advisory_lock(${e}::bigint)`, []).pipe(p.flatMap((e) => {
+			let t = V(e);
+			return t === !0 || t === "t" || t === 1 ? p.void : Date.now() - n >= B() ? p.die(/* @__PURE__ */ Error(`could not acquire the cluster migration advisory lock within ${Math.round(B() / 1e3)}s. Another runner is migrating the cluster schema, or a prior boot wedged while holding it. (Raise VOLTRO_CLUSTER_MIGRATION_LOCK_TIMEOUT_MS, or inspect pg_locks for the holder.)`)) : p.sleep(`${ke} millis`).pipe(p.zipRight(r));
 		}));
 		return r;
 	}
-}), z = (e, t) => e.onDialectOrElse({
+}), je = (e, t) => e.onDialectOrElse({
 	sqlite: () => p.void,
-	mysql: () => p.asVoid(t.executeValues(`SELECT RELEASE_LOCK('${I()}')`, [])),
-	mssql: () => p.asVoid(t.executeValues(`EXEC sp_releaseapplock @Resource = '${I()}', @LockOwner = 'Session'`, [])),
-	orElse: () => p.asVoid(t.executeValues(`SELECT pg_advisory_unlock(${F().toString()}::bigint)`, []))
-}), ke = (e, t) => e.onDialectOrElse({
+	mysql: () => p.asVoid(t.executeValues(`SELECT RELEASE_LOCK('${z()}')`, [])),
+	mssql: () => p.asVoid(t.executeValues(`EXEC sp_releaseapplock @Resource = '${z()}', @LockOwner = 'Session'`, [])),
+	orElse: () => p.asVoid(t.executeValues(`SELECT pg_advisory_unlock(${R().toString()}::bigint)`, []))
+}), Me = (e, t) => e.onDialectOrElse({
 	sqlite: () => !0,
 	orElse: () => !1
 }) ? t : p.scoped(p.gen(function* () {
 	let n = yield* e.reserve;
-	return yield* p.acquireRelease(Oe(e, n), () => z(e, n).pipe(p.orDie)), yield* t;
-})), Ae = (e) => e === "sql" ? ["cluster_messages", "cluster_runners"] : ["cluster_messages"], je = (e, t) => e.onDialectOrElse({
+	return yield* p.acquireRelease(Ae(e, n), () => je(e, n).pipe(p.orDie)), yield* t;
+})), Ne = (e) => e === "sql" ? ["cluster_messages", "cluster_runners"] : ["cluster_messages"], Pe = (e, t) => e.onDialectOrElse({
 	pg: () => e`SELECT (to_regclass(${t}) IS NOT NULL) AS present`,
 	mysql: () => e`SELECT (COUNT(*) > 0) AS present FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ${t}`,
 	mssql: () => e`SELECT CASE WHEN OBJECT_ID(${t}, 'U') IS NULL THEN 0 ELSE 1 END AS present`,
@@ -345,63 +346,63 @@ var ge = (e) => p.gen(function* () {
 }).pipe(p.map((e) => {
 	let t = e[0]?.present;
 	return t === !0 || t === 1 || t === "1" || t === "t" || typeof t == "bigint" && t === 1n;
-}), p.catchAll(() => p.succeed(!1))), Me = (e, t) => p.forEach(Ae(t), (t) => je(e, t), { concurrency: 1 }).pipe(p.map((e) => e.every(Boolean))), B = (e, t) => p.scoped(p.asVoid(m.build(t.pipe(m.provide(oe.layerDefaults), m.provide(m.succeed(y.SqlClient, e)))))), V = (e, t) => B(e, ce.layer).pipe(p.zipRight(t === "sql" ? B(e, le.layer) : p.void)), H = (e) => p.map(e.unsafe("SELECT table_name AS t FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name LIKE 'cluster\\_%'"), (e) => new Set(e.map((e) => String(e.t ?? e.T)))), Ne = (e, t, n) => n === void 0 || !e.onDialectOrElse({
+}), p.catchAll(() => p.succeed(!1))), Fe = (e, t) => p.forEach(Ne(t), (t) => Pe(e, t), { concurrency: 1 }).pipe(p.map((e) => e.every(Boolean))), H = (e, t) => p.scoped(p.asVoid(m.build(t.pipe(m.provide(ce.layerDefaults), m.provide(m.succeed(b.SqlClient, e)))))), U = (e, t) => H(e, E.layer).pipe(p.zipRight(t === "sql" ? H(e, D.layer) : p.void)), W = (e) => p.map(e.unsafe("SELECT table_name AS t FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name LIKE 'cluster\\_%'"), (e) => new Set(e.map((e) => String(e.t ?? e.T)))), Ie = (e, t, n) => n === void 0 || !e.onDialectOrElse({
 	mysql: () => !0,
 	orElse: () => !1
-}) ? V(e, t) : p.gen(function* () {
-	let r = yield* H(e);
-	yield* V(e, t);
-	let i = new Set([...yield* H(e)].filter((e) => !r.has(e)));
+}) ? U(e, t) : p.gen(function* () {
+	let r = yield* W(e);
+	yield* U(e, t);
+	let i = new Set([...yield* W(e)].filter((e) => !r.has(e)));
 	i.size !== 0 && (yield* ue(e, pe(yield* de(e), /* @__PURE__ */ new Set(), n, i)));
-}), Pe = (e) => e === "sql" ? 3 : 2, U = (e) => m.effect(y.SqlClient, p.gen(function* () {
-	let t = yield* y.SqlClient, n = Pe(e.runnerStorage), r = fe(t);
-	return r !== void 0 && r < n ? yield* p.dieMessage(`the workflow runtime needs ${n} database connections at once and the pool has ${r}: ` + (e.runnerStorage === "sql" ? "on a first boot it holds its migration lock and its runner lock while the migration runs on a third, and once running it keeps the runner lock and needs one more to work. " : "on a first boot it holds its migration lock while the migration runs on a second. ") + `Set DB_MAX_CONNECTIONS (or PG_MAX_CONNECTIONS) to at least ${n}` + (e.runnerStorage === "sql" ? ", or VOLTRO_WORKFLOW_RUNNER_STORAGE=memory for a single-process runtime that needs 2 (no cross-pod shard handoff)." : ".")) : ((yield* Me(t, e.runnerStorage)) || (yield* ke(t, Ne(t, e.runnerStorage, e.charset))), t);
-})), Fe = (e) => p.gen(function* () {
+}), Le = (e) => e === "sql" ? 3 : 2, G = (e) => m.effect(b.SqlClient, p.gen(function* () {
+	let t = yield* b.SqlClient, n = Le(e.runnerStorage), r = fe(t);
+	return r !== void 0 && r < n ? yield* p.dieMessage(`the workflow runtime needs ${n} database connections at once and the pool has ${r}: ` + (e.runnerStorage === "sql" ? "on a first boot it holds its migration lock and its runner lock while the migration runs on a third, and once running it keeps the runner lock and needs one more to work. " : "on a first boot it holds its migration lock while the migration runs on a second. ") + `Set DB_MAX_CONNECTIONS (or PG_MAX_CONNECTIONS) to at least ${n}` + (e.runnerStorage === "sql" ? ", or VOLTRO_WORKFLOW_RUNNER_STORAGE=memory for a single-process runtime that needs 2 (no cross-pod shard handoff)." : ".")) : ((yield* Fe(t, e.runnerStorage)) || (yield* Me(t, Ie(t, e.runnerStorage, e.charset))), t);
+})), Re = (e) => p.gen(function* () {
 	let t = yield* p.tryPromise({
 		try: () => me(e),
-		catch: W
+		catch: K
 	}), n = yield* p.acquireRelease(p.tryPromise({
 		try: async () => {
 			let { DatabaseSync: e } = await import("node:sqlite");
 			return new e(`${t}.voltro-workflow-lock`);
 		},
-		catch: W
+		catch: K
 	}), (e) => p.sync(() => e.close()));
 	yield* p.try({
 		try: () => {
 			n.exec("PRAGMA busy_timeout=0"), n.exec("CREATE TABLE IF NOT EXISTS workflow_mutex(id INTEGER PRIMARY KEY)"), n.exec("BEGIN IMMEDIATE");
 		},
 		catch: (e) => {
-			let n = W(e), r = n.errcode;
+			let n = K(e), r = n.errcode;
 			return r !== void 0 && ((r & 255) == 5 || (r & 255) == 6) ? Error("SQLite workflow journal is already owned by another engine. Stop that engine before starting another, or use a SQL dialect with shared runner coordination for replicas.", { cause: n }) : Error(`Could not acquire SQLite workflow ownership for ${t}`, { cause: n });
 		}
 	});
-}), W = (e) => e instanceof Error ? e : Error(String(e)), G = (e) => {
+}), K = (e) => e instanceof Error ? e : Error(String(e)), q = (e) => {
 	if (e instanceof Date) return Number.isNaN(e.getTime()) ? null : e;
 	if (typeof e == "string" || typeof e == "number") {
 		let t = typeof e == "string" && /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(e) ? `${e.replace(" ", "T")}Z` : e, n = new Date(t);
 		return Number.isNaN(n.getTime()) ? null : n;
 	}
 	return null;
-}, K = (e) => e === !0 || e === 1 || e === "1" || e instanceof Uint8Array && e[0] === 1, Ie = (e, t) => {
+}, J = (e) => e === !0 || e === 1 || e === "1" || e instanceof Uint8Array && e[0] === 1, ze = (e, t) => {
 	let n = e.lastIndexOf(":");
 	if (n < 0) return;
 	let r = e.slice(0, n), i = Number(e.slice(n + 1)), a = t.indexOf(r);
 	if (!(a < 0 || !Number.isInteger(i))) return (a + 1) * 1e6 + i;
-}, q = (e) => p.gen(function* () {
-	let t = yield* y.SqlClient, n = e.shardGroups ?? ["default"], r = e.dialectId ?? "", i = r.startsWith("postgres") || r === "pg", a = (e) => e.pipe(p.catchAll(() => p.succeed([]))), o = t.onDialectOrElse({
+}, Y = (e) => p.gen(function* () {
+	let t = yield* b.SqlClient, n = e.shardGroups ?? ["default"], r = e.dialectId ?? "", i = r.startsWith("postgres") || r === "pg", a = (e) => e.pipe(p.catchAll(() => p.succeed([]))), o = t.onDialectOrElse({
 		pg: () => t.literal("NOW()"),
 		mysql: () => t.literal("NOW()"),
 		mssql: () => t.literal("GETDATE()"),
 		orElse: () => t.literal("CURRENT_TIMESTAMP")
 	}), s = (e, t) => {
-		let n = G(e.last_read), r = G(e.server_now), i = n !== null && r !== null ? Math.max(0, r.getTime() - n.getTime()) : null;
+		let n = q(e.last_read), r = q(e.server_now), i = n !== null && r !== null ? Math.max(0, r.getTime() - n.getTime()) : null;
 		return {
 			tag: typeof e.tag == "string" ? e.tag : null,
 			entityType: String(e.entity_type ?? ""),
 			entityId: String(e.entity_id ?? ""),
 			shardId: String(e.shard_id ?? ""),
-			processed: K(e.processed),
+			processed: J(e.processed),
 			lastRead: i === null ? null : new Date(t - i)
 		};
 	}, c = (e) => t.onDialectOrElse({
@@ -446,12 +447,12 @@ var ge = (e) => p.gen(function* () {
 			mode: "row",
 			holder: r,
 			own: r === e.ownAddress || (e.hasShard?.(i) ?? !1),
-			since: G(n.acquired_at)
+			since: q(n.acquired_at)
 		};
 	}))), h = (r) => {
 		let i = /* @__PURE__ */ new Map();
 		for (let e of r) {
-			let t = Ie(e, n);
+			let t = ze(e, n);
 			t !== void 0 && i.set(t, e);
 		}
 		return i.size === 0 ? p.succeed([]) : p.gen(function* () {
@@ -469,14 +470,14 @@ var ge = (e) => p.gen(function* () {
 					mode: "advisory",
 					holder: s ? `this runner (pid ${a})` : `pid ${a} from ${o}`,
 					own: s,
-					since: G(n.state_change),
+					since: q(n.state_change),
 					detail: {
 						pid: a,
 						clientAddr: n.client_addr ?? null,
 						applicationName: n.application_name ?? null,
 						state: n.state ?? null,
 						waitEventType: n.wait_event_type ?? null,
-						backendStart: G(n.backend_start)?.toISOString() ?? null
+						backendStart: q(n.backend_start)?.toISOString() ?? null
 					}
 				}];
 			});
@@ -492,8 +493,8 @@ var ge = (e) => p.gen(function* () {
 			let t = e;
 			return {
 				address: String(t.address ?? ""),
-				lastHeartbeat: G(t.last_heartbeat),
-				healthy: K(t.healthy)
+				lastHeartbeat: q(t.last_heartbeat),
+				healthy: J(t.healthy)
 			};
 		}))) : p.succeed([]),
 		discardMessages: (e) => {
@@ -509,38 +510,62 @@ var ge = (e) => p.gen(function* () {
 			}).pipe(t.withTransaction, p.catchAll(() => p.succeed(0)));
 		}
 	};
-}), J = he({ scope: "voltro:workflow" }), Le = 34e3, Y = (e) => {
+}), Be = (e, t) => p.gen(function* () {
+	let n = f.decode(t.interval), r = f.toMillis(t.giveUpAfter), i, a, o = !1, s = !1;
+	yield* t.probe.pipe(p.timeout(n), p.isSuccess, p.flatMap((e) => p.sync(() => {
+		let n = Date.now();
+		if (e) {
+			if (i = void 0, !o || (a ??= n, n - a < r)) return;
+			t.log.info("workflow runner: its storage answers again — taking part in shard assignment again"), a = void 0, o = !1, s = !1;
+			return;
+		}
+		a = void 0, i ??= n, !o && n - i >= r && (o = !0, t.log.warn(`workflow runner: its connection pool has not served a query for ${Math.round((n - i) / 1e3)}s — released its shards and stopped its heartbeat, so another runner takes them`, { fix: "look for what holds this process's database connections (`voltro_db_operations_in_flight` against DB_MAX_CONNECTIONS)" }));
+	}))).pipe(p.repeat(_.spaced(n)), p.forkScoped);
+	let c = () => new ge({ cause: /* @__PURE__ */ Error("workflow runner gave its shards up: its connection pool does not serve queries") });
+	return w.RunnerStorage.of({
+		...e,
+		register: (t, n) => p.suspend(() => o ? p.fail(c()) : e.register(t, n)),
+		acquire: (t, n) => p.suspend(() => o ? p.succeed([]) : e.acquire(t, n)),
+		refresh: (t, n) => p.suspend(() => {
+			if (!o) return e.refresh(t, n);
+			let r = s ? p.void : e.releaseAll(t).pipe(p.tap(() => p.sync(() => {
+				s = !0;
+			})), p.ignore);
+			return p.zipRight(r, p.fail(c()));
+		})
+	});
+}), X = he({ scope: "voltro:workflow" }), Ve = 34e3, Z = (e) => {
 	let t = e.runnerListenHost ?? process.env.POD_IP ?? process.env.VOLTRO_WORKFLOW_RUNNER_HOST ?? "localhost";
 	return {
 		host: t,
 		port: e.runnerListenPort ?? 34e3,
 		localhostRisk: e.runnerStorage === "sql" && (t === "localhost" || t === "127.0.0.1")
 	};
-}, Re = (e) => {
+}, He = (e) => {
 	let t = e.sqlRunnerSupported ?? !e.sqliteFamily, n = t ? "sql" : "memory", r = e.override ?? process.env.VOLTRO_WORKFLOW_RUNNER_STORAGE;
 	if (r === void 0 || r.trim() === "") return n;
 	let i = r.trim().toLowerCase();
 	if (i !== "memory" && i !== "sql") throw Error(`VOLTRO_WORKFLOW_RUNNER_STORAGE must be 'memory' or 'sql', got '${r}'.`);
 	if (i === "sql" && !t) throw Error("VOLTRO_WORKFLOW_RUNNER_STORAGE='sql' is unsupported for this database connection. Use 'memory' for a local or embedded SQLite-family connection.");
 	return i;
-}, X = (e = process.env.VOLTRO_WORKFLOW_SHARD_LOCK) => {
+}, Q = (e = process.env.VOLTRO_WORKFLOW_SHARD_LOCK) => {
 	if (e === void 0 || e.trim() === "") return "auto";
 	let t = e.trim().toLowerCase();
 	if (t === "auto" || t === "row" || t === "advisory") return t;
 	throw Error(`VOLTRO_WORKFLOW_SHARD_LOCK must be 'auto', 'row', or 'advisory', got '${e}'.`);
-}, Z = p.gen(function* () {
-	let e = yield* (yield* y.SqlClient)`SELECT @@wsrep_on AS wsrep`.pipe(p.catchAll(() => p.succeed([])));
+}, Ue = p.gen(function* () {
+	let e = yield* (yield* b.SqlClient)`SELECT @@wsrep_on AS wsrep`.pipe(p.catchAll(() => p.succeed([])));
 	if (e.length === 0) return !1;
 	let t = e[0].wsrep;
 	return t === 1 || t === "1" || t === !0 || t === "ON" || t === "on";
-}), Q = (e) => p.gen(function* () {
-	return e.runnerStorage === "memory" ? !1 : e.dialectId === "sqlite" || e.dialectId === "turso" || e.mode === "row" ? !0 : e.mode === "advisory" || e.dialectId !== "mysql" && e.dialectId !== "mariadb" ? !1 : yield* Z;
-}), ze = (e) => ({
+}), We = (e) => p.gen(function* () {
+	return e.runnerStorage === "memory" ? !1 : e.dialectId === "sqlite" || e.dialectId === "turso" || e.mode === "row" ? !0 : e.mode === "advisory" || e.dialectId !== "mysql" && e.dialectId !== "mariadb" ? !1 : yield* Ue;
+}), Ge = (e) => ({
 	coordination: e.disableAdvisory ? "row-based" : "advisory",
 	dialect: e.dialectId ?? "unknown",
 	mode: e.mode,
 	...e.disableAdvisory && e.mode === "auto" ? { reason: e.dialectId === "sqlite" || e.dialectId === "turso" ? "SQLite-family connection uses row locks" : "wsrep/Galera cluster detected" } : {}
-}), Be = () => {
+}), Ke = () => {
 	let e = (e) => {
 		if (e === void 0) return;
 		let t = Number(e);
@@ -551,54 +576,62 @@ var ge = (e) => p.gen(function* () {
 		...n === void 0 ? {} : { failoverHeartbeatSeconds: n },
 		...r === void 0 ? {} : { messagePollSeconds: r }
 	};
-}, Ve = p.flatMap(C.Sharding, (e) => e.pollStorage), $ = /* @__PURE__ */ new Set(), He = (t) => {
-	let n = Y({
+}, qe = p.flatMap(T.Sharding, (e) => e.pollStorage), $ = /* @__PURE__ */ new Set(), Je = (t) => {
+	let n = Z({
 		runnerStorage: t.runnerStorage,
 		runnerListenHost: t.runnerListenHost,
 		runnerListenPort: t.runnerListenPort
 	}), r = n.host, i = n.port;
-	n.localhostRisk && !$.has(r) && ($.add(r), J.warn("runner host is not routable from other pods with SQL cluster storage — a workflow cannot resume on another pod after a reschedule. Inject POD_IP via the K8s downward API (fieldRef: status.podIP) or set runnerListenHost / VOLTRO_WORKFLOW_RUNNER_HOST.", { runnerHost: r }));
-	let a = t.dialectId === "sqlite" ? m.scoped(y.SqlClient, p.gen(function* () {
-		let e = yield* y.SqlClient, t = (yield* e.unsafe("PRAGMA database_list")).find((e) => e.name === "main");
-		return t === void 0 ? yield* p.die(/* @__PURE__ */ Error("SQLite workflow connection has no main database")) : (t.file !== "" && (yield* Fe(t.file)), e);
-	}).pipe(p.orDie)).pipe(m.provide(t.sqlClientLayer)) : t.sqlClientLayer, s = U({
+	n.localhostRisk && !$.has(r) && ($.add(r), X.warn("runner host is not routable from other pods with SQL cluster storage — a workflow cannot resume on another pod after a reschedule. Inject POD_IP via the K8s downward API (fieldRef: status.podIP) or set runnerListenHost / VOLTRO_WORKFLOW_RUNNER_HOST.", { runnerHost: r }));
+	let a = t.dialectId === "sqlite" ? m.scoped(b.SqlClient, p.gen(function* () {
+		let e = yield* b.SqlClient, t = (yield* e.unsafe("PRAGMA database_list")).find((e) => e.name === "main");
+		return t === void 0 ? yield* p.die(/* @__PURE__ */ Error("SQLite workflow connection has no main database")) : (t.file !== "" && (yield* Re(t.file)), e);
+	}).pipe(p.orDie)).pipe(m.provide(t.sqlClientLayer)) : t.sqlClientLayer, s = G({
 		runnerStorage: t.runnerStorage,
 		charset: t.charset
-	}).pipe(m.provide(a)), c = t.shardLockMode ?? X(), l = m.unwrapEffect(Q({
+	}).pipe(m.provide(a)), c = t.shardLockMode ?? Q(), l = m.unwrapEffect(We({
 		dialectId: t.dialectId,
 		runnerStorage: t.runnerStorage,
 		mode: c
 	}).pipe(p.map((e) => {
-		t.runnerStorage === "sql" && J.info("shard-lock coordination", ze({
+		t.runnerStorage === "sql" && X.info("shard-lock coordination", Ge({
 			disableAdvisory: e,
 			dialectId: t.dialectId,
 			mode: c
 		}));
-		let n = m.effect(o, p.flatMap(C.Sharding, (n) => q({
-			ownAddress: S.make(r, i).toString(),
+		let n = m.effect(o, p.flatMap(T.Sharding, (n) => Y({
+			ownAddress: C.make(r, i).toString(),
 			shardLockMode: t.runnerStorage === "sql" ? e ? "row" : "advisory" : "none",
 			runnerStorage: t.runnerStorage,
 			dialectId: t.dialectId,
-			hasShard: (e) => n.hasShardId(ae.ShardId.fromString(e))
-		}))), a = se.layer({
+			hasShard: (e) => n.hasShardId(se.ShardId.fromString(e))
+		}))), a = {
+			runnerAddress: h.some(C.make(r, i)),
+			runnerListenAddress: h.some(C.make(r, i)),
+			...e ? { shardLockDisableAdvisory: !0 } : {},
+			...t.failoverLeaseSeconds === void 0 ? {} : { shardLockExpiration: f.seconds(t.failoverLeaseSeconds) },
+			...t.failoverHeartbeatSeconds === void 0 ? {} : { shardLockRefreshInterval: f.seconds(t.failoverHeartbeatSeconds) },
+			...t.messagePollSeconds === void 0 ? {} : { entityMessagePollInterval: f.seconds(t.messagePollSeconds) }
+		}, s = f.seconds(t.failoverHeartbeatSeconds ?? 10), l = t.runnerStorage === "sql" ? T.layer.pipe(m.provideMerge(oe.layerNoop), m.provideMerge(E.layer), m.provide([m.scoped(w.RunnerStorage, p.gen(function* () {
+			let e = yield* b.SqlClient;
+			return yield* Be(yield* w.RunnerStorage, {
+				probe: e`SELECT 1`,
+				interval: s,
+				giveUpAfter: f.times(s, 3),
+				log: X
+			});
+		})).pipe(m.provide(m.orDie(D.layer))), ae.layerNoop]), m.provide(ce.layerFromEnv(a))) : le.layer({
 			runnerStorage: t.runnerStorage,
-			shardingConfig: {
-				runnerAddress: h.some(S.make(r, i)),
-				runnerListenAddress: h.some(S.make(r, i)),
-				...e ? { shardLockDisableAdvisory: !0 } : {},
-				...t.failoverLeaseSeconds === void 0 ? {} : { shardLockExpiration: f.seconds(t.failoverLeaseSeconds) },
-				...t.failoverHeartbeatSeconds === void 0 ? {} : { shardLockRefreshInterval: f.seconds(t.failoverHeartbeatSeconds) },
-				...t.messagePollSeconds === void 0 ? {} : { entityMessagePollInterval: f.seconds(t.messagePollSeconds) }
-			}
+			shardingConfig: a
 		});
-		return m.mergeAll(n, m.effect(E, _e), m.effect(N, Ee)).pipe(m.provideMerge(a));
-	}))).pipe(m.provide(s)), u = t.extraShardingLayers ?? [], g = u.length === 0 ? b.layer : m.mergeAll(b.layer, ...u), _ = l.pipe(m.map((t) => d.add(t, C.Sharding, e(d.get(t, C.Sharding)))));
+		return m.mergeAll(n, m.effect(A, ve), m.effect(I, Oe)).pipe(m.provideMerge(l));
+	}))).pipe(m.provide(s)), u = t.extraShardingLayers ?? [], g = u.length === 0 ? x.layer : m.mergeAll(x.layer, ...u), _ = l.pipe(m.map((t) => d.add(t, T.Sharding, e(d.get(t, T.Sharding)))));
 	return g.pipe(m.provideMerge(_));
-}, Ue = (e) => ee.make({
+}, Ye = (e) => ee.make({
 	name: e.name,
 	cron: e.cron,
 	execute: p.promise(() => e.execute()),
 	skipIfOlderThan: e.skipIfOlderThan ?? f.days(1)
 });
 //#endregion
-export { j as _, Ve as a, Re as c, q as d, U as f, Se as g, xe as h, X as i, Q as l, D as m, Le as n, Be as o, N as p, Z as r, Y as s, Ue as t, He as u, E as v };
+export { we as _, qe as a, He as c, Y as d, G as f, Ce as g, Se as h, Q as i, We as l, j as m, Ve as n, Ke as o, I as p, Ue as r, Z as s, Ye as t, Je as u, A as v };

package/dist/cluster.js

@@ -1,2 +1,2 @@
-import { _ as e, a as t, c as n, d as r, f as i, g as a, h as o, i as s, l as c, m as l, n as u, o as d, p as f, r as p, s as m, t as h, u as g, v as _ } from "./cluster-NnChVZoK.js";
+import { _ as e, a as t, c as n, d as r, f as i, g as a, h as o, i as s, l as c, m as l, n as u, o as d, p as f, r as p, s as m, t as h, u as g, v as _ } from "./cluster--WvQZFwk.js";
 export { u as DEFAULT_RUNNER_LISTEN_PORT, f as WorkflowRestartDispatcher, _ as WorkflowResumeDispatcher, l as activityPrimaryKey, i as clusterMigrationGateLayer, p as detectWsrepCluster, h as makeClusterCronLayer, r as makeClusterDiagnostics, s as parseShardLockMode, t as pollWorkflowStorage, o as readWorkflowRunReply, a as redriveFailedRun, d as resolveFailoverTuning, m as resolveRunnerIdentity, n as resolveRunnerStorage, c as resolveShardLockDisableAdvisory, e as resumeRunFromStep, g as workflowEngineLayer };

package/dist/clusterTestSuite.js

@@ -1,7 +1,7 @@
 import { F as e, R as t, S as n, g as r, v as i } from "./primitives-Btjhjm8P.js";
 import { E as a, a as o } from "./journalEncryption--n5VDdYO.js";
-import { C as s, r as c, w as l } from "./src--sCbH8Fs.js";
-import { t as u, u as d } from "./cluster-NnChVZoK.js";
+import { C as s, r as c, w as l } from "./src-DP-lKqaG.js";
+import { t as u, u as d } from "./cluster--WvQZFwk.js";
 import { Cause as f, Cron as p, Deferred as m, Effect as h, Fiber as g, Layer as _, Schema as v } from "effect";
 import { SqlClient as y } from "@effect/sql";
 import { afterEach as b, beforeAll as x, describe as S, expect as C, test as w } from "vitest";