@voltro/workflow 0.25.0 → 0.27.0

package/dist/cluster-BQ0H5M9n.js

@@ -0,0 +1,131 @@
+import { Context as e, Duration as t, Effect as n, Layer as r, Option as i } from "effect";
+import { ClusterCron as a, ClusterSchema as o, ClusterWorkflowEngine as s, EntityAddress as c, EntityId as l, EntityType as u, MessageStorage as d, RunnerAddress as f, Sharding as p, SingleRunner as m } from "@effect/cluster";
+import { SqlClient as h } from "@effect/sql";
+//#region src/clusterLayer.ts
+var g = 34e3, _ = (e) => {
+	let t = e.runnerListenHost ?? process.env.POD_IP ?? process.env.VOLTRO_WORKFLOW_RUNNER_HOST ?? "localhost";
+	return {
+		host: t,
+		port: e.runnerListenPort ?? 34e3,
+		localhostRisk: e.runnerStorage === "sql" && (t === "localhost" || t === "127.0.0.1")
+	};
+}, v = (e) => {
+	let t = e.sqliteFamily ? "memory" : "sql", n = e.override ?? process.env.VOLTRO_WORKFLOW_RUNNER_STORAGE;
+	if (n === void 0 || n.trim() === "") return t;
+	let r = n.trim().toLowerCase();
+	if (r !== "memory" && r !== "sql") throw Error(`VOLTRO_WORKFLOW_RUNNER_STORAGE must be 'memory' or 'sql', got '${n}'.`);
+	if (r === "sql" && e.sqliteFamily) throw Error("VOLTRO_WORKFLOW_RUNNER_STORAGE='sql' is unsupported on sqlite / turso — @effect/cluster has no sqlite advisory-lock branch. Use 'memory'.");
+	return r;
+}, y = (e = process.env.VOLTRO_WORKFLOW_SHARD_LOCK) => {
+	if (e === void 0 || e.trim() === "") return "auto";
+	let t = e.trim().toLowerCase();
+	if (t === "auto" || t === "row" || t === "advisory") return t;
+	throw Error(`VOLTRO_WORKFLOW_SHARD_LOCK must be 'auto', 'row', or 'advisory', got '${e}'.`);
+}, b = n.gen(function* () {
+	let e = yield* (yield* h.SqlClient)`SELECT @@wsrep_on AS wsrep`.pipe(n.catchAll(() => n.succeed([])));
+	if (e.length === 0) return !1;
+	let t = e[0].wsrep;
+	return t === 1 || t === "1" || t === !0 || t === "ON" || t === "on";
+}), x = (e) => n.gen(function* () {
+	return e.runnerStorage === "memory" ? !1 : e.mode === "row" ? !0 : e.mode === "advisory" || e.dialectId !== "mysql" && e.dialectId !== "mariadb" ? !1 : yield* b;
+}), S = () => {
+	let e = (e) => {
+		if (e === void 0) return;
+		let t = Number(e);
+		return Number.isFinite(t) && t > 0 ? t : void 0;
+	}, t = e(process.env.VOLTRO_WORKFLOW_FAILOVER_LEASE), n = e(process.env.VOLTRO_WORKFLOW_FAILOVER_HEARTBEAT), r = e(process.env.VOLTRO_WORKFLOW_POLL_INTERVAL);
+	return {
+		...t === void 0 ? {} : { failoverLeaseSeconds: t },
+		...n === void 0 ? {} : { failoverHeartbeatSeconds: n },
+		...r === void 0 ? {} : { messagePollSeconds: r }
+	};
+}, C = n.flatMap(p.Sharding, (e) => e.pollStorage), w = /* @__PURE__ */ new Set(), T = (e) => {
+	let a = _({
+		runnerStorage: e.runnerStorage,
+		runnerListenHost: e.runnerListenHost,
+		runnerListenPort: e.runnerListenPort
+	}), o = a.host, c = a.port;
+	a.localhostRisk && !w.has(o) && (w.add(o), process.stderr.write(`[voltro:workflow] runner host is '${o}' with SQL cluster storage — other pods cannot reach this runner, so a workflow cannot resume on another pod after a reschedule. Inject POD_IP via the K8s downward API (fieldRef: status.podIP) or set runnerListenHost / VOLTRO_WORKFLOW_RUNNER_HOST.
+`));
+	let l = e.shardLockMode ?? y(), u = r.unwrapEffect(x({
+		dialectId: e.dialectId,
+		runnerStorage: e.runnerStorage,
+		mode: l
+	}).pipe(n.map((n) => (e.runnerStorage === "sql" && process.stderr.write(`[voltro:workflow] shard-lock coordination: ${n ? "row-based" : "advisory"} (dialect=${e.dialectId ?? "unknown"}, mode=${l}` + (n && l === "auto" ? ", wsrep/Galera cluster detected" : "") + ")\n"), m.layer({
+		runnerStorage: e.runnerStorage,
+		shardingConfig: {
+			runnerAddress: i.some(f.make(o, c)),
+			runnerListenAddress: i.some(f.make(o, c)),
+			...n ? { shardLockDisableAdvisory: !0 } : {},
+			...e.failoverLeaseSeconds === void 0 ? {} : { shardLockExpiration: t.seconds(e.failoverLeaseSeconds) },
+			...e.failoverHeartbeatSeconds === void 0 ? {} : { shardLockRefreshInterval: t.seconds(e.failoverHeartbeatSeconds) },
+			...e.messagePollSeconds === void 0 ? {} : { entityMessagePollInterval: t.seconds(e.messagePollSeconds) }
+		}
+	}))))).pipe(r.provide(e.sqlClientLayer)), d = e.extraShardingLayers ?? [];
+	return (d.length === 0 ? s.layer : r.mergeAll(s.layer, ...d)).pipe(r.provideMerge(u));
+}, E = (e) => a.make({
+	name: e.name,
+	cron: e.cron,
+	execute: n.promise(() => e.execute()),
+	skipIfOlderThan: e.skipIfOlderThan ?? t.days(1)
+}), D = (e, t) => `${e}/${t}`, O = (t, n, r) => {
+	let i = l.make(n), a = e.get(t.annotations, o.ShardGroup)(i);
+	return new c.EntityAddress({
+		entityType: u.EntityType.make(`Workflow/${t.name}`),
+		entityId: i,
+		shardId: r.getShardId(i, a)
+	});
+}, k = (e) => {
+	let t = (e) => e && typeof e == "object" && "_tag" in e ? String(e._tag) : void 0, n = t(e);
+	if (n === "Failure" || n !== "Success") return "failed";
+	let r = e.value, i = t(r);
+	if (i === "Suspended") return "suspended";
+	if (i === "Complete") {
+		let e = r.exit;
+		return t(e) === "Success" ? "succeeded" : "failed";
+	}
+	return "failed";
+}, A = (e) => n.gen(function* () {
+	let t = yield* p.Sharding, r = yield* d.MessageStorage, a = O(e.workflow, e.executionId, t), o = yield* r.requestIdForPrimaryKey({
+		address: a,
+		tag: "run",
+		id: ""
+	}).pipe(n.orElseSucceed(() => i.none()));
+	if (i.isNone(o)) return {
+		redriven: !1,
+		runReplyCleared: !1,
+		activitiesReset: 0,
+		reason: "no-journal"
+	};
+	let s = (yield* r.repliesForUnfiltered([o.value]).pipe(n.orElseSucceed(() => []))).filter((e) => e._tag === "WithExit").at(-1);
+	if (s === void 0) return {
+		redriven: !1,
+		runReplyCleared: !1,
+		activitiesReset: 0,
+		reason: "no-reply"
+	};
+	if (k(s.exit) === "succeeded") return {
+		redriven: !1,
+		runReplyCleared: !1,
+		activitiesReset: 0,
+		reason: "succeeded"
+	};
+	let c = 0;
+	for (let o of e.failedActivities) {
+		let e = yield* r.requestIdForPrimaryKey({
+			address: a,
+			tag: "activity",
+			id: D(o.name, o.attempt)
+		}).pipe(n.orElseSucceed(() => i.none()));
+		i.isSome(e) && (yield* t.reset(e.value)) && (c += 1);
+	}
+	let l = yield* t.reset(o.value);
+	return yield* t.pollStorage, {
+		redriven: !0,
+		runReplyCleared: l,
+		activitiesReset: c,
+		reason: null
+	};
+});
+//#endregion
+export { b as a, S as c, x as d, T as f, g as i, _ as l, A as n, y as o, E as r, C as s, D as t, v as u };

package/dist/cluster.d.ts

@@ -3,11 +3,22 @@ import { Cron } from 'effect';
 import { Duration } from 'effect';
 import { Effect } from 'effect';
 import { Layer } from 'effect';
+import { MessageStorage } from '@effect/cluster';
 import { Sharding } from '@effect/cluster';
 import { SqlClient } from '@effect/sql';
 import { SqlError } from '@effect/sql';
+import { Workflow } from '@effect/workflow';
 import { WorkflowEngine } from '@effect/workflow';
 
+/**
+ * The cluster's own activity primary-key format — an activity RPC's
+ * `primaryKey` is `` `${name}/${attempt}` `` (`ClusterWorkflowEngine`'s
+ * `activityPrimaryKey`). Mirrored here so we can address a specific
+ * failed attempt's stored reply; the contract test pins this against the
+ * real engine so a format change is caught, not silently missed.
+ */
+export declare const activityPrimaryKey: (activityName: string, attempt: number) => string;
+
 export declare interface ClusterCronSpec {
     /** Stable schedule name — used as the cluster cron's identity. */
     readonly name: string;
@@ -34,6 +45,12 @@ export declare const DEFAULT_RUNNER_LISTEN_PORT: 34000;
  *  somehow can't be probed can still force `VOLTRO_WORKFLOW_SHARD_LOCK=row`. */
 export declare const detectWsrepCluster: Effect.Effect<boolean, never, SqlClient.SqlClient>;
 
+/** A step attempt to reset so it re-executes on the next drive. */
+export declare interface FailedActivityRef {
+    readonly name: string;
+    readonly attempt: number;
+}
+
 /**
  * Build the cluster-cron layer for one schedule. Returns a
  * `Layer<never, never, Sharding>` — the framework provides `Sharding`
@@ -47,6 +64,94 @@ export declare const makeClusterCronLayer: (spec: ClusterCronSpec) => Layer.Laye
  *  the advisory path that breaks cross-pod handoff on Galera. */
 export declare const parseShardLockMode: (raw?: string | undefined) => ShardLockMode;
 
+/**
+ * Re-poll the cluster's durable storage NOW — the effect behind a cross-replica
+ * "wake". Run it on the workflow runtime (`workflowRuntime.runPromise`), which
+ * carries `Sharding`; it opens the storage-read latch so a message just enqueued
+ * by another replica is picked up immediately instead of on the next poll tick.
+ * Lives here so a boot path doesn't import `@effect/cluster` (and its ~420ms
+ * load) just to reach the `Sharding` tag.
+ */
+export declare const pollWorkflowStorage: Effect.Effect<void, never, Sharding.Sharding>;
+
+/**
+ * Re-drive a terminally-`failed` workflow run from the step it died on.
+ *
+ * Clears the failed step attempts' journaled replies (so they re-execute)
+ * and the terminal `run` reply, then re-polls cluster storage so the
+ * engine re-delivers the `run` request. Completed steps replay from the
+ * journal; only the cleared (failed) steps run again.
+ *
+ * Idempotent-safe to REFUSE: if the run has no journal, is still running,
+ * or already succeeded, it returns `redriven: false` with a `reason` and
+ * touches nothing.
+ *
+ * Requires the SAME cluster instance's `Sharding` + `MessageStorage` that
+ * the workflow engine was built from (see `workflowEngineLayer`, which
+ * re-exports both for exactly this).
+ */
+export declare const redriveFailedRun: (target: RedriveTarget) => Effect.Effect<RedriveOutcome, never, Sharding.Sharding | MessageStorage.MessageStorage>;
+
+export declare interface RedriveOutcome {
+    /** True iff the `run` reply was cleared and storage re-polled. */
+    readonly redriven: boolean;
+    /** True iff the terminal `run` reply was cleared. */
+    readonly runReplyCleared: boolean;
+    /** How many failed activity replies were cleared (so they re-execute). */
+    readonly activitiesReset: number;
+    /** Set when `redriven` is false. */
+    readonly reason: RedriveSkipReason | null;
+}
+
+/** Why a re-drive did nothing — surfaced so the caller reports honestly. */
+export declare type RedriveSkipReason = 
+/** No `run` request found in the cluster journal — a memory engine, or
+*  a run that was never durably journaled. Nothing to re-drive. */
+'no-journal'
+/** The `run` request exists but carries no terminal reply — the run is
+*  still processing. Re-driving would race the live fiber; refuse. */
+| 'no-reply'
+/** The stored reply is a SUCCESS — the run completed cleanly. Nothing
+*  to re-drive; refuse rather than resurrect a finished run. */
+| 'succeeded';
+
+export declare interface RedriveTarget {
+    /** The workflow definition — for its `name` (entity type) + `annotations` (shard group). */
+    readonly workflow: Workflow.Any;
+    /** The engine-assigned execution id (= cluster `entityId`) of the run to re-drive. */
+    readonly executionId: string;
+    /**
+     * Failed step attempts to reset so they re-execute fresh. Succeeded
+     * steps are intentionally NOT listed — their journaled replies stay,
+     * so they replay rather than re-run. Typically the `status='failed'`
+     * rows of `_voltro_workflow_run_steps` for this run.
+     */
+    readonly failedActivities: ReadonlyArray<FailedActivityRef>;
+}
+
+/**
+ * Read the failover-tuning knobs from the environment, for the boot paths to
+ * spread into {@link workflowEngineLayer}. Operators set:
+ *
+ * - `VOLTRO_WORKFLOW_FAILOVER_LEASE` (seconds → `failoverLeaseSeconds`) — how
+ *   long after a crashed replica's last heartbeat its in-flight workflows
+ *   become claimable by a survivor. Default 35. Lower = faster failover, at the
+ *   cost of false-positive reclaims under a long GC / DB-latency stall.
+ * - `VOLTRO_WORKFLOW_FAILOVER_HEARTBEAT` (seconds → `failoverHeartbeatSeconds`)
+ *   — the lease refresh cadence. Default 10. Keep it ≈ lease/3.
+ * - `VOLTRO_WORKFLOW_POLL_INTERVAL` (seconds → `messagePollSeconds`) — the
+ *   NEW-message pickup latency for a workflow whose shard is owned by ANOTHER
+ *   replica. Default 10. Lower for latency-sensitive multi-replica workloads,
+ *   at the cost of more idle poll queries. No effect on a single replica.
+ *
+ * A malformed value is ignored (the default stands) rather than failing boot.
+ */
+export declare const resolveFailoverTuning: () => {
+    failoverLeaseSeconds?: number;
+    failoverHeartbeatSeconds?: number;
+    messagePollSeconds?: number;
+};
+
 /** Resolve the runner host/port the SAME way `workflowEngineLayer` does:
  *  explicit option → `POD_IP` → `VOLTRO_WORKFLOW_RUNNER_HOST` → loopback;
  *  port from option → `VOLTRO_WORKFLOW_RUNNER_PORT` → default. */
@@ -155,19 +260,7 @@ export declare type RunnerStorage = 'sql' | 'memory';
  */
 export declare type ShardLockMode = 'auto' | 'row' | 'advisory';
 
-/**
- * Compose the dialect's SqlClient → SingleRunner → ClusterWorkflowEngine
- * into a single layer that user code provides to its program to get
- * durable workflows.
- *
- * `extraShardingLayers` (e.g. `ClusterCron` layers from
- * `cluster`-coordinated schedules) are merged into the same runtime so
- * they share the runner identity + shard map.
- *
- * For dev / tests where no real engine is available, see
- * `inMemoryWorkflowEngineLayer`.
- */
-export declare const workflowEngineLayer: (options: WorkflowEngineLayerOptions) => Layer.Layer<WorkflowEngine.WorkflowEngine>;
+export declare const workflowEngineLayer: (options: WorkflowEngineLayerOptions) => Layer.Layer<WorkflowEngine.WorkflowEngine | Sharding.Sharding | MessageStorage.MessageStorage>;
 
 export declare interface WorkflowEngineLayerOptions {
     /**
@@ -196,6 +289,40 @@ export declare interface WorkflowEngineLayerOptions {
     readonly shardLockMode?: ShardLockMode;
     readonly runnerListenHost?: string;
     readonly runnerListenPort?: number;
+    /**
+     * FAILOVER TUNING — the two knobs that govern how fast a SURVIVING replica
+     * takes over a CRASHED replica's in-flight workflows. Reclaim is a
+     * lease-expiry problem, NOT a polling one: a dead runner keeps its shards
+     * until its heartbeat goes stale, so the floor is the lease TTL.
+     *
+     * - `failoverLeaseSeconds` (default 35, `@effect/cluster`
+     *   `shardLockExpiration`) — how long after a runner's last heartbeat its
+     *   shards become claimable. Lower = faster failover on a hard crash, at the
+     *   cost of false-positive reclaims if a healthy runner is paused longer than
+     *   this by a GC pause or a DB-latency spike (two runners could then briefly
+     *   both think they own a shard).
+     * - `failoverHeartbeatSeconds` (default 10, `shardLockRefreshInterval`) — how
+     *   often a runner refreshes its lease + heartbeat. Keep it well below the
+     *   lease (≈ lease/3) so a single slow refresh doesn't trip a false reclaim.
+     *
+     * Omit both for production defaults. A push wake (LISTEN/NOTIFY) does NOT help
+     * here — reclaim is time-bound, not notification-bound; only these two (or a
+     * K8s liveness probe) move the floor.
+     */
+    readonly failoverLeaseSeconds?: number;
+    readonly failoverHeartbeatSeconds?: number;
+    /**
+     * NEW-MESSAGE latency across replicas (default 10, `@effect/cluster`
+     * `entityMessagePollInterval`). DISTINCT from failover: this is how long a
+     * freshly-triggered workflow can wait before the replica that OWNS its shard
+     * picks it up. When the triggering replica owns the shard it's immediate
+     * (a same-process push); when ANOTHER replica owns it, Voltro's single-runner
+     * topology has no cross-runner push, so the owner sees it on its next poll —
+     * up to this interval. Lower it for latency-sensitive multi-replica workloads,
+     * at the cost of more idle poll queries. (The lower-load fix is a pg
+     * LISTEN/NOTIFY wake, not yet wired.) No effect on a single replica.
+     */
+    readonly messagePollSeconds?: number;
     /**
      * Extra layers that need the cluster's `Sharding` service — most
      * prominently `ClusterCron` layers for `cluster`-coordinated

package/dist/cluster.js

@@ -1,2 +1,2 @@
-import { a as e, c as t, i as n, n as r, o as i, r as a, s as o, t as s } from "./cluster-DuFweKq0.js";
-export { r as DEFAULT_RUNNER_LISTEN_PORT, a as detectWsrepCluster, s as makeClusterCronLayer, n as parseShardLockMode, e as resolveRunnerIdentity, i as resolveRunnerStorage, o as resolveShardLockDisableAdvisory, t as workflowEngineLayer };
+import { a as e, c as t, d as n, f as r, i, l as a, n as o, o as s, r as c, s as l, t as u, u as d } from "./cluster-BQ0H5M9n.js";
+export { i as DEFAULT_RUNNER_LISTEN_PORT, u as activityPrimaryKey, e as detectWsrepCluster, c as makeClusterCronLayer, s as parseShardLockMode, l as pollWorkflowStorage, o as redriveFailedRun, t as resolveFailoverTuning, a as resolveRunnerIdentity, d as resolveRunnerStorage, n as resolveShardLockDisableAdvisory, r as workflowEngineLayer };

package/dist/clusterTestSuite.d.ts

@@ -116,6 +116,40 @@ declare interface WorkflowEngineLayerOptions {
     readonly shardLockMode?: ShardLockMode;
     readonly runnerListenHost?: string;
     readonly runnerListenPort?: number;
+    /**
+     * FAILOVER TUNING — the two knobs that govern how fast a SURVIVING replica
+     * takes over a CRASHED replica's in-flight workflows. Reclaim is a
+     * lease-expiry problem, NOT a polling one: a dead runner keeps its shards
+     * until its heartbeat goes stale, so the floor is the lease TTL.
+     *
+     * - `failoverLeaseSeconds` (default 35, `@effect/cluster`
+     *   `shardLockExpiration`) — how long after a runner's last heartbeat its
+     *   shards become claimable. Lower = faster failover on a hard crash, at the
+     *   cost of false-positive reclaims if a healthy runner is paused longer than
+     *   this by a GC pause or a DB-latency spike (two runners could then briefly
+     *   both think they own a shard).
+     * - `failoverHeartbeatSeconds` (default 10, `shardLockRefreshInterval`) — how
+     *   often a runner refreshes its lease + heartbeat. Keep it well below the
+     *   lease (≈ lease/3) so a single slow refresh doesn't trip a false reclaim.
+     *
+     * Omit both for production defaults. A push wake (LISTEN/NOTIFY) does NOT help
+     * here — reclaim is time-bound, not notification-bound; only these two (or a
+     * K8s liveness probe) move the floor.
+     */
+    readonly failoverLeaseSeconds?: number;
+    readonly failoverHeartbeatSeconds?: number;
+    /**
+     * NEW-MESSAGE latency across replicas (default 10, `@effect/cluster`
+     * `entityMessagePollInterval`). DISTINCT from failover: this is how long a
+     * freshly-triggered workflow can wait before the replica that OWNS its shard
+     * picks it up. When the triggering replica owns the shard it's immediate
+     * (a same-process push); when ANOTHER replica owns it, Voltro's single-runner
+     * topology has no cross-runner push, so the owner sees it on its next poll —
+     * up to this interval. Lower it for latency-sensitive multi-replica workloads,
+     * at the cost of more idle poll queries. (The lower-load fix is a pg
+     * LISTEN/NOTIFY wake, not yet wired.) No effect on a single replica.
+     */
+    readonly messagePollSeconds?: number;
     /**
      * Extra layers that need the cluster's `Sharding` service — most
      * prominently `ClusterCron` layers for `cluster`-coordinated

package/dist/clusterTestSuite.js

@@ -1,6 +1,6 @@
-import { C as e, m as t, p as n, v as r, x as i } from "./primitives-CWy1iu5w.js";
-import { i as a } from "./src-DQww25rP.js";
-import { c as o, t as s } from "./cluster-DuFweKq0.js";
+import { C as e, m as t, p as n, v as r, x as i } from "./primitives-Dgu3O55Q.js";
+import { i as a } from "./src-KDx7NTsa.js";
+import { f as o, r as s } from "./cluster-BQ0H5M9n.js";
 import { Cron as c, Deferred as l, Effect as u, Fiber as d, Layer as f, Schema as p } from "effect";
 import { SqlClient as m } from "@effect/sql";
 import { beforeAll as h, describe as g, expect as _, test as v } from "vitest";
@@ -33,14 +33,14 @@ var y = async (e, t, n = 750) => {
 			for (let t of b) yield* e.unsafe(`DELETE FROM ${t}`).pipe(u.ignore);
 		}).pipe(u.scoped, u.provide(e.sqlClientLayer)));
 	} catch {}
-}, S = (e, t) => u.gen(function* () {
+}, S = "10 seconds", C = "30 seconds", w = (e, t) => u.gen(function* () {
 	let n = yield* u.sync(() => Date.now() + t);
 	for (; !e();) {
 		if (Date.now() > n) return !1;
 		yield* u.sleep("250 millis");
 	}
 	return !0;
-}), C = (m) => {
+}), T = (m) => {
 	let y = `${m.name}-${Date.now()}`, b = (e, t) => o({
 		sqlClientLayer: m.sqlClientLayer,
 		runnerStorage: m.runnerStorage,
@@ -104,7 +104,9 @@ var y = async (e, t, n = 750) => {
 			_(await u.runPromise(u.scoped(u.gen(function* () {
 				return yield* f.build(b(m.runnerPort + 1, [n])), yield* l.await(e).pipe(u.timeout("25 seconds"), u.orElseSucceed(() => !1));
 			})))).toBe(!0), _(t).toBeGreaterThanOrEqual(1);
-		}, 35e3), v.runIf(m.clusterResume)("a sleeping workflow resumes on a fresh runner after the first is torn down", async () => {
+		}, 35e3);
+		let g = m.clusterResume ? v : (() => {});
+		g("a sleeping workflow resumes on a fresh runner after the first is torn down", async () => {
 			if (!o) return;
 			let e = `dormancyResume_${y}`, i = [], a = [], s = [], c = () => r({
 				name: e,
@@ -126,7 +128,7 @@ var y = async (e, t, n = 750) => {
 					})
 				}), yield* n({
 					name: "nap",
-					duration: "30 seconds"
+					duration: S
 				}), yield* t({
 					name: "step2",
 					success: p.Void,
@@ -142,13 +144,13 @@ var y = async (e, t, n = 750) => {
 				};
 			}, v = { id: `resume-${y}` }, x = await u.runPromise(u.scoped(u.gen(function* () {
 				let { wf: e, layer: t } = g("A"), n = yield* f.build(t), r = yield* u.forkDaemon(e.execute(v).pipe(u.provide(n), u.ignore));
-				return (yield* S(() => s.length > 0, 6e4)) ? r : yield* u.die(/* @__PURE__ */ Error("runner A never journaled step1"));
+				return (yield* w(() => s.length > 0, 6e4)) ? r : yield* u.die(/* @__PURE__ */ Error("runner A never journaled step1"));
 			})));
 			await u.runPromise(d.interrupt(x).pipe(u.ignore)), _(i).toEqual(["A"]), _(a).toEqual([]), _(await u.runPromise(u.scoped(u.gen(function* () {
 				let { wf: e, layer: t } = g("B"), n = yield* f.build(t);
 				return yield* e.execute(v).pipe(u.provide(n), u.timeout("240 seconds"), u.orElseSucceed(() => "TIMEOUT"));
 			})))).toBe("done"), _(i).toEqual(["A"]), _(a).toEqual(["B"]);
-		}, 48e4), v.runIf(m.clusterResume)("dormancy loop: a wake signal survives teardown and is cleared on resume", async () => {
+		}, 48e4), g("dormancy loop: a wake signal survives teardown and is cleared on resume", async () => {
 			if (!o) return;
 			let s = `dormancyLoop_${y}`, c = `wr_${s}`, l = [], h = /* @__PURE__ */ new Map(), g = a({
 				store: {
@@ -176,7 +178,7 @@ var y = async (e, t, n = 750) => {
 					execute: u.void
 				}), yield* n({
 					name: "nap",
-					duration: "30 seconds"
+					duration: C
 				}), yield* t({
 					name: "step2",
 					success: p.Void,
@@ -184,22 +186,22 @@ var y = async (e, t, n = 750) => {
 						l.push(a);
 					})
 				}), "done";
-			}).pipe(u.locally(i, c), u.provideService(e, g))), C = m.runnerPort + 2, w = (e) => {
+			}).pipe(u.locally(i, c), u.provideService(e, g))), S = m.runnerPort + 2, T = (e) => {
 				let t = v();
 				return {
 					wf: t,
-					layer: x(t, e).pipe(f.provideMerge(b(C)))
+					layer: x(t, e).pipe(f.provideMerge(b(S)))
 				};
-			}, T = { id: `loop-${y}` }, E = await u.runPromise(u.scoped(u.gen(function* () {
-				let { wf: e, layer: t } = w("A"), n = yield* f.build(t), r = yield* u.forkDaemon(e.execute(T).pipe(u.provide(n), u.ignore));
-				return (yield* S(() => h.has(c), 6e4)) ? (yield* u.sleep("1 second"), r) : yield* u.die(/* @__PURE__ */ Error("workflow never registered a wakeup"));
+			}, E = { id: `loop-${y}` }, D = await u.runPromise(u.scoped(u.gen(function* () {
+				let { wf: e, layer: t } = T("A"), n = yield* f.build(t), r = yield* u.forkDaemon(e.execute(E).pipe(u.provide(n), u.ignore));
+				return (yield* w(() => h.has(c), 6e4)) ? (yield* u.sleep("1 second"), r) : yield* u.die(/* @__PURE__ */ Error("workflow never registered a wakeup"));
 			})));
-			await u.runPromise(d.interrupt(E).pipe(u.ignore)), _(h.has(c)).toBe(!0), _(l).toEqual([]), _(await u.runPromise(u.scoped(u.gen(function* () {
-				let { wf: e, layer: t } = w("B"), n = yield* f.build(t);
-				return yield* e.execute(T).pipe(u.provide(n), u.timeout("240 seconds"), u.orElseSucceed(() => "TIMEOUT"));
+			await u.runPromise(d.interrupt(D).pipe(u.ignore)), _(h.has(c)).toBe(!0), _(l).toEqual([]), _(await u.runPromise(u.scoped(u.gen(function* () {
+				let { wf: e, layer: t } = T("B"), n = yield* f.build(t);
+				return yield* e.execute(E).pipe(u.provide(n), u.timeout("600 seconds"), u.orElseSucceed(() => "TIMEOUT"));
 			})))).toBe("done"), _(l).toEqual(["B"]), _(h.has(c)).toBe(!1);
-		}, 48e4);
+		}, 9e5);
 	});
 };
 //#endregion
-export { C as runClusterEngineSuite, y as tcpReachable };
+export { T as runClusterEngineSuite, y as tcpReachable };

package/dist/index.d.ts

@@ -426,42 +426,58 @@ declare type StepOptions<R, Success extends Schema.Schema.Any, Error extends Sch
     /** Structured value recorded into `_voltro_workflow_run_steps.input`.
      *  Pass whatever makes the step debuggable in isolation. */
     readonly input?: unknown;
-    /** Declarative retry-policy summary recorded into
-     *  `_voltro_workflow_run_steps.retryPolicy`. Pure metadata — does
-     *  NOT change retry behavior on its own. Wire actual retries via
-     *  `Effect.retry` inside `execute:` or via `interruptRetryPolicy`
-     *  above; this field tells the dashboard what the intended policy
-     *  looks like so the per-attempt panel can show "attempt 2 of 5,
-     *  exponential 1s base" instead of just "attempt 2". */
+    /** Declarative retry policy — ENFORCED. The framework compiles it to an
+     *  Effect `Schedule` and retries `execute` accordingly (backoff, jitter,
+     *  attempt cap, time budget, error classification, provider `Retry-After`).
+     *  See {@link StepRetryPolicy}. Retries run INSIDE this one step and are
+     *  transparent to the durable engine; the step's FINAL outcome is recorded.
+     *  The serialisable knobs are also stored as `_voltro_workflow_run_steps.
+     *  retryPolicy` for the dashboard. `retry: { maxAttempts: 5 }` is enough. */
     readonly retry?: StepRetryPolicy;
 };
 
 /**
- * Declarative retry-policy summary. Pure metadata for the dashboard;
- * the framework does NOT apply this. Users wire actual retry behavior
- * via `Effect.retry(Schedule.*)` inside `execute:` or
- * `Activity.make`'s `interruptRetryPolicy`.
- *
- * Shape is intentionally close to Effect's Schedule combinators so a
- * future framework version can auto-translate declarative policies
- * into a Schedule + apply it. For now: record + display.
- *
- *   { strategy: 'exponential', maxAttempts: 5, baseDelay: '1 second', maxDelay: '30 seconds' }
- *   { strategy: 'fixed',       maxAttempts: 3, baseDelay: '500 millis' }
- *   { strategy: 'linear',      maxAttempts: 4, baseDelay: '1 second',  step: '2 seconds' }
+ * A step's retry policy. Every field is optional — `retry: {}` is already a
+ * sensible policy (3 attempts, exponential backoff, jittered). All fields are
+ * JSON-serialisable EXCEPT `retryable` (a predicate), which the recorder drops
+ * when it stores the policy as dashboard metadata.
  */
 export declare interface StepRetryPolicy {
-    readonly strategy: 'exponential' | 'fixed' | 'linear';
-    readonly maxAttempts: number;
-    /** Effect Duration string, e.g. `'1 second'`, `'500 millis'`. */
+    /** Total attempts INCLUDING the first. Default `3`. `1` disables retry. */
+    readonly maxAttempts?: number;
+    /** Backoff shape between attempts. Default `'exponential'`. */
+    readonly strategy?: 'exponential' | 'fixed' | 'linear';
+    /** First delay (Effect Duration string, e.g. `'500 millis'`). Default `'200 millis'`. */
     readonly baseDelay?: string;
-    /** Cap on exponential / linear growth, e.g. `'30 seconds'`. */
-    readonly maxDelay?: string;
-    /** Increment for `linear` strategy. */
+    /** Growth factor for `'exponential'`. Default `2`. */
+    readonly factor?: number;
+    /** Increment per attempt for `'linear'`. Default = `baseDelay`. */
     readonly step?: string;
-    /** Free-form note rendered as a chip in the dashboard. Useful when
-     *  the retry behavior is partially implemented in the user's
-     *  Effect chain and the summary is just documentation. */
+    /** Ceiling on any single delay, so exponential growth can't run away
+     *  (e.g. `'30 seconds'`). Applied before jitter. */
+    readonly maxDelay?: string;
+    /** Full jitter on each delay — spreads retries so a fleet doesn't
+     *  re-hit a recovering dependency in lockstep. Default `true`. */
+    readonly jitter?: boolean;
+    /** A total wall-clock BUDGET across all attempts: stop retrying once
+     *  this much time has elapsed since the first attempt, even if attempts
+     *  remain (e.g. `'5 minutes'`). A deadline, not a count. */
+    readonly maxElapsed?: string;
+    /** Retry ONLY failures whose typed-error `_tag` is in this list; every
+     *  other error fails fast. The declarative "retry transient, fail
+     *  permanent" — e.g. `['ProviderDown', 'RateLimited']`. */
+    readonly retryableErrors?: ReadonlyArray<string>;
+    /** Retry-predicate on the raw error (wins over `retryableErrors` when
+     *  both are set). Not serialised into dashboard metadata. Default:
+     *  retry every failure. */
+    readonly retryable?: (error: unknown) => boolean;
+    /** Honor a provider's own backoff: if a retryable error carries a
+     *  `retryAfterMillis` number (or `retryAfter` in seconds), use exactly that as
+     *  the delay before the next attempt — REPLACING the computed backoff for
+     *  that attempt (a 429 `Retry-After`, say). Falls back to the normal backoff
+     *  when the error carries no hint. Default `false`. */
+    readonly respectRetryAfter?: boolean;
+    /** Free-form note rendered as a chip in the dashboard. */
     readonly note?: string;
 }
 
@@ -637,7 +653,13 @@ export declare interface WorkflowResolvedRun {
  * open-text on purpose (future phases add new types without a column
  * migration) — but at emit time we still want type-safety.
  */
-export declare type WorkflowRunEventType = 'run-started' | 'run-suspended' | 'run-resumed' | 'run-cancelled' | 'run-succeeded' | 'run-failed'
+export declare type WorkflowRunEventType = 'run-started' | 'run-suspended' | 'run-resumed'
+/** Emitted when an operator re-drives a terminally-FAILED run from its
+*  durable journal (`voltro workflows redrive` / `ctx.workflows.redrive`).
+*  `payload` carries `{ reason, activitiesReset }` — how many failed step
+*  attempts were reset so they re-execute (completed steps replay). Distinct
+*  from `run-resumed`, which continues a *suspended* run. */
+| 'run-redriven' | 'run-cancelled' | 'run-succeeded' | 'run-failed'
 /** Emitted when `sleep({ name, duration })` enters the wait — `payload`
 *  carries `{ name, durationMs, scheduledWakeAt }`. The dashboard
 *  Gantt renders these as a hatched bar in the timeline so a
@@ -843,6 +865,25 @@ export declare interface WorkflowVersionOptions {
     readonly version?: string | number;
     readonly compatibleWith?: ReadonlyArray<string | number>;
     readonly patches?: ReadonlyArray<string>;
+    /**
+     * When `true`, a workflow whose top-level body FAILS does not become a
+     * terminal `failed` run — it **suspends** with its durable journal intact, so
+     * `voltro workflows resume <id>` (or `ctx.workflows.resume`) re-drives it from
+     * the point of failure: every completed activity replays from the journal
+     * (NOT re-executed) and only the failed activity runs again. This is the
+     * durable-execution way to make a workflow recoverable across a transient
+     * downstream outage without re-doing prior work.
+     *
+     * Default `false` — a failure is terminal (the dead-letter state; see
+     * `voltro workflows list --dead-letter`). Choose per workflow: `true` for a
+     * long multi-step pipeline where re-doing step 1..N-1 is expensive or unsafe;
+     * `false` for a short idempotent job where a fresh `retry` is simpler. A
+     * suspended-on-failure run shows up under `--status suspended`, NOT in the
+     * dead-letter view — it is recoverable, not dead.
+     *
+     * Maps to `@effect/workflow`'s `SuspendOnFailure` annotation.
+     */
+    readonly suspendOnFailure?: boolean;
 }
 
 export declare const WorkflowVersionTypeId: unique symbol;

package/dist/index.js

@@ -1,3 +1,3 @@
-import { C as e, E as t, T as n, _ as r, a as i, b as a, c as o, d as s, f as c, g as l, h as u, i as d, l as f, m as p, n as m, o as h, p as g, r as _, s as v, t as y, u as b, v as x, w as S, x as C, y as w } from "./primitives-CWy1iu5w.js";
-import { _ as T, a as E, b as D, c as O, d as k, f as A, g as j, h as M, i as N, l as P, m as F, n as I, o as L, p as R, r as z, s as B, t as V, u as H, v as U, x as W, y as G } from "./src-DQww25rP.js";
+import { C as e, E as t, T as n, _ as r, a as i, b as a, c as o, d as s, f as c, g as l, h as u, i as d, l as f, m as p, n as m, o as h, p as g, r as _, s as v, t as y, u as b, v as x, w as S, x as C, y as w } from "./primitives-Dgu3O55Q.js";
+import { _ as T, a as E, b as D, c as O, d as k, f as A, g as j, h as M, i as N, l as P, m as F, n as I, o as L, p as R, r as z, s as B, t as V, u as H, v as U, x as W, y as G } from "./src-KDx7NTsa.js";
 export { a as CurrentWorkflowExecutionId, C as CurrentWorkflowRunId, y as WorkflowMessagesProperty, e as WorkflowRunRecorder, S as WorkflowStepInterceptorTag, m as WorkflowVersionTypeId, _ as WorkflowWorkerLayerTypeId, U as _voltroWorkflowRunEventsTable, G as _voltroWorkflowRunStepsTable, D as _voltroWorkflowRunsTable, T as _voltroWorkflowStartContextsTable, j as awaitSignal, P as awaitSignalSuspending, O as awaitUpdate, z as closeWorkflowChildrenForParent, H as completeSuspendingSignal, d as durableClock, i as durableQueue, h as durableQueueModule, v as durableRateLimiterModule, n as getCurrentWorkflowExecutionId, t as getCurrentWorkflowRunId, o as getWorkflowVersionMetadata, W as inMemoryWorkflowEngineLayer, I as inspectWorkflow, f as isWorkflowWorkerLayer, V as makeInMemoryRecorder, N as makeWorkflowRunRecorder, A as makeWorkflowUpdateId, b as processQueue, s as queueWorker, c as rateLimit, R as resolveWorkflowMessageRun, F as sendWorkflowSignal, M as sendWorkflowUpdate, E as serialiseWorkflowRowForWire, g as sleep, p as step, u as stepIdempotencyKey, l as stepModule, k as suspendingSignalDeferredName, L as truncateWorkflowValue, r as withCompensation, x as workflow, w as workflowModule, B as wrapWorkflowExecuteWithRunRecording };