@melra/runtime-core 0.3.0-alpha.11 → 0.3.0-alpha.13

package/dist/workflow-controller.js

@@ -5,6 +5,17 @@ import { WorkflowDefinitionSchema, WorkflowEventSchema, WorkflowRunSchema, Workf
 import { canonicalJson, redactStructuredValue, sha256, } from "@melra/receipt-schema";
 import { applyWorkflowEvent, rebuildWorkflow, } from "./workflow-events.js";
 import { readyNodeIds, validateWorkflow, } from "./workflow-graph.js";
+const RECOVERABLE = [
+    "planned",
+    "running",
+    "awaiting_approval",
+    "recovery_required",
+];
+// A failed run's advance runs its compensations, so an advance that died on
+// one leaves the same hazard as any other: a compensation it started, planned
+// and run again. Repaired too, but only once an expired lease says an advance
+// died there; the full startup pass leaves failed runs as they are.
+const ABANDONABLE = [...RECOVERABLE, "failed"];
 export class WorkflowController {
     store;
     tasks;
@@ -15,6 +26,12 @@ export class WorkflowController {
     // process from racing itself without touching the disk on every call.
     advances = new Map();
     owner = `${process.pid}:${randomUUID()}`;
+    // Distinct from `owner`, so this process's own advance in flight also keeps
+    // its recovery away from the run.
+    recoveryOwner = `${this.owner}:recovery`;
+    // Runs an advance in this process holds the lease for. Recovery leaves them
+    // alone even once the lease has lapsed, say across a laptop's sleep.
+    leased = new Set();
     constructor(store, tasks, payloadCipher, leaseDurationMs = 60_000) {
         this.store = store;
         this.tasks = tasks;
@@ -53,7 +70,18 @@ export class WorkflowController {
         });
         const planned = applyWorkflowEvent(draft, plannedEvent);
         const redacted = WorkflowDefinitionSchema.parse(redactStructuredValue(parsed).value);
-        this.store.createWorkflow(redacted, this.payloadCipher.seal(parsed, `workflow:${parsed.id}:${parsed.version}:definition`), planned, [created, plannedEvent]);
+        // A definition is stored once per id and version, and any number of runs
+        // may use it: planning the same file twice is a second run, not an error.
+        // The same id and version with different content is refused, because
+        // runs already planned from the stored one must keep meaning what they did.
+        const purpose = `workflow:${parsed.id}:${parsed.version}:definition`;
+        const stored = this.store.getWorkflowPayload(parsed.id, parsed.version);
+        if (stored !== undefined &&
+            canonicalJson(WorkflowDefinitionSchema.parse(this.payloadCipher.open(stored, purpose))) !==
+                canonicalJson(parsed)) {
+            throw new Error(`workflow_definition_conflict:${parsed.id}@${parsed.version}: a different definition is stored under this id and version; bump "version"`);
+        }
+        this.store.createWorkflow(redacted, stored === undefined ? this.payloadCipher.seal(parsed, purpose) : undefined, planned, [created, plannedEvent]);
         return planned;
     }
     status(workflowId) {
@@ -86,7 +114,40 @@ export class WorkflowController {
     // start duplicate side effects while this one is mid-flight. `transitionWorkflow`
     // still guards the commit; this guards the work leading up to it.
     async leasedAdvance(workflowId, approvals, inputs) {
-        this.status(workflowId);
+        const current = this.status(workflowId);
+        // Refused before the lease is touched: taking it would replace, and the
+        // release then delete, an expired lease a dead advance left, which is the
+        // only sign the run still needs recovering once it is resumed.
+        if (["paused", "suspended"].includes(current.status)) {
+            throw new Error(`workflow_halted:${current.status}`);
+        }
+        // Every task would refuse these anyway, but only once its node's turn came,
+        // after siblings in the same wave had already been cancelled, planned
+        // again or run on the strength of a request that is refused. Refused
+        // before anything moves.
+        if (this.tasks.enforced && approvals.length > 0)
+            throw new Error("approval_in_band_refused");
+        // An expired lease someone else holds is an advance that died mid-flight.
+        // Taking it over would read a projection from before that advance's tasks
+        // and plan them again under keys that were never committed, so the run is
+        // recovered first. A lease left exactly as it was means a task of that
+        // advance is still running somewhere: wait, as for a live holder.
+        const left = this.store.getWorkflowLease(workflowId);
+        if (left !== undefined &&
+            left.owner !== this.owner &&
+            left.expiresAt <= new Date().toISOString() &&
+            ABANDONABLE.includes(current.status)) {
+            // A repaired run has no lease left. One still there is a task still
+            // running, a live process that got in first, or a repair that failed.
+            // Only this run's own failure is reported as itself; the rest of the
+            // pass's belong to other runs or tasks and reach the sweep's warnings.
+            const pass = await this.recoveryPass(true);
+            if (this.store.getWorkflowLease(workflowId) !== undefined) {
+                if (pass.runFailures.has(workflowId))
+                    throw pass.runFailures.get(workflowId);
+                throw new Error("workflow_lease_held");
+            }
+        }
         const now = Date.now();
         if (!this.store.acquireWorkflowLease(workflowId, this.owner, new Date(now).toISOString(), new Date(now + this.leaseDurationMs).toISOString())) {
             throw new Error("workflow_lease_held");
@@ -97,10 +158,12 @@ export class WorkflowController {
             this.store.renewWorkflowLease(workflowId, this.owner, new Date(Date.now() + this.leaseDurationMs).toISOString());
         }, Math.max(1_000, Math.floor(this.leaseDurationMs / 2)));
         renewal.unref();
+        this.leased.add(workflowId);
         try {
             return await this.advanceOnce(workflowId, approvals, inputs);
         }
         finally {
+            this.leased.delete(workflowId);
             clearInterval(renewal);
             this.store.releaseWorkflowLease(workflowId, this.owner);
         }
@@ -137,7 +200,7 @@ export class WorkflowController {
             ["failed", "recovery_required"].includes(current.nodes[node.id]?.status ?? ""))) {
             return { run: current, tasks: [], events: [] };
         }
-        // A node parked for a human — approval phrase or supplied input — is no
+        // A node parked for a human — approval or supplied input — is no
         // longer `pending`, so `readyNodeIds` will not return it. Re-offer it here
         // or the workflow would stall forever with the answer in hand.
         const awaiting = [...graph.nodes.values()]
@@ -153,6 +216,11 @@ export class WorkflowController {
         if (nodeIds.length === 0) {
             return { run: current, tasks: [], events: [] };
         }
+        // A sibling's refusal is reported only after the others' changes are
+        // committed: a node another sibling cancelled and planned again in this
+        // wave would otherwise keep pointing at the task it cancelled, and stay
+        // cancelled for good.
+        let refused;
         const results = await (rollback.length >
             0
             ? // Compensations undo each other's preconditions, so the order they run
@@ -163,13 +231,19 @@ export class WorkflowController {
                 // continuing past a failed unwind would strand the saga in a state no
                 // node describes.
                 this.unwind(rollback, graph, definition, current, approvals, inputs)
-            : Promise.all(nodeIds.map(async (nodeId) => {
+            : Promise.allSettled(nodeIds.map(async (nodeId) => {
                 const node = graph.nodes.get(nodeId);
                 return {
                     nodeId,
                     result: await this.advanceNode(node, definition, current.nodes[nodeId], current, approvals, inputs),
                 };
-            })));
+            })).then((settled) => {
+                // Every sibling finishes before one's error is reported. Rejecting
+                // at the first throw released the lease while other adapters still
+                // ran, and a retry then read their tasks as failed.
+                refused = settled.find((outcome) => outcome.status === "rejected");
+                return settled.flatMap((outcome) => (outcome.status === "fulfilled" ? [outcome.value] : []));
+            }));
         // A *compensation* that failed must not open a second unwind. The saga is
         // already stranded, and undoing the next step while this inverse is still
         // outstanding is precisely what the halt exists to prevent — so only a
@@ -276,78 +350,223 @@ export class WorkflowController {
                 createdAt: run.updatedAt,
             }));
         }
+        if (refused !== undefined)
+            throw refused.reason;
         return {
             run,
             tasks: results.flatMap(({ result }) => result.tasks),
             events: this.events(workflowId, current.stateVersion),
         };
     }
-    async recoverInterrupted() {
-        await this.tasks.recoverInterrupted();
+    /**
+     * Repairs runs whose advance is not coming back. A run is touched only under
+     * its lease, so a live advance in any process (this one included) keeps its
+     * run; and only when none of its tasks is still running, which after the
+     * task pass means a live process owns that task. `abandonedOnly` limits the
+     * pass to runs whose advance died holding the lease — the periodic sweep.
+     */
+    async recoverInterrupted({ abandonedOnly = false } = {}) {
+        // A task or run that cannot be recovered is reported once everything else
+        // has been: one bad record must not hold up the rest.
+        const pass = await this.recoveryPass(abandonedOnly);
+        if (pass.failures.length > 0)
+            throw pass.failures[0];
+        return pass.recovered;
+    }
+    async recoveryPass(abandonedOnly) {
+        const taskFailures = new Map();
+        const failures = [];
+        await this.tasks.recoverInterrupted(taskFailures).catch((error) => {
+            failures.push(error);
+        });
+        const runFailures = new Map();
+        const recovered = this.repairRuns(abandonedOnly, taskFailures, runFailures);
+        failures.push(...taskFailures.values(), ...runFailures.values());
+        return { recovered, failures, runFailures };
+    }
+    /**
+     * Repairs each run under its lease. `runFailures` gets each run's own reason
+     * for not being repaired, including a task of its that could not be.
+     */
+    repairRuns(abandonedOnly, taskFailures, runFailures) {
+        const at = Date.now();
+        // The sweep reads only runs with an expired lease, so a home's history of
+        // finished and failed runs costs it nothing.
+        const runs = abandonedOnly
+            ? this.store
+                .expiredWorkflowLeases(new Date(at).toISOString())
+                .map((id) => this.store.getWorkflowRun(id))
+                .filter((run) => run !== undefined && ABANDONABLE.includes(run.status))
+            : this.store.listWorkflowRuns(RECOVERABLE);
+        if (runs.length === 0)
+            return [];
         const recovered = [];
         const allTasks = this.store.listTasks(100_000);
-        for (const stored of this.store.listWorkflowRuns([
-            "planned",
-            "running",
-            "awaiting_approval",
-            "recovery_required",
-        ])) {
-            const replayed = this.replay(stored);
-            if (replayed.stateVersion !== stored.stateVersion) {
-                throw new Error("workflow_event_history_invalid");
+        for (const stored of runs) {
+            if (this.leased.has(stored.id))
+                continue;
+            let definition;
+            try {
+                definition = this.loadDefinition(stored);
+            }
+            catch (error) {
+                runFailures.set(stored.id, error);
+                continue;
+            }
+            const keys = new Set(definition.nodes.flatMap((node) => expectedTaskKeys(stored.id, node, definition)));
+            // Skipped runs keep any abandoned lease, which is how the sweep finds
+            // them again once that task settles.
+            const unsettled = allTasks.filter((task) => task.idempotencyKey !== undefined &&
+                keys.has(task.idempotencyKey) &&
+                (task.status === "running" || task.status === "verifying"));
+            if (unsettled.length > 0) {
+                const failed = unsettled.find((task) => taskFailures.has(task.id));
+                if (failed !== undefined)
+                    runFailures.set(stored.id, taskFailures.get(failed.id));
+                continue;
+            }
+            const taken = this.store.takeWorkflowLease(stored.id, this.recoveryOwner, new Date(at).toISOString(), new Date(at + this.leaseDurationMs).toISOString());
+            if (taken === "refused")
+                continue;
+            let repaired = false;
+            let failed = false;
+            try {
+                // Changed since the listing, by an advance or by an operator's pause,
+                // so the tasks read above may be stale.
+                if (this.store.getWorkflowRun(stored.id)?.stateVersion === stored.stateVersion) {
+                    recovered.push(this.recoverRun(stored, definition, allTasks));
+                    repaired = true;
+                }
+            }
+            catch (error) {
+                failed = true;
+                runFailures.set(stored.id, error);
+            }
+            finally {
+                // A run not repaired keeps a lease, expired rather than deleted, when
+                // it still needs one: its repair failed, or the lease it replaced was
+                // a dead advance's or an earlier repair's. That lease is how the sweep
+                // and the next advance know, and without it an advance would plan its
+                // tasks again. A lease this pass created replaced nothing: whoever
+                // held one last released it with the run left straight.
+                if (repaired || (!failed && taken === "created")) {
+                    this.store.releaseWorkflowLease(stored.id, this.recoveryOwner);
+                }
+                else {
+                    this.store.renewWorkflowLease(stored.id, this.recoveryOwner, new Date(0).toISOString());
+                }
+            }
+        }
+        return recovered;
+    }
+    recoverRun(stored, definition, allTasks) {
+        const replayed = this.replay(stored);
+        if (replayed.stateVersion !== stored.stateVersion) {
+            throw new Error("workflow_event_history_invalid");
+        }
+        return this.transition(stored, (draft, emit) => {
+            for (const node of definition.nodes) {
+                // An advance on a failed run only ever runs compensations, so only
+                // they can be stale. Re-deriving the forward nodes would rank their
+                // tasks differently from the live path and could take the run out of
+                // failed: a branch awaiting approval beside a failed one reopened.
+                if (stored.status === "failed" && node.type !== "compensation")
+                    continue;
+                const found = expectedTaskKeys(stored.id, node, definition).map((key) => preferredTask(allTasks, key));
+                let state = recoveredNodeState(node, found.filter((task) => task !== undefined), replayed.nodes[node.id]);
+                // An operation behind an approval shares its task with the approval
+                // node. Waiting for the approval is that node's state; mirrored onto
+                // the operation, both were re-offered, the approval ran the effect,
+                // and the operation read the running task as failed.
+                if (node.type === "operation" &&
+                    state.status === "awaiting_approval" &&
+                    definition.nodes.some((other) => other.type === "approval" && other.forNodeId === node.id)) {
+                    state = replayed.nodes[node.id];
+                }
+                // A sequence whose finished steps all verified is not finished if
+                // steps remain: a crash between a loop's iterations, or between a
+                // branch's steps, recovered as complete with the rest never run.
+                // Pending, so the next advance carries on from the tasks it has.
+                // Unless the projection recorded it finished in a way only the live
+                // path finishes one short: a loop stopped by `until` at a whole
+                // iteration, or a condition whose chosen branch is empty. Reopened,
+                // `until` or the condition would be asked again of a world that has
+                // moved on. Anything else recorded finished but short was written by
+                // a recovery that never downgraded (alpha.12), and is downgraded now.
+                const recorded = replayed.nodes[node.id];
+                const finishedShort = recorded.status === "verified_complete" &&
+                    canonicalJson(recorded.taskIds) === canonicalJson(state.taskIds) &&
+                    ((node.type === "bounded_loop" &&
+                        node.until !== undefined &&
+                        state.taskIds.length > 0 &&
+                        state.taskIds.length % node.body.length === 0) ||
+                        (node.type === "condition" && state.taskIds.length === 0));
+                if (state.status === "verified_complete" && !sequenceFinished(node, found) && !finishedShort) {
+                    state = {
+                        status: "pending",
+                        taskIds: state.taskIds,
+                        // Whole iterations only: the loop resumes at the one in progress.
+                        ...(node.type === "bounded_loop"
+                            ? { iterations: Math.floor(state.taskIds.length / node.body.length) }
+                            : {}),
+                    };
+                }
+                // A loop that stopped on a step keeps the iteration it stopped in, as
+                // runLoop records it. Without it the next advance starts again at
+                // iteration 0 and asks `until` of a task long finished.
+                if (node.type === "bounded_loop" && state.iterations === undefined && state.taskIds.length > 0) {
+                    state = { ...state, iterations: Math.floor((state.taskIds.length - 1) / node.body.length) };
+                }
+                if (canonicalJson(draft.nodes[node.id]) === canonicalJson(state)) {
+                    continue;
+                }
+                const from = draft.nodes[node.id].status;
+                draft.nodes[node.id] = state;
+                emit("workflow.node_changed", {
+                    nodeId: node.id,
+                    from,
+                    state,
+                });
             }
-            const definition = this.loadDefinition(stored);
-            let run = this.transition(stored, (draft, emit) => {
+            let status = this.deriveStatus(draft, definition);
+            if (status === "verified_complete") {
                 for (const node of definition.nodes) {
-                    const state = recoveredNodeState(node, expectedTaskKeys(stored.id, node, definition)
-                        .map((key) => preferredTask(allTasks, key))
-                        .filter((task) => task !== undefined), replayed.nodes[node.id]);
-                    if (canonicalJson(draft.nodes[node.id]) === canonicalJson(state)) {
+                    const state = draft.nodes[node.id];
+                    if (node.type !== "compensation" || state.status !== "pending") {
                         continue;
                     }
-                    const from = draft.nodes[node.id].status;
-                    draft.nodes[node.id] = state;
+                    state.status = "skipped";
                     emit("workflow.node_changed", {
                         nodeId: node.id,
-                        from,
+                        from: "pending",
                         state,
                     });
                 }
-                let status = this.deriveStatus(draft, definition);
-                if (status === "verified_complete") {
-                    for (const node of definition.nodes) {
-                        const state = draft.nodes[node.id];
-                        if (node.type !== "compensation" || state.status !== "pending") {
-                            continue;
-                        }
-                        state.status = "skipped";
-                        emit("workflow.node_changed", {
-                            nodeId: node.id,
-                            from: "pending",
-                            state,
-                        });
-                    }
-                    status = this.deriveStatus(draft, definition);
-                }
-                const from = draft.status;
-                draft.status = status;
-                if (status === "recovery_required") {
-                    draft.error = "workflow_contains_unreconciled_mutation";
-                }
-                else {
-                    delete draft.error;
-                }
-                emit(status === "recovery_required"
-                    ? "workflow.recovery_required"
-                    : "workflow.recovered", {
-                    from,
-                    to: status,
-                    ...(draft.error === undefined ? {} : { error: draft.error }),
-                });
+                status = this.deriveStatus(draft, definition);
+            }
+            const from = draft.status;
+            draft.status = status;
+            // Named the way advanceOnce names a stalled unwind, so a run reads the
+            // same whether its process died or not.
+            const stalled = definition.nodes.find((node) => node.type === "compensation" &&
+                ["failed", "recovery_required"].includes(draft.nodes[node.id]?.status ?? ""));
+            if (status === "recovery_required") {
+                draft.error =
+                    stalled === undefined
+                        ? "workflow_contains_unreconciled_mutation"
+                        : `workflow_compensation_incomplete:${stalled.id}`;
+            }
+            else {
+                delete draft.error;
+            }
+            emit(status === "recovery_required"
+                ? "workflow.recovery_required"
+                : "workflow.recovered", {
+                from,
+                to: status,
+                ...(draft.error === undefined ? {} : { error: draft.error }),
             });
-            recovered.push(run);
-        }
-        return recovered;
+        });
     }
     // Operator controls. `pause` is a soft stop an operator lifts with `resume`;
     // `suspend` is the same stop for an operator who wants the run parked
@@ -365,18 +584,22 @@ export class WorkflowController {
         if (!["paused", "suspended"].includes(current.status)) {
             throw new Error("workflow_not_paused");
         }
+        const definition = this.loadDefinition(current);
         return this.transition(current, (draft, emit) => {
             const from = draft.status;
             // Restore whatever the graph was actually blocked on before the halt,
-            // so resuming a run parked on a human does not claim it is running.
-            const nodes = Object.values(draft.nodes);
-            draft.status = nodes.some((node) => node.status === "awaiting_approval")
-                ? "awaiting_approval"
-                : nodes.some((node) => node.status === "awaiting_input")
-                    ? "awaiting_input"
-                    : "running";
-            delete draft.error;
-            emit("workflow.resumed", { from, to: draft.status });
+            // so resuming a run parked on a human, or on an effect whose outcome is
+            // unknown, does not claim it is running.
+            draft.status = this.deriveStatus(draft, definition);
+            if (draft.status !== "recovery_required")
+                delete draft.error;
+            else
+                draft.error ??= "workflow_contains_unreconciled_mutation";
+            emit("workflow.resumed", {
+                from,
+                to: draft.status,
+                ...(draft.error === undefined ? {} : { error: draft.error }),
+            });
         });
     }
     halt(workflowId, to, eventType) {
@@ -389,7 +612,9 @@ export class WorkflowController {
         return this.transition(current, (draft, emit) => {
             const from = draft.status;
             draft.status = to;
-            emit(eventType, { from, to });
+            // Kept: a run halted while it needed recovery still needs it, and the
+            // reducer drops an error the event does not carry.
+            emit(eventType, { from, to, ...(draft.error === undefined ? {} : { error: draft.error }) });
         });
     }
     cancel(workflowId) {
@@ -513,15 +738,16 @@ export class WorkflowController {
                 if (sourceTaskId === undefined) {
                     throw new Error(`workflow_condition_source_unavailable:${node.id}`);
                 }
-                const condition = await this.tasks.verifyPersisted(sourceTaskId, [
-                    node.predicate,
-                ]);
-                return await this.runSequential(condition.verified ? node.whenTrue : node.whenFalse, state, approvals, {
-                    workflowId: run.id,
-                    nodeId: node.id,
-                    iteration: 0,
-                    branch: condition.verified ? "true" : "false",
-                });
+                // Decided once. A branch that has started keeps going: re-evaluating a
+                // live predicate on a later advance could flip it and reuse the other
+                // branch's tasks by position.
+                const identity = { workflowId: run.id, nodeId: node.id, iteration: 0 };
+                const started = state.taskIds[0] === undefined ? undefined : this.tasks.status(state.taskIds[0]).idempotencyKey;
+                const chosen = started === undefined
+                    ? (await this.tasks.verifyPersisted(sourceTaskId, [node.predicate])).verified
+                    : node.whenTrue.length > 0 &&
+                        started === taskIdempotencyKey({ ...identity, branch: "true:0" }, node.whenTrue[0]);
+                return await this.runSequential(chosen ? node.whenTrue : node.whenFalse, state, approvals, { ...identity, branch: chosen ? "true" : "false" });
             }
             case "parallel":
                 return await this.runParallel(node.branches, state, approvals, run.id, node.id);
@@ -623,11 +849,29 @@ export class WorkflowController {
             ? this.tasks.plan(request, {
                 idempotencyKey: taskIdempotencyKey(identity, request),
                 attempt: 1,
+                stored: true,
             })
             : this.tasks.status(existingTaskId);
+        // A challenge that expired before anyone answered it can never be
+        // answered, and reusing it left the node unapprovable for good. Ask again:
+        // the expired task is cancelled and the same request planned afresh.
+        // A challenge an approver answered in time is not asked again, however
+        // long the answer took to act on.
+        const decided = task.status === "awaiting_approval" && this.tasks.approvalDecision(task) !== undefined;
+        if (task.status === "awaiting_approval" &&
+            task.approval !== undefined &&
+            Date.parse(task.approval.expiresAt) <= Date.now() &&
+            !decided) {
+            this.tasks.cancel(task.id);
+            task = this.tasks.plan(request, {
+                idempotencyKey: taskIdempotencyKey(identity, request),
+                attempt: (task.attempt ?? 1) + 1,
+                stored: true,
+            });
+        }
         if (task.status === "awaiting_approval") {
             const approval = approvals.find((item) => item.approvalId === task.approval?.approvalId);
-            if (approval === undefined) {
+            if (approval === undefined && !decided) {
                 return {
                     state: {
                         status: "awaiting_approval",
@@ -688,6 +932,16 @@ export class WorkflowController {
         const taskIds = [...state.taskIds];
         const tasks = [];
         let iterations = state.iterations ?? 0;
+        // Resumed after recovery between iterations: the last one finished but its
+        // `until` was never checked, and running another would be one too many.
+        const finished = taskIds[iterations * node.body.length - 1];
+        if (iterations > 0 &&
+            taskIds.length === iterations * node.body.length &&
+            finished !== undefined &&
+            node.until !== undefined &&
+            (await this.tasks.verifyPersisted(finished, [node.until])).verified) {
+            return { state: { status: "verified_complete", taskIds, iterations }, tasks };
+        }
         while (iterations < node.maxIterations) {
             const offset = iterations * node.body.length;
             for (const [index, request] of node.body.entries()) {
@@ -881,6 +1135,25 @@ function expectedTaskKeys(workflowId, node, definition) {
             return [];
     }
 }
+/**
+ * Whether a recovered node that runs steps in sequence ran all of them, given
+ * its expected tasks in `expectedTaskKeys` order (missing ones undefined).
+ */
+function sequenceFinished(node, found) {
+    if (node.type === "condition") {
+        // Only one branch ever runs; finished means the branch that ran is whole.
+        const whenTrue = found.slice(0, node.whenTrue.length);
+        const whenFalse = found.slice(node.whenTrue.length);
+        const whole = (steps) => steps.length > 0 && steps.every(Boolean);
+        return whole(whenTrue) || whole(whenFalse);
+    }
+    if (node.type === "bounded_loop") {
+        // Recovery cannot evaluate `until`, so short of the last iteration it is
+        // left for the loop itself to decide.
+        return found.every(Boolean);
+    }
+    return true;
+}
 function preferredTask(tasks, idempotencyKey) {
     return tasks
         .filter((task) => task.idempotencyKey === idempotencyKey)
@@ -894,6 +1167,23 @@ function recoveredNodeState(node, tasks, previous) {
         return previous;
     const taskIds = tasks.map((task) => task.id);
     const withStatus = (status) => tasks.find((task) => task.status === status);
+    // Ranked as aggregateNodeState ranks a live node, failed first. A parallel
+    // node with one failed branch and one awaiting approval was failed live and
+    // came back from recovery awaiting approval, reopening the waiting branch on
+    // a run whose rollback had begun.
+    const failed = tasks.find((task) => [
+        "failed",
+        "partial",
+        "budget_exhausted",
+        "policy_blocked",
+    ].includes(task.status));
+    if (failed !== undefined) {
+        return {
+            status: "failed",
+            taskIds,
+            ...(failed.error === undefined ? {} : { error: failed.error }),
+        };
+    }
     const recovery = withStatus("recovery_required");
     if (recovery !== undefined) {
         return {
@@ -912,19 +1202,6 @@ function recoveredNodeState(node, tasks, previous) {
                 : { approval: awaiting.approval }),
         };
     }
-    const failed = tasks.find((task) => [
-        "failed",
-        "partial",
-        "budget_exhausted",
-        "policy_blocked",
-    ].includes(task.status));
-    if (failed !== undefined) {
-        return {
-            status: "failed",
-            taskIds,
-            ...(failed.error === undefined ? {} : { error: failed.error }),
-        };
-    }
     if (tasks.some((task) => task.status === "planned")) {
         return {
             status: "pending",