@inflexa-ai/harness 0.37.0 → 0.38.0

package/dist/workflows/execute-analysis.js

@@ -31,23 +31,30 @@
  *     - await child results via `getResult` (these are themselves cached
  *       in DBOS, so parent recovery does NOT re-run completed children)
  *  3. Failure isolation
- *     - a child settling `failed`/`blocked` (or throwing for a non-budget
- *       cause) dooms only its transitive dependents: they can never become
- *       dependency-satisfied, are marked `skipped` on the dag-state part, and
- *       are never dispatched. In-flight siblings and independent ready steps
- *       continue; the loop drains and the run finalises (typically `partial`).
+ *     - a child settling `failed`/`blocked` (or throwing for a cause that is
+ *       not a suspension) dooms only its transitive dependents: they can never
+ *       become dependency-satisfied, are marked `skipped` on the dag-state
+ *       part, and are never dispatched. In-flight siblings and independent
+ *       ready steps continue; the loop drains and the run finalises (typically
+ *       `partial`).
  *     - the halt cascade (cancel in-flight via
  *       `Promise.allSettled(handles.map(h => DBOS.cancelWorkflow(h.workflowID)))`,
- *       stop scheduling) survives only for the budget paths: a
- *       `budget_exceeded` settlement and the `neverFits` plan-validation guard.
- *  4. Pause cascade (402)
- *     - on a child cancelling itself with `budget_exceeded`: cancel siblings
- *       then self-cancel the parent to `CANCELLED` (NOT `ERROR` — ERROR
- *       isn't resumable; NOTES #3)
+ *       stop scheduling) survives only for a suspension of a child and the
+ *       `neverFits` plan-validation guard.
+ *  4. Suspension cascade (workflow-suspension spec)
+ *     - a child that suspends sends a typed message on `CHILD_SUSPENDED_TOPIC`
+ *       and cancels itself. The parent reads the kind of the message, never a
+ *       reason string: it cancels the siblings, then self-cancels to
+ *       `CANCELLED` (NOT `ERROR` — ERROR isn't resumable; NOTES #3)
+ *     - a refusal of the `RunCharge.open` gate with the suspend flag suspends
+ *       the run before any step starts
+ *     - a `suspend` error of a synthesis model request suspends the run in
+ *       place of the synthesis failure
  *     - the analysis flips to `suspended_insufficient_funds` in
  *       `collectAndComplete`. The paused parent lands in `CANCELLED`, a
- *       durably reschedulable state; re-driving it after a top-up is a
- *       future enhancement, not yet wired here.
+ *       durably reschedulable state; re-driving it is a future enhancement,
+ *       not yet wired here. The reason of the host rides to the run row, the
+ *       charge close, the metric, and the terminal part, unread.
  *  5. `synthesizeFindings` (single sequential block, no sandbox)
  *  6. `collectAndComplete` (terminal — runs on ALL paths)
  *     - determine final status from completed/cancelled/failed counts
@@ -62,15 +69,17 @@
  *       via `inspectRun`)
  */
 import { DBOS, Error as DBOSErrors } from "@dbos-inc/dbos-sdk";
+import { ResultAsync, err, ok } from "neverthrow";
 import { mkdir } from "node:fs/promises";
 import { join } from "node:path";
 import { forStep } from "../auth/types.js";
 import { createNoopLogger } from "../lib/console-logger.js";
+import { deliverNotice, passGate } from "../lib/hooks.js";
 import { ATTR_INFLEXA_STEP_ID, stableSpan } from "../lib/otel-spans.js";
 import { recordRunCompleted, recordStepCompleted, stepOutcomeOf } from "../lib/metrics.js";
+import { describeDbError } from "../lib/db-result.js";
 import { unwrapOrThrow } from "../lib/result.js";
-import { RunDedupCollisionError, countArtifactsForRun, insertStepExecution, loadDataProfileStatus, queryActiveRun, queryStepArtifactPaths, queryStepsByRun, seedStepExecutions, setRunSynthesisOutcome, suspendAnalysis as suspendAnalysisQuery, sweepPendingStepExecutions, updateRunStatus, updateStepExecution, } from "../state/index.js";
-import { isBudgetExceeded } from "../loop/budget-exceeded.js";
+import { countArtifactsForRun, insertStepExecution, loadDataProfileStatus, queryActiveRun, queryStepArtifactPaths, queryStepsByRun, seedStepExecutions, setRunSynthesisOutcome, suspendAnalysis as suspendAnalysisQuery, sweepPendingStepExecutions, updateRunStatus, updateStepExecution, } from "../state/index.js";
 import { addChatUsage, hasReportedUsage } from "../loop/metrics.js";
 import { SYNTHESIS_AGENT_ID, loadStepSummariesFromDisk } from "../execution/run-synthesis.js";
 import { MAX_UPSTREAM_ARTIFACTS, composeStepBriefing } from "../prompts/briefing.js";
@@ -79,12 +88,26 @@ import { isChatDataPart } from "../sandbox/sandbox-step-translate.js";
 import { synthesizeRun } from "../app/synthesize-run.js";
 import { computeTopologicalLevels, scheduleReady, validatePlanDag } from "./execute-analysis-scheduler.js";
 import { recordCancelledChild } from "./metrics.js";
-import { BUDGET_EXCEEDED_TOPIC } from "./sandbox-step.js";
+import { CHILD_SUSPENDED_TOPIC } from "./sandbox-step.js";
+import { cancelSelf, suspensionOfFailure, suspensionOfRefusal } from "./suspension.js";
 // ── In-body orchestration helpers ────────────────────────────────────
 /** Emit a chat data part on the parent's DBOS stream. */
 function emitStreamPart(part) {
     return DBOS.writeStream("events", part);
 }
+/**
+ * A failed write rolls back no side effect that succeeded and never fails the
+ * run. A DBOS exception passes through with no change.
+ */
+async function ledgerStep(log, name, write) {
+    await DBOS.runStep(async () => {
+        const written = await write();
+        if (written.isOk())
+            return ok(undefined);
+        log.named(name).error("ledger write failed", { error: describeDbError(written.error), ...log.errorFields(written.error.cause) });
+        return err(describeDbError(written.error));
+    }, { name });
+}
 /**
  * Fire the optional run-lifecycle provenance observer, isolating a throwing
  * host observer from the workflow. The harness is host-agnostic and cannot
@@ -281,10 +304,13 @@ async function collectUpstreamHandoffs(args) {
  * its reason in `blockedReason` — an honest agent-declared blocker, not an
  * error — and a failure carries the same reason string
  * `persist-synthesis-outcome` writes to `cortex_runs.synthesis_reason`, so the
- * two ledgers cannot disagree about why synthesis died. Exported for the
- * projection tests (the `buildChildInput` pattern).
+ * two ledgers cannot disagree about why synthesis died. A suspension is not an
+ * outcome: the row is canceled, as the row of a suspended step. Exported for
+ * the projection tests (the `buildChildInput` pattern).
  */
 export function synthesisRowUpdate(outcome, durationMs) {
+    if ("kind" in outcome)
+        return { status: "canceled", durationMs, error: outcome.reason };
     switch (outcome.status) {
         case "produced":
             return { status: "completed", durationMs };
@@ -394,6 +420,29 @@ export async function runExecuteAnalysisBody(input, deps) {
             canceledSteps: [],
         };
     }
+    if (init.kind === "charge-refused") {
+        const suspension = suspensionOfRefusal(init.refusal) ?? null;
+        const refusedAtMs = await DBOS.now();
+        const refused = await collectAndComplete({
+            input,
+            runId,
+            workflowId,
+            startedAtMs: refusedAtMs,
+            completed: new Set(),
+            failed: new Set(),
+            canceled: new Set(),
+            suspension,
+            failureReason: init.refusal.reason,
+            forceFailed: suspension === null,
+            findings: [],
+            synthesisOutcome: null,
+            deps,
+        });
+        observeRunGuarded(deps, buildRunObservation({ input, runId, status: refused.status, stepStates: new Map() }));
+        if (suspension !== null)
+            return cancelSelf("self-cancel-suspended");
+        return refused;
+    }
     // Seed the full DAG as `pending` rows so ledger readers see every step from
     // run start (honest done/total), not just the ones that began executing.
     // The seed's DO NOTHING conflict action makes this replay-safe; the agent
@@ -462,10 +511,12 @@ export async function runExecuteAnalysisBody(input, deps) {
     // still run the terminal block (close charge + revoke run authorization), and re-throw
     // after so the workflow record goes to ERROR.
     let synthesisError = null;
+    let synthesisSuspension = null;
     let synthesisFindings = [];
     // Null when synthesis never ran (disabled or no completed steps): the ledger
     // columns stay NULL — "unknown", not a recorded skip — and the seeded
-    // `synthesis` row (if any) stays `pending` for the terminal sweep.
+    // `synthesis` row (if any) stays `pending` for the terminal sweep. Also null
+    // when synthesis suspended: a resume of the run gives the outcome.
     let synthesisOutcome = null;
     if (synthesisEnabled && final.completed.size > 0) {
         // Bracketing clock reads for the row's duration_ms — checkpointed, so a
@@ -474,25 +525,16 @@ export async function runExecuteAnalysisBody(input, deps) {
         // Mark the seeded `synthesis` row running. The same upsert the DAG children
         // use: it stamps started_at, and it creates the row outright if a legacy
         // persisted input did not seed it. Log-don't-fail: a progress row must never fail an
-        // otherwise-healthy run. Cancellation still re-propagates — it is not a
-        // write failure, and swallowing it here would only defer it one operation.
-        try {
-            await DBOS.runStep(async () => {
-                unwrapOrThrow(await insertStepExecution(deps.pool, {
-                    runId,
-                    stepId: SYNTHESIS_STEP_ID,
-                    analysisId: input.analysisId,
-                    wave: synthesisWave,
-                    agentId: SYNTHESIS_AGENT_ID,
-                    childWorkflowId: null,
-                }));
-            }, { name: "mark-synthesis-running" });
-        }
-        catch (err) {
-            if (err instanceof DBOSErrors.DBOSWorkflowCancelledError)
-                throw err;
-            logger.error("mark-synthesis-running failed", { runId, ...logger.errorFields(err) });
-        }
+        // otherwise-healthy run.
+        await ledgerStep(logger.with({ runId }), "mark-synthesis-running", () => insertStepExecution(deps.pool, {
+            runId,
+            stepId: SYNTHESIS_STEP_ID,
+            analysisId: input.analysisId,
+            wave: synthesisWave,
+            agentId: SYNTHESIS_AGENT_ID,
+            childWorkflowId: null,
+        }));
+        let rowOutcome;
         try {
             const synthOut = await DBOS.runStep(() => synthesizeFindings({
                 analysisId: input.analysisId,
@@ -503,6 +545,7 @@ export async function runExecuteAnalysisBody(input, deps) {
             }), { name: "synthesize-findings" });
             synthesisFindings = synthOut.findings;
             synthesisOutcome = { status: synthOut.synthesisStatus, reason: synthOut.synthesisReason };
+            rowOutcome = synthesisOutcome;
         }
         catch (err) {
             // Cancellation is not a synthesis outcome: re-propagate unchanged (the
@@ -514,37 +557,38 @@ export async function runExecuteAnalysisBody(input, deps) {
             // `inflexa run` already detects via its DBOS-status cross-check.
             if (err instanceof DBOSErrors.DBOSWorkflowCancelledError)
                 throw err;
-            synthesisError = err;
-            synthesisOutcome = { status: "failed", reason: err instanceof Error ? err.message : String(err) };
-            logger.error("synthesizeFindings failed — failing the run", { runId, ...logger.errorFields(err) });
+            const suspended = suspensionOfFailure(err);
+            if (suspended !== undefined) {
+                synthesisSuspension = suspended;
+                rowOutcome = suspended;
+                logger.warn("synthesizeFindings suspended — suspending the run", { runId, reason: suspended.reason });
+            }
+            else {
+                synthesisError = err;
+                synthesisOutcome = { status: "failed", reason: err instanceof Error ? err.message : String(err) };
+                rowOutcome = synthesisOutcome;
+                logger.error("synthesizeFindings failed — failing the run", { runId, ...logger.errorFields(err) });
+            }
         }
         // Terminal stamp for the `synthesis` row — deliberately BEFORE
         // `collectAndComplete` writes the run row's terminal status, so no reader
-        // can observe a terminal run beside a still-`running` synthesis row. Both
-        // branches above set `synthesisOutcome` (or threw), so it is non-null here
-        // and the compiler narrows it — no guard needed. Log-don't-fail, like the
-        // sibling finalisation writes.
-        const rowOutcome = synthesisOutcome;
-        try {
-            const synthEndedAtMs = await DBOS.now();
-            const rowUpdate = synthesisRowUpdate(rowOutcome, synthEndedAtMs - synthStartedAtMs);
-            await DBOS.runStep(async () => {
-                unwrapOrThrow(await updateStepExecution(deps.pool, runId, SYNTHESIS_STEP_ID, rowUpdate));
-                // Recorded inside the step that writes the row, so a replayed body records nothing again.
-                const outcome = stepOutcomeOf(rowUpdate.status);
-                if (outcome !== undefined)
-                    recordStepCompleted({ agentId: SYNTHESIS_AGENT_ID, status: outcome, durationMs: synthEndedAtMs - synthStartedAtMs });
-            }, { name: "persist-synthesis-step" });
-        }
-        catch (err) {
-            if (err instanceof DBOSErrors.DBOSWorkflowCancelledError)
-                throw err;
-            logger.error("persist-synthesis-step failed", { runId, ...logger.errorFields(err) });
-        }
+        // can observe a terminal run beside a still-`running` synthesis row.
+        // Log-don't-fail, like the sibling finalisation writes.
+        const synthEndedAtMs = await DBOS.now();
+        const rowUpdate = synthesisRowUpdate(rowOutcome, synthEndedAtMs - synthStartedAtMs);
+        await ledgerStep(logger.with({ runId }), "persist-synthesis-step", async () => {
+            const written = await updateStepExecution(deps.pool, runId, SYNTHESIS_STEP_ID, rowUpdate);
+            // Recorded inside the step that writes the row, so a replayed body records nothing again.
+            const outcome = stepOutcomeOf(rowUpdate.status);
+            if (written.isOk() && outcome !== undefined)
+                recordStepCompleted({ agentId: SYNTHESIS_AGENT_ID, status: outcome, durationMs: synthEndedAtMs - synthStartedAtMs });
+            return written;
+        });
     }
     // (6) collectAndComplete — terminal block. Runs on EVERY path. When synthesis
     // failed, force a failed terminal status so the charge/run-authorization close correctly
     // and the run row + stream report the failure before we re-throw.
+    const suspension = final.suspension ?? synthesisSuspension;
     const result = await collectAndComplete({
         input,
         runId,
@@ -553,13 +597,15 @@ export async function runExecuteAnalysisBody(input, deps) {
         completed: final.completed,
         failed: final.failed,
         canceled: final.canceled,
-        budgetExceeded: final.budgetExceeded,
+        suspension,
         ...(final.usage ? { usage: final.usage } : {}),
         failureReason: synthesisError
             ? synthesisError instanceof Error
                 ? `synthesis-failed: ${synthesisError.message}`
                 : "synthesis-failed"
-            : final.failureReason,
+            : final.suspension === null && synthesisSuspension !== null
+                ? synthesisSuspension.reason
+                : final.failureReason,
         forceFailed: synthesisError !== null,
         findings: synthesisFindings,
         synthesisOutcome,
@@ -567,19 +613,14 @@ export async function runExecuteAnalysisBody(input, deps) {
     });
     // Terminal snapshot, fired here rather than inside `collectAndComplete` because this is the
     // first point where BOTH the derived final status and the scheduler's final step map are in
-    // hand. It precedes the budget self-cancel below on purpose: that branch raises out of the
-    // body, so an observer placed after it would never see a budget-paused run's last state.
+    // hand. It precedes the suspension self-cancel below on purpose: that branch raises out of the
+    // body, so an observer placed after it would never see a suspended run's last state.
     observeRunGuarded(deps, buildRunObservation({ input, runId, status: result.status, stepStates: final.stepStates }));
-    // Synthesis failure takes priority over the budget-exceeded cascade: a run
-    // whose synthesis threw is definitively failed (ERROR), not a resumable
-    // budget pause. Only self-cancel when synthesis succeeded.
-    if (final.budgetExceeded && synthesisError === null) {
-        await DBOS.cancelWorkflow(DBOS.workflowID);
-        await DBOS.runStep(async () => undefined, {
-            name: "self-cancel-budget-exceeded",
-        });
-        // Unreachable — the runStep above raises DBOSWorkflowCancelledError.
-    }
+    // Synthesis failure takes priority over the suspension cascade: a run whose
+    // synthesis threw is definitively failed (ERROR), not a resumable
+    // suspension. Only self-cancel when synthesis did not throw a failure.
+    if (suspension !== null && synthesisError === null)
+        return cancelSelf("self-cancel-suspended");
     // Re-throw after the terminal block so the workflow record goes to ERROR.
     if (synthesisError !== null)
         throw synthesisError;
@@ -590,19 +631,9 @@ async function validateAndInit(input, runId, deps) {
     // check that no other active run for the same plan won the race — the
     // partial-unique index would have rejected the insert if so, so this is
     // defense-in-depth only.
-    try {
-        const existing = unwrapOrThrow(await queryActiveRun(deps.pool, input.analysisId, input.planId));
-        if (existing && existing.runId !== runId) {
-            return { kind: "joined-existing", runId: existing.runId };
-        }
-    }
-    catch (err) {
-        if (err instanceof RunDedupCollisionError) {
-            const existing = unwrapOrThrow(await queryActiveRun(deps.pool, input.analysisId, input.planId));
-            if (existing)
-                return { kind: "joined-existing", runId: existing.runId };
-        }
-        throw err;
+    const existing = unwrapOrThrow(await queryActiveRun(deps.pool, input.analysisId, input.planId));
+    if (existing && existing.runId !== runId) {
+        return { kind: "joined-existing", runId: existing.runId };
     }
     await DBOS.runStep(async () => {
         // A resolver throw here (unknown analysis, unresolvable root) fails the
@@ -612,11 +643,13 @@ async function validateAndInit(input, runId, deps) {
             recursive: true,
         });
     }, { name: "init-run-filesystem" });
-    await DBOS.runStep(() => deps.runCharge.open({
+    const opened = await DBOS.runStep(async () => await passGate("RunCharge.open", deps.runCharge.open({
         analysisId: input.analysisId,
         runId,
         session: input.runSession,
-    }), { name: "open-running-charge" });
+    })), { name: "open-running-charge" });
+    if (opened.isErr())
+        return { kind: "charge-refused", refusal: opened.error };
     return { kind: "fresh" };
 }
 async function runSchedulerLoop(args) {
@@ -625,14 +658,14 @@ async function runSchedulerLoop(args) {
     const failed = new Set();
     const canceled = new Set();
     const inFlight = new Map();
-    const budgetExceededChildIds = new Set();
+    const childSuspensions = new Map();
     // Run-level usage aggregate, folded from the children's own durable results.
     // `getResult` IS the replay-safe source: a recovered parent reads back the
     // identical value from each child's workflow record, so the sum reproduces
     // exactly without a second channel. A child that settled by throwing carries
     // no result and therefore contributes nothing — same rule as its duration.
     const runUsage = {};
-    // Child workflow ids the parent itself cancelled (budget / neverFits halt cascade). Their `getResult`
+    // Child workflow ids the parent itself cancelled (suspension / neverFits halt cascade). Their `getResult`
     // rejects with a DBOS cancellation error and lands in the settlement error branch — but the step was
     // CANCELED by us, not failed on its own merits. Keyed here (deterministic: `cancelInFlight` runs the
     // same way on a replay) so that branch records it as canceled everywhere, matching how a child that
@@ -672,12 +705,20 @@ async function runSchedulerLoop(args) {
             }
         }
     };
-    // Halts scheduling entirely — reserved for the budget paths (a
-    // `budget_exceeded` settlement, the `neverFits` guard). An ordinary step
-    // failure does NOT set it: it dooms only that step's transitive dependents.
+    // Halts scheduling entirely — reserved for a suspension of a child and the
+    // `neverFits` guard. An ordinary step failure does NOT set it: it dooms
+    // only that step's transitive dependents.
     let halted = false;
-    let budgetExceeded = false;
+    let suspension = null;
     let failureReason = null;
+    const haltForSuspension = async (child) => {
+        if (suspension === null) {
+            suspension = { kind: "suspended", reason: child.reason };
+            halted = true;
+            failureReason = child.reason;
+        }
+        await cancelInFlight(inFlight, child.reason, canceledByParent);
+    };
     const stepIndexById = new Map(input.steps.map((s, i) => [s.id, i]));
     const emitDagSnapshot = async () => {
         await emitStreamPart({
@@ -824,8 +865,8 @@ async function runSchedulerLoop(args) {
         catch (err) {
             settled = { childId, stepId: entry.stepId, kind: "error", err };
         }
-        await drainBudgetExceededNotifications(budgetExceededChildIds);
-        const childWasBudgetExceeded = budgetExceededChildIds.has(settled.childId);
+        await drainChildSuspensions(childSuspensions);
+        const childSuspension = childSuspensions.get(settled.childId);
         // Terminal step status + duration for the provenance emission below. Every
         // settlement branch assigns both, so the emission fires exactly once per
         // settled child with the mapped status.
@@ -847,13 +888,8 @@ async function runSchedulerLoop(args) {
             }
             else if (r.status === "canceled") {
                 canceled.add(settled.stepId);
-                const isBudgetCancel = childWasBudgetExceeded || r.error === "budget_exceeded";
-                if (isBudgetCancel && !budgetExceeded) {
-                    budgetExceeded = true;
-                    halted = true;
-                    failureReason = "budget_exceeded";
-                    await cancelInFlight(inFlight, "budget_exceeded", canceledByParent);
-                }
+                if (childSuspension !== undefined)
+                    await haltForSuspension(childSuspension);
                 stepRuntime.set(settled.stepId, {
                     status: "failed",
                     durationMs: r.durationMs ?? undefined,
@@ -884,7 +920,7 @@ async function runSchedulerLoop(args) {
             }
         }
         else if (canceledByParent.has(settled.childId)) {
-            // The child threw because the PARENT cancelled it (budget / neverFits halt cascade), not
+            // The child threw because the PARENT cancelled it (suspension / neverFits halt cascade), not
             // because it failed on its own — its `getResult` rejects with a DBOS cancellation error.
             // Record it as canceled, mirroring the graceful `{status:"canceled"}` branch above so the
             // `canceled` set, the terminal result, and the `step_completed` provenance all agree. No
@@ -900,25 +936,25 @@ async function runSchedulerLoop(args) {
             stepDurationMs = undefined;
             await emitDagSnapshot();
         }
+        else if (childSuspension !== undefined) {
+            // The child suspended: it sent its typed suspension, then cancelled
+            // itself, thus `getResult` rejects with a DBOS cancellation. The
+            // suspension halts the run — the cascade is a decision of the host,
+            // not a DAG one — and it dooms no dependent: the resumed run needs them.
+            canceled.add(settled.stepId);
+            await haltForSuspension(childSuspension);
+            stepRuntime.set(settled.stepId, { status: "failed", error: childSuspension.reason });
+            stepStatus = "canceled";
+            stepDurationMs = undefined;
+            await emitDagSnapshot();
+        }
         else {
-            // The child threw on its own (not a parent-driven cancel). A
-            // budget_exceeded throw still halts the run — the pause cascade is a
-            // money decision, not a DAG one. Any other throw is treated exactly
-            // like a step failure: dependents doomed, siblings continue.
+            // The child threw on its own (not a parent-driven cancel, not a
+            // suspension). A throw is treated exactly like a step failure:
+            // dependents doomed, siblings continue.
             failed.add(settled.stepId);
-            const isBudgetThrow = childWasBudgetExceeded || isBudgetExceeded(settled.err);
-            if (isBudgetThrow) {
-                if (!budgetExceeded) {
-                    budgetExceeded = true;
-                    halted = true;
-                    failureReason = "budget_exceeded";
-                }
-                await cancelInFlight(inFlight, "budget_exceeded", canceledByParent);
-            }
-            else {
-                failureReason ??= settled.err instanceof Error ? settled.err.message : String(settled.err);
-                markDependentsSkipped(settled.stepId);
-            }
+            failureReason ??= settled.err instanceof Error ? settled.err.message : String(settled.err);
+            markDependentsSkipped(settled.stepId);
             stepRuntime.set(settled.stepId, {
                 status: "failed",
                 error: settled.err instanceof Error ? settled.err.message : String(settled.err),
@@ -927,8 +963,7 @@ async function runSchedulerLoop(args) {
             // A child that settled by throwing carries no durable duration.
             stepDurationMs = undefined;
             await emitDagSnapshot();
-            if (!isBudgetThrow)
-                await dispatchReady();
+            await dispatchReady();
         }
         // One `step_completed` provenance emission per settled child — this
         // settlement loop is the only site that observes every executed step.
@@ -947,23 +982,24 @@ async function runSchedulerLoop(args) {
             atMs: settledAtMs,
         }, input.runSession);
     }
-    await drainBudgetExceededNotifications(budgetExceededChildIds);
+    await drainChildSuspensions(childSuspensions);
     return {
         completed,
         failed,
         canceled,
-        budgetExceeded,
+        suspension,
         failureReason,
         stepStates: stepRuntime,
         ...(hasReportedUsage(runUsage) ? { usage: { ...runUsage } } : {}),
     };
 }
-async function drainBudgetExceededNotifications(childIds) {
+async function drainChildSuspensions(suspensions) {
     while (true) {
-        const msg = await DBOS.recv(BUDGET_EXCEEDED_TOPIC, 0);
+        const msg = await DBOS.recv(CHILD_SUSPENDED_TOPIC, 0);
         if (!msg)
             return;
-        childIds.add(msg.childWorkflowId);
+        if (msg.kind === "suspended" && !suspensions.has(msg.childWorkflowId))
+            suspensions.set(msg.childWorkflowId, msg);
     }
 }
 async function cancelInFlight(inFlight, cause, canceledByParent) {
@@ -978,7 +1014,9 @@ async function cancelInFlight(inFlight, cause, canceledByParent) {
     }
 }
 async function collectAndComplete(args) {
-    const { input, runId, workflowId, startedAtMs, completed, failed, canceled, budgetExceeded, failureReason, forceFailed, deps } = args;
+    const { input, runId, workflowId, startedAtMs, completed, failed, canceled, suspension, failureReason, forceFailed, deps } = args;
+    // A forced failure is terminal and never a resumable suspension.
+    const suspended = suspension !== null && !forceFailed ? suspension : null;
     const logger = (deps.logger ?? createNoopLogger()).named("executeAnalysis").with({ runId, analysisId: input.analysisId });
     const status = forceFailed
         ? "failed"
@@ -987,14 +1025,20 @@ async function collectAndComplete(args) {
             completed,
             failed,
             canceled,
-            budgetExceeded,
+            suspended: suspended !== null,
         });
-    const chargeReason = status === "completed" || status === "partial" ? "ok" : status === "failed" ? "error" : budgetExceeded ? "budget_exceeded" : "canceled";
+    const chargeOutcome = status === "completed" || status === "partial"
+        ? { kind: "ok" }
+        : status === "failed"
+            ? { kind: "error" }
+            : suspended !== null
+                ? { kind: "suspended", reason: suspended.reason }
+                : { kind: "canceled" };
     const revokeReason = status === "completed" || status === "partial"
         ? "workflow-completed"
         : status === "failed"
             ? "workflow-failed"
-            : budgetExceeded
+            : suspended !== null
                 ? "workflow-suspended"
                 : "workflow-canceled";
     // Terminal clock read + run-completion provenance, emitted BEFORE the `cortex_runs`
@@ -1016,36 +1060,30 @@ async function collectAndComplete(args) {
         atMs: terminalAtMs,
         durationMs: terminalAtMs - startedAtMs,
     }, input.runSession);
-    try {
-        await DBOS.runStep(async () => {
-            unwrapOrThrow(await updateRunStatus(deps.pool, runId, status, failureReason ?? (status === "canceled" && !budgetExceeded ? "external_cancel" : null)));
-            // The outcome metrics ride inside the step that persists the terminal status: DBOS
-            // caches the step, so a recovered body records nothing again. A step's own terminal
-            // row already carried its metric; the residue counted here is every settled step
-            // without one — a child the cancel cut before `mark-canceled` (the cancel check runs
-            // before a step body), or one that died before `failStep`. The ledger read is kept
-            // apart from the status write: a failed read costs the metrics, never the write.
-            let stepRows = [];
-            try {
-                stepRows = unwrapOrThrow(await queryStepsByRun(deps.pool, runId));
-            }
-            catch (err) {
-                logger.warn("run metrics: step ledger read failed, residual step outcomes skipped", logger.errorFields(err));
-            }
-            for (const row of stepRows) {
-                if (row.completedAt !== null)
-                    continue;
-                if (canceled.has(row.stepId))
-                    recordStepCompleted({ agentId: row.agentId, status: "canceled" });
-                else if (failed.has(row.stepId))
-                    recordStepCompleted({ agentId: row.agentId, status: "failed" });
-            }
-            recordRunCompleted({ workflow: "analysis", status, durationMs: terminalAtMs - startedAtMs });
-        }, { name: "persist-final-status" });
-    }
-    catch (err) {
-        logger.error("persist-final-status failed", { status, ...logger.errorFields(err) });
-    }
+    await ledgerStep(logger.with({ status }), "persist-final-status", async () => {
+        const written = await updateRunStatus(deps.pool, runId, status, failureReason ?? (status === "canceled" && suspended === null ? "external_cancel" : null));
+        if (written.isErr())
+            return written;
+        // The outcome metrics ride inside the step that persists the terminal status: DBOS
+        // caches the step, so a recovered body records nothing again. A step's own terminal
+        // row already carried its metric; the residue counted here is every settled step
+        // without one — a child the cancel cut before `mark-canceled` (the cancel check runs
+        // before a step body), or one that died before `failStep`. The ledger read is kept
+        // apart from the status write: a failed read costs the metrics, never the write.
+        const stepRows = await queryStepsByRun(deps.pool, runId);
+        if (stepRows.isErr())
+            logger.warn("run metrics: step ledger read failed, residual step outcomes skipped", { error: describeDbError(stepRows.error) });
+        for (const row of stepRows.unwrapOr([])) {
+            if (row.completedAt !== null)
+                continue;
+            if (canceled.has(row.stepId))
+                recordStepCompleted({ agentId: row.agentId, status: "canceled" });
+            else if (failed.has(row.stepId))
+                recordStepCompleted({ agentId: row.agentId, status: "failed" });
+        }
+        recordRunCompleted({ workflow: "analysis", status, durationMs: terminalAtMs - startedAtMs });
+        return written;
+    });
     // Record the run's synthesis outcome AFTER the status write — a reader that
     // sees a terminal status then also sees whether (and why) synthesis ran. A
     // null outcome means synthesis never ran; leave the columns NULL ("unknown").
@@ -1053,53 +1091,25 @@ async function collectAndComplete(args) {
     // must not undo an otherwise-complete run.
     const outcome = args.synthesisOutcome;
     if (outcome !== null) {
-        try {
-            await DBOS.runStep(async () => {
-                unwrapOrThrow(await setRunSynthesisOutcome(deps.pool, runId, outcome.status, outcome.reason));
-            }, { name: "persist-synthesis-outcome" });
-        }
-        catch (err) {
-            logger.error("persist-synthesis-outcome failed", { synthesisStatus: outcome.status, ...logger.errorFields(err) });
-        }
+        await ledgerStep(logger.with({ synthesisStatus: outcome.status }), "persist-synthesis-outcome", () => setRunSynthesisOutcome(deps.pool, runId, outcome.status, outcome.reason));
     }
     // Sweep never-started steps to `skipped` — but ONLY on genuinely-terminal
     // paths. The gate is the BRANCH, not the written run status: the resumable
-    // 402 pause below also writes "canceled", yet its pending rows must survive
+    // suspension below also writes "canceled", yet its pending rows must survive
     // for the resumed workflow to execute. Log-don't-rollback, like every other
     // finalisation step here.
-    if (!(budgetExceeded && !forceFailed)) {
-        try {
-            await DBOS.runStep(async () => {
-                unwrapOrThrow(await sweepPendingStepExecutions(deps.pool, runId));
-            }, { name: "sweep-pending-steps" });
-        }
-        catch (err) {
-            logger.error("sweep-pending-steps failed", logger.errorFields(err));
-        }
+    if (suspended === null) {
+        await ledgerStep(logger, "sweep-pending-steps", () => sweepPendingStepExecutions(deps.pool, runId));
     }
-    // A forced failure (synthesis threw) is terminal, not a resumable budget
-    // pause — never suspend it, even when the budget was also exceeded.
-    if (budgetExceeded && !forceFailed) {
-        try {
-            await DBOS.runStep(async () => {
-                unwrapOrThrow(await suspendAnalysisQuery(deps.pool, input.analysisId));
-            }, { name: "suspend-analysis" });
-        }
-        catch (err) {
-            logger.error("suspend-analysis failed", logger.errorFields(err));
-        }
-    }
-    try {
-        await DBOS.runStep(() => deps.runCharge.close({
-            analysisId: input.analysisId,
-            runId,
-            reason: chargeReason,
-            session: input.runSession,
-        }), { name: "close-running-charge" });
-    }
-    catch (err) {
-        logger.error("closeRunningCharge failed", { chargeReason, ...logger.errorFields(err) });
+    if (suspended !== null) {
+        await ledgerStep(logger, "suspend-analysis", () => suspendAnalysisQuery(deps.pool, input.analysisId));
     }
+    await DBOS.runStep(() => deliverNotice(logger.with({ chargeOutcome: chargeOutcome.kind }), "RunCharge.close", deps.runCharge.close({
+        analysisId: input.analysisId,
+        runId,
+        outcome: chargeOutcome,
+        session: input.runSession,
+    })), { name: "close-running-charge" });
     // Revoke the run authorization for the terminal run state. Ownership
     // defaults to true for inputs persisted before the field existed (a
     // workflow recovered across this deploy) — those were always Cortex-owned.
@@ -1107,17 +1117,14 @@ async function collectAndComplete(args) {
         runSession: input.runSession,
         ownsMandate: input.ownsMandate ?? true, // oss-core-managed-ok
     };
-    try {
-        await DBOS.runStep(() => deps.runAuthorizer.revoke(authorization, revokeReason), { name: "revoke-run-auth" });
-    }
-    catch (err) {
-        logger.error("revokeRunAuthorization failed", { revokeReason, ...logger.errorFields(err) });
-    }
+    await DBOS.runStep(() => deliverNotice(logger.with({ revokeReason }), "RunAuthorizer.revoke", deps.runAuthorizer.revoke(authorization, revokeReason)), {
+        name: "revoke-run-auth",
+    });
     // The `run_completed` provenance was already emitted above (before the status write, to
     // beat the CLI's poll-and-shutdown). These branches only fan out the UI stream part, whose
     // completed/failed shapes genuinely differ — the provenance record does not.
     if (status === "completed" || status === "partial") {
-        const artifactCount = await DBOS.runStep(() => countArtifactsForRun(deps.pool, input.analysisId, runId), { name: "count-run-artifacts" }).catch(() => 0);
+        const artifactCount = await DBOS.runStep(async () => await ResultAsync.fromPromise(countArtifactsForRun(deps.pool, input.analysisId, runId), () => "the artifact count failed").unwrapOr(0), { name: "count-run-artifacts" });
         await emitStreamPart({
             type: "data-run-completed",
             runId,
@@ -1141,8 +1148,8 @@ async function collectAndComplete(args) {
         await emitStreamPart({
             type: "data-run-failed",
             runId,
-            error: failureReason ?? (budgetExceeded ? "Run paused: insufficient budget" : "Run canceled before completion"),
-            ...(budgetExceeded ? { reason: "budget_exceeded" } : {}),
+            error: failureReason ?? (suspended !== null ? suspended.reason : "Run canceled before completion"),
+            ...(suspended !== null ? { reason: suspended.reason } : {}),
         });
     }
     return {
@@ -1161,8 +1168,8 @@ async function collectAndComplete(args) {
  * status vocabulary without a cast.
  */
 function deriveFinalStatus(args) {
-    const { totalSteps, completed, failed, canceled, budgetExceeded } = args;
-    if (budgetExceeded)
+    const { totalSteps, completed, failed, canceled, suspended } = args;
+    if (suspended)
         return "canceled";
     if (failed.size > 0) {
         return completed.size > 0 ? "partial" : "failed";