@inflexa-ai/harness 0.1.1

package/dist/workflows/execute-analysis.js

@@ -0,0 +1,746 @@
+/**
+ * `executeAnalysis` — the parent DBOS workflow.
+ *
+ * Shape (every external interaction is a `DBOS.runStep` so a crashed parent
+ * replays from the cache instead of repeating side effects):
+ *
+ *  1. `validateAndInit`
+ *     - validate the plan DAG (cycle / missing-dep / dup-id)
+ *     - create the run dir
+ *     - open the running charge
+ *     - emit `data-run-started`
+ *     Inputs are NOT materialized here: the workspace tree's `data/` inputs must
+ *     already be populated by the embedder before the run is triggered — the
+ *     workflow neither downloads nor stages input data.
+ *     The run authorization is minted at the async edge (`executePlan` / TA
+ *     trigger / `runDataProfile`) and rides in `input.runSession` — the
+ *     workflow body never mints, never reads the JWT back from `cortex_runs`.
+ *  2. Scheduler loop
+ *     - maintain `completedSet` + `inFlightHandles`
+ *     - dispatch every `scheduleReady(plan, completedSet, startedSet)` via
+ *       `DBOS.startWorkflow` with deterministic child id
+ *       `"${parent.workflowID}-${N}"` (N is the step's stable index)
+ *     - each child input is `forStep(input.runSession, stepId)` for the
+ *       session field
+ *     - emit `data-dag-state` (reconciling by id) on every dispatch +
+ *       completion
+ *     - await child results via `getResult` (these are themselves cached
+ *       in DBOS, so parent recovery does NOT re-run completed children)
+ *  3. Fail-fast cascade
+ *     - on the first non-resumable child failure (or on external parent
+ *       cancel): stop scheduling, then
+ *       `Promise.allSettled(handles.map(h => DBOS.cancelWorkflow(h.workflowID)))`
+ *     - each cancelled child tears down its sandbox in its own terminal path
+ *  4. Pause cascade (402)
+ *     - on a child cancelling itself with `budget_exceeded`: cancel siblings
+ *       then self-cancel the parent to `CANCELLED` (NOT `ERROR` — ERROR
+ *       isn't resumable; NOTES #3)
+ *     - the analysis flips to `suspended_insufficient_funds` in
+ *       `collectAndComplete`. The paused parent lands in `CANCELLED`, a
+ *       durably reschedulable state; re-driving it after a top-up is a
+ *       future enhancement, not yet wired here.
+ *  5. `synthesizeFindings` (single sequential block, no sandbox)
+ *  6. `collectAndComplete` (terminal — runs on ALL paths)
+ *     - determine final status from completed/cancelled/failed counts
+ *     - update `cortex_runs.status` + `error`
+ *     - close running charge with matching reason
+ *     - revoke run authorization via the injected `runAuthorizer` seam
+ *     - emit terminal stream part
+ *     - WRITES NOTHING to the conversation thread (results are pull-only
+ *       via `inspectRun`)
+ */
+import { DBOS } from "@dbos-inc/dbos-sdk";
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { forStep } from "../auth/types.js";
+import { unwrapOrThrow } from "../lib/result.js";
+import { RunDedupCollisionError, countArtifactsForRun, queryActiveRun, seedStepExecutions, suspendAnalysis as suspendAnalysisQuery, sweepPendingStepExecutions, updateRunStatus, } from "../state/index.js";
+import { isBudgetExceeded } from "../loop/budget-exceeded.js";
+import { runDir } from "../workspace/paths.js";
+import { isChatDataPart } from "../sandbox/sandbox-step-translate.js";
+import { synthesizeRun } from "../app/synthesize-run.js";
+import { computeTopologicalLevels, scheduleReady, validatePlanDag } from "./execute-analysis-scheduler.js";
+import { recordCancelledChild } from "./metrics.js";
+import { BUDGET_EXCEEDED_TOPIC } from "./sandbox-step.js";
+// ── In-body orchestration helpers ────────────────────────────────────
+/** Emit a chat data part on the parent's DBOS stream. */
+function emitStreamPart(part) {
+    return DBOS.writeStream("events", part);
+}
+/**
+ * Fire the optional run-lifecycle provenance observer, isolating a throwing
+ * host observer from the workflow. The harness is host-agnostic and cannot
+ * assume the callback is total; a defect in it must log loudly and be
+ * swallowed, never corrupt run state — integrity enforcement lives at the
+ * host's own ledger, not here.
+ */
+function emitProvenanceGuarded(deps, event) {
+    if (!deps.emitProvenance)
+        return;
+    try {
+        deps.emitProvenance(event);
+    }
+    catch (err) {
+        console.error(`[executeAnalysis] emitProvenance threw for run ${event.runId} (event=${event.type}):`, err);
+    }
+}
+/**
+ * Project the sandbox-step input handed to each child. The parent owns the
+ * level computation; the rest comes from `ExecuteAnalysisInput`. The body
+ * fills in `runSession` via `forStep` after this returns. Exported for the
+ * projection/timeout-propagation tests.
+ */
+export function buildChildInput(args) {
+    const { input, stepId, level, runId, workflowId } = args;
+    const resources = input.resourcesByStepId[stepId];
+    if (!resources) {
+        throw new Error(`executeAnalysis: step "${stepId}" missing from resourcesByStepId — every step must declare resources`);
+    }
+    return {
+        analysisId: input.analysisId,
+        runId,
+        stepId,
+        agentId: input.agentByStepId[stepId] ?? "scientific-executor",
+        level,
+        prompt: input.promptByStepId[stepId] ?? "",
+        parentWorkflowId: workflowId,
+        resources,
+        timeoutSeconds: input.timeoutByStepId?.[stepId],
+    };
+}
+/**
+ * Run-level literature-grounded synthesis (no sandbox). Returns the
+ * quick-display findings for the run-completed card; empty when synthesis is
+ * skipped or produced no synthesizable content. A genuine failure re-throws
+ * inside the caller's `DBOS.runStep`, failing the run (D10). Exported so
+ * integration tests can drive the wired synthesis without a full parent body.
+ */
+export function synthesizeFindings(args) {
+    const { analysisId, runId, completedSteps, session, deps } = args;
+    const emit = (event) => {
+        const part = isChatDataPart(event)
+            ? event
+            : {
+                type: "data-loop-event",
+                data: { stepId: "synthesis", event },
+            };
+        return DBOS.writeStream("events", part);
+    };
+    return synthesizeRun({
+        pool: deps.pool,
+        provider: deps.provider,
+        embedding: deps.embedding,
+        resolveWorkspaceRoot: deps.resolveWorkspaceRoot,
+        synthesisModel: deps.synthesisModel,
+        bioKeys: deps.bioKeys,
+    }, {
+        analysisId,
+        runId,
+        completedSteps,
+        session,
+        emit,
+        onProgress: (phase, activity, extra = {}) => DBOS.writeStream("events", {
+            type: "data-synthesis-progress",
+            id: `synthesis-progress-${runId}`,
+            runId,
+            phase,
+            activity,
+            ...extra,
+        }),
+    });
+}
+// ── Workflow registration ────────────────────────────────────────────
+/**
+ * Register the executeAnalysis parent workflow with DBOS. Caller is
+ * responsible for setting `workflowID = runId` via
+ * `DBOS.startWorkflow(_, { workflowID })` — the body assumes that contract
+ * and uses `DBOS.workflowID` as the source of truth.
+ */
+export function registerExecuteAnalysis(deps) {
+    return DBOS.registerWorkflow(async (input) => {
+        return runExecuteAnalysisBody(input, deps);
+    }, { name: "executeAnalysis" });
+}
+/**
+ * Body extracted so tests can drive it without registering a workflow
+ * (the DBOS calls inside still rely on a workflow context being present).
+ */
+export async function runExecuteAnalysisBody(input, deps) {
+    // (0) Validate up-front — no side effects yet, so a malformed plan does
+    // NOT leak a runId, run authorization, or running charge.
+    validatePlanDag(input.steps);
+    const levels = computeTopologicalLevels(input.steps);
+    // (1) validateAndInit — create the run dir + open the charge. Inputs are
+    // NOT materialized here: the embedder must have populated the workspace tree's
+    // `data/` before triggering (the workflow neither downloads nor stages). The
+    // `cortex_runs` row was inserted by `executePlan` at the async edge;
+    // the run authorization is already minted and rides in `input.runSession`.
+    const runId = input.runSession.runFrame.runId;
+    const workflowId = runId;
+    const init = await validateAndInit(input, runId, deps);
+    if (init.kind === "joined-existing") {
+        return {
+            runId: init.runId,
+            workflowId: init.runId,
+            status: "running",
+            completedSteps: [],
+            failedSteps: [],
+            canceledSteps: [],
+        };
+    }
+    // Seed the full DAG as `pending` rows so ledger readers see every step from
+    // run start (honest done/total), not just the ones that began executing.
+    // The seed's DO NOTHING conflict action makes this replay-safe; the agent
+    // fallback mirrors `buildChildInput` so seed and mark-running agree on the
+    // row a step's child later upserts.
+    await DBOS.runStep(async () => {
+        unwrapOrThrow(await seedStepExecutions(deps.pool, input.steps.map((step) => ({
+            runId,
+            stepId: step.id,
+            analysisId: input.analysisId,
+            wave: levels.get(step.id) ?? 0,
+            agentId: input.agentByStepId[step.id] ?? "scientific-executor",
+        }))));
+    }, { name: "seed-step-executions" });
+    // One checkpointed clock read stamps the run's start; the terminal read in
+    // `collectAndComplete` subtracts it for the true run span. Checkpointed →
+    // replay-stable (see `RunProvenanceEvent`).
+    const startedAtMs = await DBOS.now();
+    await emitStreamPart({
+        type: "data-run-started",
+        runId,
+        planSummary: input.planSummary,
+        stepCount: input.steps.length,
+    });
+    emitProvenanceGuarded(deps, {
+        type: "run_started",
+        analysisId: input.analysisId,
+        runId,
+        planSummary: input.planSummary,
+        stepCount: input.steps.length,
+        atMs: startedAtMs,
+    });
+    // (2-4) Scheduler loop + fail-fast + pause cascade.
+    const final = await runSchedulerLoop({
+        input,
+        runId,
+        workflowId,
+        levels,
+        deps,
+    });
+    // (5) Run-level synthesis — a genuine failure must fail the run (D10). The
+    // honest non-fatal outcomes (report_blocker / no-summaries) resolve normally
+    // inside the dep; only real exceptions reach here. We capture the error,
+    // still run the terminal block (close charge + revoke run authorization), and re-throw
+    // after so the workflow record goes to ERROR.
+    let synthesisError = null;
+    let synthesisFindings = [];
+    const synthesisEnabled = deps.synthesisEnabled ?? true;
+    if (synthesisEnabled && final.completed.size > 0) {
+        try {
+            const synthOut = await DBOS.runStep(() => synthesizeFindings({
+                analysisId: input.analysisId,
+                runId,
+                completedSteps: [...final.completed],
+                session: input.runSession,
+                deps,
+            }), { name: "synthesize-findings" });
+            synthesisFindings = synthOut.findings;
+        }
+        catch (err) {
+            synthesisError = err;
+            console.error(`[executeAnalysis] synthesizeFindings failed for run ${runId} — failing the run:`, err instanceof Error ? err.message : err);
+        }
+    }
+    // (6) collectAndComplete — terminal block. Runs on EVERY path. When synthesis
+    // failed, force a failed terminal status so the charge/run-authorization close correctly
+    // and the run row + stream report the failure before we re-throw.
+    const result = await collectAndComplete({
+        input,
+        runId,
+        workflowId,
+        startedAtMs,
+        completed: final.completed,
+        failed: final.failed,
+        canceled: final.canceled,
+        budgetExceeded: final.budgetExceeded,
+        failureReason: synthesisError
+            ? synthesisError instanceof Error
+                ? `synthesis-failed: ${synthesisError.message}`
+                : "synthesis-failed"
+            : final.failureReason,
+        forceFailed: synthesisError !== null,
+        findings: synthesisFindings,
+        deps,
+    });
+    // Synthesis failure takes priority over the budget-exceeded cascade: a run
+    // whose synthesis threw is definitively failed (ERROR), not a resumable
+    // budget pause. Only self-cancel when synthesis succeeded.
+    if (final.budgetExceeded && synthesisError === null) {
+        await DBOS.cancelWorkflow(DBOS.workflowID);
+        await DBOS.runStep(async () => undefined, {
+            name: "self-cancel-budget-exceeded",
+        });
+        // Unreachable — the runStep above raises DBOSWorkflowCancelledError.
+    }
+    // Re-throw after the terminal block so the workflow record goes to ERROR.
+    if (synthesisError !== null)
+        throw synthesisError;
+    return result;
+}
+async function validateAndInit(input, runId, deps) {
+    // The cortex_runs row already exists (inserted by `executePlan`). Sanity-
+    // check that no other active run for the same plan won the race — the
+    // partial-unique index would have rejected the insert if so, so this is
+    // defense-in-depth only.
+    try {
+        const existing = unwrapOrThrow(await queryActiveRun(deps.pool, input.analysisId, input.planId));
+        if (existing && existing.runId !== runId) {
+            return { kind: "joined-existing", runId: existing.runId };
+        }
+    }
+    catch (err) {
+        if (err instanceof RunDedupCollisionError) {
+            const existing = unwrapOrThrow(await queryActiveRun(deps.pool, input.analysisId, input.planId));
+            if (existing)
+                return { kind: "joined-existing", runId: existing.runId };
+        }
+        throw err;
+    }
+    await DBOS.runStep(async () => {
+        // A resolver throw here (unknown analysis, unresolvable root) fails the
+        // step durably — exactly the contract the workspace-root-resolution spec
+        // requires of failures inside DBOS bodies.
+        await mkdir(join(deps.resolveWorkspaceRoot(input.analysisId), runDir(runId)), {
+            recursive: true,
+        });
+    }, { name: "init-run-filesystem" });
+    await DBOS.runStep(() => deps.runCharge.open({
+        analysisId: input.analysisId,
+        runId,
+        session: input.runSession,
+    }), { name: "open-running-charge" });
+    return { kind: "fresh" };
+}
+async function runSchedulerLoop(args) {
+    const { input, runId, workflowId, levels, deps } = args;
+    const completed = new Set();
+    const failed = new Set();
+    const canceled = new Set();
+    const inFlight = new Map();
+    const budgetExceededChildIds = new Set();
+    // Child workflow ids the parent itself cancelled (fail-fast / budget cascade). Their `getResult`
+    // rejects with a DBOS cancellation error and lands in the settlement error branch — but the step was
+    // CANCELED by us, not failed on its own merits. Keyed here (deterministic: `cancelInFlight` runs the
+    // same way on a replay) so that branch records it as canceled everywhere, matching how a child that
+    // returns a graceful `{status:"canceled"}` is already handled.
+    const canceledByParent = new Set();
+    const stepRuntime = new Map();
+    for (const step of input.steps) {
+        stepRuntime.set(step.id, { status: "pending" });
+    }
+    let failFast = false;
+    let budgetExceeded = false;
+    let failureReason = null;
+    const stepIndexById = new Map(input.steps.map((s, i) => [s.id, i]));
+    const emitDagSnapshot = async () => {
+        await emitStreamPart({
+            type: "data-dag-state",
+            id: `dag-${runId}`,
+            runId,
+            steps: input.steps.map((s) => {
+                const rt = stepRuntime.get(s.id);
+                const entry = {
+                    id: s.id,
+                    name: s.id,
+                    agent: input.agentByStepId[s.id] ?? "unknown",
+                    status: rt.status,
+                    level: levels.get(s.id) ?? 0,
+                    dependsOn: [...s.depends_on],
+                };
+                if (rt.durationMs !== undefined)
+                    entry.durationMs = rt.durationMs;
+                if (rt.error !== undefined)
+                    entry.error = rt.error;
+                return entry;
+            }),
+        });
+    };
+    const dbosParentId = DBOS.workflowID ?? workflowId;
+    const dispatchReady = async () => {
+        if (failFast)
+            return;
+        const inFlightStepIds = new Set([...inFlight.values()].map((h) => h.stepId));
+        let admit;
+        let queuedChanged = false;
+        if (input.budget) {
+            const scheduled = scheduleReady(input.steps, completed, inFlightStepIds, {
+                budget: input.budget,
+                resourcesByStepId: input.resourcesByStepId,
+            });
+            if (scheduled.neverFits.length > 0) {
+                // An over-budget step can never be admitted — plan-time validation
+                // makes this unreachable for new plans; stored plans fail loudly
+                // instead of waiting forever. Standard fail-fast follows.
+                for (const stepId of scheduled.neverFits) {
+                    const r = input.resourcesByStepId[stepId];
+                    const declared = r ? `${r.cpu} CPU / ${r.memoryGb} GB` : "undeclared";
+                    failed.add(stepId);
+                    stepRuntime.set(stepId, {
+                        status: "failed",
+                        error: `step resources (${declared}) exceed the machine budget (${input.budget.cpu} CPU / ${input.budget.memoryGb} GB)`,
+                    });
+                }
+                failFast = true;
+                failureReason = `step "${scheduled.neverFits[0]}" exceeds the machine resource budget`;
+                await cancelInFlight(inFlight, "fail_fast", canceledByParent);
+                await emitDagSnapshot();
+                return;
+            }
+            for (const stepId of scheduled.heldForCapacity) {
+                if (stepRuntime.get(stepId).status !== "queued") {
+                    stepRuntime.set(stepId, { status: "queued" });
+                    queuedChanged = true;
+                }
+            }
+            admit = scheduled.admit;
+        }
+        else {
+            admit = scheduleReady(input.steps, completed, inFlightStepIds);
+        }
+        if (admit.length === 0) {
+            if (queuedChanged)
+                await emitDagSnapshot();
+            return;
+        }
+        for (const stepId of admit) {
+            const idx = stepIndexById.get(stepId);
+            const childWorkflowId = `${workflowId}-${idx}`;
+            const baseChildInput = buildChildInput({
+                input,
+                stepId,
+                level: levels.get(stepId) ?? 0,
+                runId,
+                workflowId,
+            });
+            const childInput = {
+                ...baseChildInput,
+                parentWorkflowId: dbosParentId,
+                runSession: forStep(input.runSession, stepId),
+            };
+            const handle = (await DBOS.startWorkflow(deps.sandboxStepCallable, {
+                workflowID: childWorkflowId,
+            })(childInput));
+            inFlight.set(childWorkflowId, { stepId, handle });
+            stepRuntime.set(stepId, { status: "running" });
+        }
+        await emitDagSnapshot();
+    };
+    await dispatchReady();
+    while (inFlight.size > 0) {
+        // `waitFirst` checkpoints the "who finished first" decision (see the harness-durable-runtime spec), so
+        // the winning workflow id replays identically — unlike `Promise.race` over
+        // `getResult`, whose uncheckpointed winner would reorder the downstream
+        // function-ID-consuming ops and diverge the counter on replay.
+        const winner = await DBOS.waitFirst([...inFlight.values()].map((e) => e.handle));
+        const childId = winner.workflowID;
+        const entry = inFlight.get(childId);
+        inFlight.delete(childId);
+        let settled;
+        try {
+            const result = await entry.handle.getResult();
+            settled = { childId, stepId: entry.stepId, kind: "result", result };
+        }
+        catch (err) {
+            settled = { childId, stepId: entry.stepId, kind: "error", err };
+        }
+        await drainBudgetExceededNotifications(budgetExceededChildIds);
+        const childWasBudgetExceeded = budgetExceededChildIds.has(settled.childId);
+        // Terminal step status + duration for the provenance emission below. Every
+        // settlement branch assigns both, so the emission fires exactly once per
+        // settled child with the mapped status.
+        let stepStatus;
+        let stepDurationMs;
+        if (settled.kind === "result") {
+            const r = settled.result;
+            if (r.status === "complete") {
+                completed.add(settled.stepId);
+                stepRuntime.set(settled.stepId, {
+                    status: "completed",
+                    durationMs: r.durationMs ?? undefined,
+                });
+                stepStatus = "completed";
+                stepDurationMs = r.durationMs ?? undefined;
+                await emitDagSnapshot();
+                await dispatchReady();
+            }
+            else if (r.status === "canceled") {
+                canceled.add(settled.stepId);
+                const isBudgetCancel = childWasBudgetExceeded || r.error === "budget_exceeded";
+                if (isBudgetCancel && !budgetExceeded) {
+                    budgetExceeded = true;
+                    failFast = true;
+                    failureReason = "budget_exceeded";
+                    await cancelInFlight(inFlight, "budget_exceeded", canceledByParent);
+                }
+                stepRuntime.set(settled.stepId, {
+                    status: "failed",
+                    durationMs: r.durationMs ?? undefined,
+                    error: r.error ?? "canceled",
+                });
+                stepStatus = "canceled";
+                stepDurationMs = r.durationMs ?? undefined;
+                await emitDagSnapshot();
+            }
+            else {
+                // failed or blocked — a blocker (see the harness-sandbox-agents spec) fails fast exactly like a
+                // failure: the step declared it cannot deliver, so its dependents can
+                // never run. Same cancel-siblings + stop-scheduling path.
+                failed.add(settled.stepId);
+                if (!failFast) {
+                    failFast = true;
+                    failureReason = r.error ?? (r.status === "blocked" ? "step_blocked" : "step_failed");
+                    await cancelInFlight(inFlight, "fail_fast", canceledByParent);
+                }
+                stepRuntime.set(settled.stepId, {
+                    status: "failed",
+                    durationMs: r.durationMs ?? undefined,
+                    error: r.error ?? (r.status === "blocked" ? "step_blocked" : "step_failed"),
+                });
+                stepStatus = "failed";
+                stepDurationMs = r.durationMs ?? undefined;
+                await emitDagSnapshot();
+            }
+        }
+        else if (canceledByParent.has(settled.childId)) {
+            // The child threw because the PARENT cancelled it (fail-fast / budget cascade), not because
+            // it failed on its own — its `getResult` rejects with a DBOS cancellation error. Record it as
+            // canceled, mirroring the graceful `{status:"canceled"}` branch above so the `canceled` set,
+            // the terminal result, and the `step_completed` provenance all agree. No failFast / cancel
+            // cascade here: the run is already failing fast (that is why this child was cancelled).
+            canceled.add(settled.stepId);
+            stepRuntime.set(settled.stepId, {
+                // `stepRuntime` (the DAG snapshot) has no canceled tier; the graceful-cancel branch above
+                // likewise renders canceled steps as "failed", so this stays consistent with it.
+                status: "failed",
+                error: settled.err instanceof Error ? settled.err.message : String(settled.err),
+            });
+            stepStatus = "canceled";
+            stepDurationMs = undefined;
+            await emitDagSnapshot();
+        }
+        else {
+            failed.add(settled.stepId);
+            const cause = childWasBudgetExceeded || isBudgetExceeded(settled.err) ? "budget_exceeded" : "fail_fast";
+            if (cause === "budget_exceeded" && !budgetExceeded) {
+                budgetExceeded = true;
+                failFast = true;
+                failureReason = "budget_exceeded";
+            }
+            else if (!failFast) {
+                failFast = true;
+                failureReason = settled.err instanceof Error ? settled.err.message : String(settled.err);
+            }
+            await cancelInFlight(inFlight, cause, canceledByParent);
+            stepRuntime.set(settled.stepId, {
+                status: "failed",
+                error: settled.err instanceof Error ? settled.err.message : String(settled.err),
+            });
+            stepStatus = "failed";
+            // A child that settled by throwing carries no durable duration.
+            stepDurationMs = undefined;
+            await emitDagSnapshot();
+        }
+        // One `step_completed` provenance emission per settled child — this
+        // settlement loop is the only site that observes every executed step.
+        // Plain guarded call (NOT `DBOS.runStep`): body re-execution on recovery
+        // must re-fire it, and the checkpointed `DBOS.now()` read keeps the
+        // timestamp replay-stable. Never-dispatched steps reach no settlement, so
+        // they emit nothing by design.
+        const settledAtMs = await DBOS.now();
+        emitProvenanceGuarded(deps, {
+            type: "step_completed",
+            analysisId: input.analysisId,
+            runId,
+            stepId: settled.stepId,
+            status: stepStatus,
+            ...(stepDurationMs !== undefined ? { durationMs: stepDurationMs } : {}),
+            atMs: settledAtMs,
+        });
+    }
+    await drainBudgetExceededNotifications(budgetExceededChildIds);
+    return { completed, failed, canceled, budgetExceeded, failureReason };
+}
+async function drainBudgetExceededNotifications(childIds) {
+    while (true) {
+        const msg = await DBOS.recv(BUDGET_EXCEEDED_TOPIC, 0);
+        if (!msg)
+            return;
+        childIds.add(msg.childWorkflowId);
+    }
+}
+async function cancelInFlight(inFlight, cause, canceledByParent) {
+    const ids = [...inFlight.keys()];
+    // Record every id BEFORE the cancel so the settlement loop classifies the resulting
+    // DBOS-cancellation rejection as canceled rather than failed (see `canceledByParent`).
+    for (const childId of ids)
+        canceledByParent.add(childId);
+    await Promise.allSettled(ids.map((childId) => DBOS.cancelWorkflow(childId)));
+    for (let i = 0; i < ids.length; i++) {
+        recordCancelledChild({ cause });
+    }
+}
+async function collectAndComplete(args) {
+    const { input, runId, workflowId, startedAtMs, completed, failed, canceled, budgetExceeded, failureReason, forceFailed, deps } = args;
+    const status = forceFailed
+        ? "failed"
+        : deriveFinalStatus({
+            totalSteps: input.steps.length,
+            completed,
+            failed,
+            canceled,
+            budgetExceeded,
+        });
+    const chargeReason = status === "completed" || status === "partial" ? "ok" : status === "failed" ? "error" : budgetExceeded ? "budget_exceeded" : "canceled";
+    const revokeReason = status === "completed" || status === "partial"
+        ? "workflow-completed"
+        : status === "failed"
+            ? "workflow-failed"
+            : budgetExceeded
+                ? "workflow-suspended"
+                : "workflow-canceled";
+    // Terminal clock read + run-completion provenance, emitted BEFORE the `cortex_runs`
+    // status write below — and thus before any of the terminal cleanup steps. The CLI
+    // watches the run by POLLING that row (`run.ts` `waitForRunTerminal`) and shuts down
+    // the instant it leaves `running`, flushing provenance and then `process.exit()`ing;
+    // emitting the record only after the status write races that shutdown and can drop
+    // the run's terminal provenance entirely. Emitting first makes the record dirty
+    // (synchronously, via the bus) before the row can be observed terminal, so the CLI's
+    // flush always captures it. Not step-wrapped — a DBOS recovery replay must re-fire it;
+    // `DBOS.now()` is checkpointed, so the span stays replay-stable (see `RunProvenanceEvent`).
+    // The duration measures from the `run_started` read threaded in as `startedAtMs`.
+    const terminalAtMs = await DBOS.now();
+    emitProvenanceGuarded(deps, {
+        type: "run_completed",
+        analysisId: input.analysisId,
+        runId,
+        status,
+        atMs: terminalAtMs,
+        durationMs: terminalAtMs - startedAtMs,
+    });
+    try {
+        await DBOS.runStep(async () => {
+            unwrapOrThrow(await updateRunStatus(deps.pool, runId, status, failureReason ?? (status === "canceled" && !budgetExceeded ? "external_cancel" : null)));
+        }, { name: "persist-final-status" });
+    }
+    catch (err) {
+        console.error(`[executeAnalysis] persist-final-status failed for run ${runId} (status=${status}):`, err);
+    }
+    // Sweep never-started steps to `skipped` — but ONLY on genuinely-terminal
+    // paths. The gate is the BRANCH, not the written run status: the resumable
+    // 402 pause below also writes "canceled", yet its pending rows must survive
+    // for the resumed workflow to execute. Log-don't-rollback, like every other
+    // finalisation step here.
+    if (!(budgetExceeded && !forceFailed)) {
+        try {
+            await DBOS.runStep(async () => {
+                unwrapOrThrow(await sweepPendingStepExecutions(deps.pool, runId));
+            }, { name: "sweep-pending-steps" });
+        }
+        catch (err) {
+            console.error(`[executeAnalysis] sweep-pending-steps failed for run ${runId}:`, err);
+        }
+    }
+    // A forced failure (synthesis threw) is terminal, not a resumable budget
+    // pause — never suspend it, even when the budget was also exceeded.
+    if (budgetExceeded && !forceFailed) {
+        try {
+            await DBOS.runStep(async () => {
+                unwrapOrThrow(await suspendAnalysisQuery(deps.pool, input.analysisId));
+            }, { name: "suspend-analysis" });
+        }
+        catch (err) {
+            console.error(`[executeAnalysis] suspend-analysis failed for run ${runId}:`, err);
+        }
+    }
+    try {
+        await DBOS.runStep(() => deps.runCharge.close({
+            analysisId: input.analysisId,
+            runId,
+            reason: chargeReason,
+            session: input.runSession,
+        }), { name: "close-running-charge" });
+    }
+    catch (err) {
+        console.error(`[executeAnalysis] closeRunningCharge failed for run ${runId} (reason=${chargeReason}):`, err);
+    }
+    // Revoke the run authorization for the terminal run state. Ownership
+    // defaults to true for inputs persisted before the field existed (a
+    // workflow recovered across this deploy) — those were always Cortex-owned.
+    const authorization = {
+        runSession: input.runSession,
+        ownsMandate: input.ownsMandate ?? true, // oss-core-managed-ok
+    };
+    try {
+        await DBOS.runStep(() => deps.runAuthorizer.revoke(authorization, revokeReason), { name: "revoke-run-auth" });
+    }
+    catch (err) {
+        console.error(`[executeAnalysis] revokeRunAuthorization failed for run ${runId} (reason=${revokeReason}):`, err);
+    }
+    // The `run_completed` provenance was already emitted above (before the status write, to
+    // beat the CLI's poll-and-shutdown). These branches only fan out the UI stream part, whose
+    // completed/failed shapes genuinely differ — the provenance record does not.
+    if (status === "completed" || status === "partial") {
+        const artifactCount = await DBOS.runStep(() => countArtifactsForRun(deps.pool, input.analysisId, runId), { name: "count-run-artifacts" }).catch(() => 0);
+        await emitStreamPart({
+            type: "data-run-completed",
+            runId,
+            status,
+            completedSteps: completed.size,
+            totalSteps: input.steps.length,
+            artifactCount,
+            findings: args.findings.map((f) => ({
+                title: f.title,
+                confidence: f.confidence,
+            })),
+            ...(status === "partial"
+                ? {
+                    note: `${completed.size} of ${input.steps.length} steps completed; results are partial.`,
+                }
+                : {}),
+        });
+    }
+    else {
+        await emitStreamPart({
+            type: "data-run-failed",
+            runId,
+            error: failureReason ?? (budgetExceeded ? "Run paused: insufficient budget" : "Run canceled before completion"),
+            ...(budgetExceeded ? { reason: "budget_exceeded" } : {}),
+        });
+    }
+    return {
+        runId,
+        workflowId,
+        status,
+        completedSteps: [...completed],
+        failedSteps: [...failed],
+        canceledSteps: [...canceled],
+    };
+}
+/**
+ * `"running"` is excluded from the return: it is only the joined-existing
+ * trigger outcome, never a settled terminal status. Narrowing it away here lets
+ * the terminal `run_completed` provenance emission carry the terminal-only
+ * status vocabulary without a cast.
+ */
+function deriveFinalStatus(args) {
+    const { totalSteps, completed, failed, canceled, budgetExceeded } = args;
+    if (budgetExceeded)
+        return "canceled";
+    if (failed.size > 0) {
+        return completed.size > 0 ? "partial" : "failed";
+    }
+    if (canceled.size > 0)
+        return "canceled";
+    return completed.size === totalSteps ? "completed" : "partial";
+}