@openclaw/lobster 2026.9.4 → 2026.9.5

package/node_modules/@clawdbot/lobster/dist/src/workflows/file.js

@@ -7,13 +7,48 @@ import { PassThrough } from "node:stream";
 import { parsePipeline } from "../parser.js";
 import { runPipeline } from "../runtime.js";
 import { encodeToken, decodeToken } from "../token.js";
-import { createApprovalIndex, deleteStateJson, readStateJson, writeStateJson, } from "../state/store.js";
+import { cleanupApprovalIndexByStateKey, consumeResumeState, createApprovalIndex, deleteStateJsonWithBoundedResumeCleanup, deleteResumeStateWithRollback, deleteStateJson, deleteUnconsumedResumeState, isConsumedResumeState, readStateJsonWithLock, restoreConsumedResumeState, writeStateJson, } from "../state/store.js";
 import { readLineFromStream } from "../read_line.js";
-import { resolveInlineShellCommand } from "../shell.js";
 import { compileCached } from "../validation.js";
-import { CostTracker } from "../core/cost_tracker.js";
+import { createLlmSpendLedger, llmProvenanceOf, restoreLlmProvenance, } from "../commands/stdlib/llm_invoke.js";
+import { CostLimitExceededError, CostTracker } from "../core/cost_tracker.js";
 import { withRetry, resolveRetryConfig } from "../core/retry.js";
 import { RequestInputResumeError, validateCommandInputState, } from "../input_request.js";
+import { runAbortableProcess } from "../abortable_process.js";
+const NON_RETRYABLE_STEP_ERROR = Symbol("lobster.nonRetryableStepError");
+function createRecord(source) {
+    const record = Object.create(null);
+    if (!source)
+        return record;
+    for (const [key, value] of Object.entries(source))
+        record[key] = value;
+    return record;
+}
+function markStepErrorNonRetryable(error) {
+    const normalized = error instanceof Error ? error : new Error(String(error));
+    Object.defineProperty(normalized, NON_RETRYABLE_STEP_ERROR, { value: true });
+    return normalized;
+}
+function isStepErrorNonRetryable(error) {
+    return Boolean(error instanceof Error &&
+        error[NON_RETRYABLE_STEP_ERROR]);
+}
+const PARALLEL_BRANCH_ABORT_SETTLE_TIMEOUT_MS = 500;
+async function waitForCancelledParallelBranches(branchesSettled) {
+    let timeoutId;
+    try {
+        await Promise.race([
+            branchesSettled,
+            new Promise((_resolve, reject) => {
+                timeoutId = setTimeout(() => reject(new Error("Parallel branches did not settle after cancellation")), PARALLEL_BRANCH_ABORT_SETTLE_TIMEOUT_MS);
+            }),
+        ]);
+    }
+    finally {
+        if (timeoutId !== undefined)
+            clearTimeout(timeoutId);
+    }
+}
 export class WorkflowResumeArgumentError extends Error {
     constructor(message) {
         super(message);
@@ -347,7 +382,7 @@ export async function loadWorkflowFile(filePath) {
     return parsed;
 }
 export function resolveWorkflowArgs(argDefs, provided) {
-    const resolved = {};
+    const resolved = createRecord();
     if (argDefs) {
         for (const [key, def] of Object.entries(argDefs)) {
             if (def && typeof def === "object" && "default" in def) {
@@ -364,11 +399,74 @@ export function resolveWorkflowArgs(argDefs, provided) {
 }
 export async function runWorkflowFile({ filePath, args, ctx, resume, approved, response, cancel, }) {
     const consumedResumeStateKey = resume?.stateKey && typeof resume.stateKey === "string"
-        ? await resolveWorkflowResumeStateKey(ctx.env, resume.stateKey)
+        ? await resolveWorkflowResumeStateKey(ctx.env, resume.stateKey, ctx.signal)
         : null;
+    if (consumedResumeStateKey)
+        ctx._onResumeStateResolved?.(consumedResumeStateKey);
     const resumeState = resume?.stateKey
-        ? await loadWorkflowResumeState(ctx.env, consumedResumeStateKey ?? resume.stateKey)
+        ? await loadWorkflowResumeState(ctx.env, consumedResumeStateKey ?? resume.stateKey, ctx.signal)
         : (resume ?? null);
+    let resumedExecutionStarted = false;
+    let resumeStateConsumed = false;
+    let executionBoundaryStarted = false;
+    const executionSignalListeners = [];
+    let executionStart;
+    const markExecutionStarted = async (signal) => {
+        if (!consumedResumeStateKey && !ctx._onExecutionStarted && !ctx._onExecutionInterrupted)
+            return;
+        if (executionStart)
+            return executionStart;
+        executionStart = (async () => {
+            signal?.throwIfAborted();
+            if (consumedResumeStateKey) {
+                const consumption = await consumeResumeState({
+                    env: ctx.env,
+                    key: consumedResumeStateKey,
+                    expectedState: resumeState,
+                    signal,
+                });
+                if (!consumption.consumed) {
+                    throw new Error("Workflow resume state not found");
+                }
+                if (consumption.signalAbortedAfterCommit) {
+                    await restoreConsumedResumeState({
+                        env: ctx.env,
+                        key: consumedResumeStateKey,
+                        expectedState: resumeState,
+                        claimId: consumption.claimId,
+                    });
+                    signal?.throwIfAborted();
+                }
+                signal?.throwIfAborted();
+                resumeStateConsumed = true;
+                resumedExecutionStarted = true;
+            }
+            await ctx._onExecutionStarted?.();
+            executionBoundaryStarted = true;
+            if (!signal)
+                return;
+            const onAbort = () => {
+                ctx._onExecutionInterrupted?.();
+            };
+            if (signal.aborted) {
+                onAbort();
+                return;
+            }
+            signal.addEventListener("abort", onAbort, { once: true });
+            executionSignalListeners.push({ signal, onAbort });
+        })();
+        try {
+            await executionStart;
+        }
+        catch (err) {
+            // A per-attempt timeout can abort while this invocation is only waiting
+            // to claim its resume state. Keep the successful boundary memoized, but
+            // let the workflow retry make a fresh claim after that pre-dispatch wait.
+            if (!resumeStateConsumed && !executionBoundaryStarted)
+                executionStart = undefined;
+            throw err;
+        }
+    };
     if (resumeState?.approvalStepId && resumeState?.inputStepId) {
         throw new Error("Invalid workflow resume state");
     }
@@ -381,14 +479,26 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
         }
         if (cancel === true || approved === false) {
             if (consumedResumeStateKey) {
-                await deleteStateJson({ env: ctx.env, key: consumedResumeStateKey });
+                const deletion = await deleteUnconsumedResumeState({
+                    env: ctx.env,
+                    key: consumedResumeStateKey,
+                    signal: ctx.signal,
+                });
+                if (deletion !== "deleted")
+                    throw new Error("Workflow resume state not found");
             }
             return { status: "cancelled", output: [] };
         }
     }
     if (resumeState?.inputStepId && cancel === true) {
         if (consumedResumeStateKey) {
-            await deleteStateJson({ env: ctx.env, key: consumedResumeStateKey });
+            const deletion = await deleteUnconsumedResumeState({
+                env: ctx.env,
+                key: consumedResumeStateKey,
+                signal: ctx.signal,
+            });
+            if (deletion !== "deleted")
+                throw new Error("Workflow resume state not found");
         }
         return { status: "cancelled", output: [] };
     }
@@ -408,7 +518,7 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
         const stepIndexById = new Map(steps.map((step, idx) => [step.id, idx]));
         const results = resumeState?.steps
             ? cloneResults(resumeState.steps)
-            : {};
+            : createRecord();
         const startIndex = resumeState?.resumeAtIndex ?? 0;
         if (resumeState?.approvalStepId && typeof approved === "boolean") {
             const previous = results[resumeState.approvalStepId] ?? { id: resumeState.approvalStepId };
@@ -459,11 +569,6 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                     stepId: resumeState.inputStepId,
                     response,
                     pipelineInput: resumeState.pipelineInput,
-                    onConsumed: consumedResumeStateKey
-                        ? async () => {
-                            await deleteStateJson({ env: ctx.env, key: consumedResumeStateKey });
-                        }
-                        : undefined,
                 };
             }
             else {
@@ -492,8 +597,30 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
             return dryRunWorkflow({ steps, resolvedArgs, results, startIndex, ctx });
         }
         const costTracker = new CostTracker(CostTracker.parsePricingFromEnv(ctx.env, ctx.stderr), ctx.stderr);
+        // A gate pauses a run, it does not reset its spend: seed this run with what the paused
+        // run recorded, so `_meta.cost` covers the whole workflow and `cost_limit` cannot be
+        // walked past one gate at a time. A replay of a call billed before the pause then stays
+        // exempt, which is what an empty ledger already means for a charge someone else settled.
+        const pausedCost = resumeState && "cost" in resumeState ? resumeState.cost : undefined;
+        costTracker.restore(pausedCost?.byStep);
+        // One ledger per run: a live call this run paid for can be recovered from the replay a
+        // retry produced, and a cache entry written by any other run has nothing to claim here.
+        const llmSpendLedger = createLlmSpendLedger(composingSpendLedger(ctx));
+        // A pipeline that paused at an `ask` gate had already paid for its LLM call without
+        // billing it, so the charge travels in the same resume state and reopens here. It is the
+        // same run continuing, which is why this cannot hand a charge to an unrelated one.
+        llmSpendLedger.restore(resumeState && "llmSpend" in resumeState ? resumeState.llmSpend : undefined);
+        // The completed steps come back as JSON, so an LLM output among them has lost everything
+        // this process attached to it. A later step that re-emits one hands on a copy that still
+        // names its call, and the restored record is what lets that copy be recognized as one the
+        // paused run already paid for rather than billed a second time on top of `cost`.
+        llmSpendLedger.restoreSettled(resumeState && "llmBilled" in resumeState ? resumeState.llmBilled : undefined);
+        // Completed results lost their symbol keys when the pause was persisted. Restore those
+        // private marks only at this trusted resume boundary; ordinary command JSON stays billable.
+        restoreLlmProvenance(results, resumeState && "llmBilled" in resumeState ? resumeState.llmBilled : undefined);
         let lastStepId = resumeState?.inputStepId ?? findLastCompletedStepId(steps, results);
         for (let idx = startIndex; idx < steps.length; idx++) {
+            ctx.signal?.throwIfAborted();
             const step = steps[idx];
             if (!evaluateCondition(step.when ?? step.condition, results)) {
                 results[step.id] = { id: step.id, skipped: true };
@@ -515,40 +642,65 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                         subject,
                         maxEnvelopeBytes: resolveToolEnvelopeMaxBytes(ctx.env),
                     });
-                    const stateKey = await saveWorkflowResumeState(ctx.env, {
-                        filePath: resolvedFilePath,
-                        resumeAtIndex: idx + 1,
-                        steps: results,
-                        args: resolvedArgs,
-                        inputStepId: step.id,
-                        inputSchema: step.input.responseSchema,
-                        // Preserve the full resolved subject for resume semantics; the tool
-                        // envelope may contain a truncated preview to stay within size limits.
-                        inputSubject: subject,
-                        createdAt: new Date().toISOString(),
-                    });
-                    if (consumedResumeStateKey && consumedResumeStateKey !== stateKey) {
-                        await deleteStateJson({ env: ctx.env, key: consumedResumeStateKey });
+                    let stateKey;
+                    try {
+                        stateKey = await saveWorkflowResumeState(ctx.env, {
+                            filePath: resolvedFilePath,
+                            resumeAtIndex: idx + 1,
+                            steps: results,
+                            args: resolvedArgs,
+                            inputStepId: step.id,
+                            inputSchema: step.input.responseSchema,
+                            // Preserve the full resolved subject for resume semantics; the tool
+                            // envelope may contain a truncated preview to stay within size limits.
+                            inputSubject: subject,
+                            ...(costTracker.hasUsage() ? { cost: costTracker.getSummary() } : null),
+                            ...(llmSpendLedger.outstanding().length
+                                ? { llmSpend: llmSpendLedger.outstanding() }
+                                : null),
+                            ...(llmSpendLedger.settled().length
+                                ? { llmBilled: llmSpendLedger.settled() }
+                                : null),
+                            supersededResumeStateKeys: collectSupersededWorkflowResumeStateKeys(consumedResumeStateKey, resumeState),
+                            createdAt: new Date().toISOString(),
+                        }, ctx.signal);
+                        const replaced = await replaceWorkflowResumeState({
+                            env: ctx.env,
+                            previousStateKey: consumedResumeStateKey,
+                            expectedPreviousState: resumeState,
+                            previousStateConsumed: resumeStateConsumed,
+                            replacementStateKey: stateKey,
+                            signal: ctx.signal,
+                        });
+                        if (!replaced) {
+                            throw new Error("Workflow resume state not found");
+                        }
+                        const resumeToken = encodeToken({
+                            protocolVersion: 1,
+                            v: 1,
+                            kind: "workflow-file",
+                            stateKey,
+                        });
+                        return {
+                            status: "needs_input",
+                            output: [],
+                            requiresInput: {
+                                ...inputRequest,
+                                resumeToken,
+                            },
+                        };
+                    }
+                    catch (err) {
+                        if (stateKey)
+                            await discardWorkflowResumeState(ctx.env, stateKey);
+                        throw err;
                     }
-                    const resumeToken = encodeToken({
-                        protocolVersion: 1,
-                        v: 1,
-                        kind: "workflow-file",
-                        stateKey,
-                    });
-                    return {
-                        status: "needs_input",
-                        output: [],
-                        requiresInput: {
-                            ...inputRequest,
-                            resumeToken,
-                        },
-                    };
                 }
                 ctx.stdout.write(`${step.input.prompt}\n`);
                 ctx.stdout.write("Enter JSON response: ");
                 const raw = await readLineFromStream(ctx.stdin, {
                     timeoutMs: parseApprovalTimeoutMs(ctx.env),
+                    signal: ctx.signal,
                 });
                 const parsed = parseResponseJson(String(raw ?? "").trim());
                 validateInputResponse({
@@ -564,130 +716,168 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                 lastStepId = step.id;
                 continue;
             }
-            if (typeof step.for_each === "string" && Array.isArray(step.steps)) {
-                const itemsRef = resolveInputValue(step.for_each, resolvedArgs, results);
-                if (!Array.isArray(itemsRef)) {
-                    throw new Error(`Workflow step ${step.id} for_each: expected array, got ${typeof itemsRef}`);
-                }
-                const itemVar = step.item_var ?? "item";
-                const indexVar = step.index_var ?? "index";
-                const batchSize = step.batch_size ?? 1;
-                const iterationResults = [];
-                for (let itemIdx = 0; itemIdx < itemsRef.length; itemIdx++) {
-                    if (step.pause_ms && itemIdx > 0 && itemIdx % batchSize === 0) {
-                        await abortableSleep(step.pause_ms, ctx.signal);
-                    }
-                    const item = itemsRef[itemIdx];
-                    const scopedResults = { ...results };
-                    scopedResults[itemVar] = {
-                        id: itemVar,
-                        json: item,
-                        stdout: typeof item === "string" ? item : JSON.stringify(item),
-                    };
-                    scopedResults[indexVar] = {
-                        id: indexVar,
-                        json: itemIdx,
-                        stdout: String(itemIdx),
-                    };
-                    for (const subStep of step.steps) {
-                        if (!evaluateCondition(subStep.when ?? subStep.condition, scopedResults)) {
-                            scopedResults[subStep.id] = { id: subStep.id, skipped: true };
-                            continue;
-                        }
-                        const loopEnvBase = mergeEnv(ctx.env, workflow.env, step.env, resolvedArgs, scopedResults);
-                        const subEnv = subStep.env
-                            ? mergeEnv(loopEnvBase, undefined, subStep.env, resolvedArgs, scopedResults)
-                            : loopEnvBase;
-                        const subCwd = resolveCwd(subStep.cwd ?? step.cwd ?? workflow.cwd, resolvedArgs) ?? ctx.cwd;
-                        const subExecution = getStepExecution(subStep);
-                        let subResult;
-                        if (subExecution.kind === "shell") {
-                            const command = resolveTemplate(subExecution.value, resolvedArgs, scopedResults);
-                            const stdinValue = resolveShellStdin(subStep.stdin, resolvedArgs, scopedResults);
-                            const { stdout } = await runShellCommand({
-                                command,
-                                stdin: stdinValue,
-                                env: subEnv,
-                                cwd: subCwd,
-                                signal: ctx.signal,
-                            });
-                            subResult = { id: subStep.id, stdout, json: parseJson(stdout) };
-                        }
-                        else if (subExecution.kind === "pipeline") {
-                            if (!ctx.registry) {
-                                throw new Error(`Workflow step ${step.id} for_each sub-step ${subStep.id} requires a command registry for pipeline execution`);
-                            }
-                            const pipelineText = resolveTemplate(subExecution.value, resolvedArgs, scopedResults);
-                            const inputValue = resolveInputValue(subStep.stdin, resolvedArgs, scopedResults);
-                            subResult = await runPipelineStep({
-                                stepId: subStep.id,
-                                pipelineText,
-                                inputValue,
-                                ctx,
-                                env: subEnv,
-                                cwd: subCwd,
-                                requestInputEnabled: false,
-                            });
-                        }
-                        else {
-                            const inputValue = resolveInputValue(subStep.stdin, resolvedArgs, scopedResults);
-                            subResult = createSyntheticStepResult(subStep.id, inputValue);
-                        }
-                        scopedResults[subStep.id] = subResult;
-                        trackStepCost(costTracker, `${step.id}.${subStep.id}`, subResult);
-                        if (workflow.cost_limit) {
-                            costTracker.checkLimit(workflow.cost_limit, ctx.stderr);
-                        }
-                    }
-                    const iterResult = { [itemVar]: item, [indexVar]: itemIdx };
-                    for (const subStep of step.steps) {
-                        const subResult = scopedResults[subStep.id];
-                        if (subResult && !subResult.skipped) {
-                            iterResult[subStep.id] =
-                                subResult.json !== undefined ? subResult.json : subResult.stdout;
-                        }
-                    }
-                    iterationResults.push(iterResult);
-                }
-                const loopResult = {
-                    id: step.id,
-                    json: iterationResults,
-                    stdout: JSON.stringify(iterationResults),
-                };
-                results[step.id] = loopResult;
-                lastStepId = step.id;
-                trackStepCost(costTracker, step.id, loopResult);
-                if (workflow.cost_limit) {
-                    costTracker.checkLimit(workflow.cost_limit, ctx.stderr);
-                }
-                continue;
-            }
             const env = mergeEnv(ctx.env, workflow.env, step.env, resolvedArgs, results);
             const cwd = resolveCwd(step.cwd ?? workflow.cwd, resolvedArgs) ?? ctx.cwd;
             const execution = getStepExecution(step);
             const retryConfig = resolveRetryConfig(step.retry);
             const executeStepAttempt = async () => {
+                let attemptHadNonRetryableSideEffect = false;
+                const markNonRetryableSideEffect = () => {
+                    attemptHadNonRetryableSideEffect = true;
+                    ctx._onNonRetryableSideEffect?.();
+                };
+                ctx.signal?.throwIfAborted();
                 // Combine external cancellation and optional per-step timeout into one signal.
                 let stepSignal = ctx.signal;
                 let timeoutId;
+                let stepTimeoutController;
                 if (step.timeout_ms) {
-                    const timeoutController = new AbortController();
-                    timeoutId = setTimeout(() => timeoutController.abort(new Error(`Step '${step.id}' timed out after ${step.timeout_ms}ms`)), step.timeout_ms);
+                    stepTimeoutController = new AbortController();
+                    timeoutId = setTimeout(() => stepTimeoutController.abort(new Error(`Step '${step.id}' timed out after ${step.timeout_ms}ms`)), step.timeout_ms);
                     stepSignal = ctx.signal
-                        ? AbortSignal.any([ctx.signal, timeoutController.signal])
-                        : timeoutController.signal;
+                        ? AbortSignal.any([ctx.signal, stepTimeoutController.signal])
+                        : stepTimeoutController.signal;
                 }
                 let result;
                 let parallelBranchResults = null;
                 try {
-                    if (execution.kind === "parallel") {
+                    if (typeof step.for_each === "string" && Array.isArray(step.steps)) {
+                        const itemsRef = resolveInputValue(step.for_each, resolvedArgs, results);
+                        if (!Array.isArray(itemsRef)) {
+                            throw new Error(`Workflow step ${step.id} for_each: expected array, got ${typeof itemsRef}`);
+                        }
+                        const itemVar = step.item_var ?? "item";
+                        const indexVar = step.index_var ?? "index";
+                        const batchSize = step.batch_size ?? 1;
+                        const iterationResults = [];
+                        for (let itemIdx = 0; itemIdx < itemsRef.length; itemIdx++) {
+                            stepSignal?.throwIfAborted();
+                            if (step.pause_ms && itemIdx > 0 && itemIdx % batchSize === 0) {
+                                await abortableSleep(step.pause_ms, stepSignal);
+                            }
+                            const item = itemsRef[itemIdx];
+                            const scopedResults = createRecord(results);
+                            scopedResults[itemVar] = {
+                                id: itemVar,
+                                json: item,
+                                stdout: typeof item === "string" ? item : JSON.stringify(item),
+                            };
+                            scopedResults[indexVar] = {
+                                id: indexVar,
+                                json: itemIdx,
+                                stdout: String(itemIdx),
+                            };
+                            for (const subStep of step.steps) {
+                                stepSignal?.throwIfAborted();
+                                if (!evaluateCondition(subStep.when ?? subStep.condition, scopedResults)) {
+                                    scopedResults[subStep.id] = { id: subStep.id, skipped: true };
+                                    continue;
+                                }
+                                const loopEnvBase = mergeEnv(ctx.env, workflow.env, step.env, resolvedArgs, scopedResults);
+                                const subEnv = subStep.env
+                                    ? mergeEnv(loopEnvBase, undefined, subStep.env, resolvedArgs, scopedResults)
+                                    : loopEnvBase;
+                                const subCwd = resolveCwd(subStep.cwd ?? step.cwd ?? workflow.cwd, resolvedArgs) ?? ctx.cwd;
+                                const subExecution = getStepExecution(subStep);
+                                let subResult;
+                                if (subExecution.kind === "shell") {
+                                    const command = resolveTemplate(subExecution.value, resolvedArgs, scopedResults);
+                                    const stdinValue = resolveShellStdin(subStep.stdin, resolvedArgs, scopedResults);
+                                    await markExecutionStarted(stepSignal);
+                                    const { stdout } = await runShellCommand({
+                                        command,
+                                        stdin: stdinValue,
+                                        env: subEnv,
+                                        cwd: subCwd,
+                                        signal: stepSignal,
+                                        forceTerminationSignal: ctx.forceTerminationSignal,
+                                        killSignal: () => stepTimeoutController?.signal.aborted
+                                            ? "SIGKILL"
+                                            : undefined,
+                                    });
+                                    subResult = { id: subStep.id, stdout, json: parseJson(stdout) };
+                                }
+                                else if (subExecution.kind === "pipeline") {
+                                    if (!ctx.registry) {
+                                        throw new Error(`Workflow step ${step.id} for_each sub-step ${subStep.id} requires a command registry for pipeline execution`);
+                                    }
+                                    const pipelineText = resolveTemplate(subExecution.value, resolvedArgs, scopedResults);
+                                    const inputValue = resolveInputValue(subStep.stdin, resolvedArgs, scopedResults);
+                                    subResult = await runPipelineStep({
+                                        stepId: subStep.id,
+                                        pipelineText,
+                                        inputValue,
+                                        ctx: {
+                                            ...ctx,
+                                            signal: stepSignal,
+                                            _onNonRetryableSideEffect: markNonRetryableSideEffect,
+                                        },
+                                        llmSpendLedger,
+                                        env: subEnv,
+                                        cwd: subCwd,
+                                        requestInputEnabled: false,
+                                        onExecutionStart: () => markExecutionStarted(stepSignal),
+                                    });
+                                }
+                                else {
+                                    const inputValue = resolveInputValue(subStep.stdin, resolvedArgs, scopedResults);
+                                    subResult = createSyntheticStepResult(subStep.id, inputValue);
+                                }
+                                scopedResults[subStep.id] = subResult;
+                                trackStepCost(costTracker, `${step.id}.${subStep.id}`, subResult, llmSpendLedger);
+                                // Before the next step runs: a charge no item carried is spend the step really made,
+                                // and leaving it for the end would let `cost_limit` wave through every step after the
+                                // one that blew the budget.
+                                settleUnbilledCharges(costTracker, llmSpendLedger);
+                                if (workflow.cost_limit) {
+                                    costTracker.checkLimit(workflow.cost_limit, ctx.stderr);
+                                }
+                            }
+                            const iterResult = createRecord();
+                            iterResult[itemVar] = item;
+                            iterResult[indexVar] = itemIdx;
+                            for (const subStep of step.steps) {
+                                const subResult = scopedResults[subStep.id];
+                                if (subResult && !subResult.skipped) {
+                                    iterResult[subStep.id] =
+                                        subResult.json !== undefined ? subResult.json : subResult.stdout;
+                                }
+                            }
+                            iterationResults.push(Object.fromEntries(Object.entries(iterResult)));
+                        }
+                        result = {
+                            id: step.id,
+                            json: iterationResults,
+                            stdout: JSON.stringify(iterationResults),
+                        };
+                    }
+                    else if (execution.kind === "parallel") {
                         const parallel = execution.value;
                         const wait = parallel.wait ?? "all";
                         const branchAbortController = new AbortController();
-                        const branchSignal = stepSignal
-                            ? AbortSignal.any([stepSignal, branchAbortController.signal])
-                            : branchAbortController.signal;
-                        const shouldForceKill = Boolean(step.timeout_ms || parallel.timeout_ms);
+                        const parallelTimeoutController = new AbortController();
+                        const parallelSignal = stepSignal
+                            ? AbortSignal.any([stepSignal, parallelTimeoutController.signal])
+                            : parallelTimeoutController.signal;
+                        const branchSignal = AbortSignal.any([parallelSignal, branchAbortController.signal]);
+                        const timeoutKillSignal = () => stepTimeoutController?.signal.aborted || parallelTimeoutController.signal.aborted
+                            ? "SIGKILL"
+                            : undefined;
+                        // `wait: "any"` keeps the first branch to answer and abandons the rest, so a
+                        // losing branch spends against a buffer of its own. Only the winner's charges
+                        // reach the run. `wait: "all"` keeps every branch, so those bill straight
+                        // through as before.
+                        const branchLedgers = new Map();
+                        const ledgerFor = (branchId) => {
+                            if (wait !== "any")
+                                return llmSpendLedger;
+                            let buffered = branchLedgers.get(branchId);
+                            if (!buffered) {
+                                buffered = branchSpendBuffer(llmSpendLedger);
+                                branchLedgers.set(branchId, buffered);
+                            }
+                            return buffered;
+                        };
                         const runBranch = async (branch) => {
                             const mergedBranchEnv = { ...(step.env ?? {}), ...(branch.env ?? {}) };
                             const branchEnv = mergeEnv(ctx.env, workflow.env, mergedBranchEnv, resolvedArgs, results);
@@ -701,13 +891,15 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                             if (branchExec.kind === "shell") {
                                 const command = resolveTemplate(branchExec.value, resolvedArgs, results);
                                 const stdinValue = resolveShellStdin(branch.stdin, resolvedArgs, results);
+                                await markExecutionStarted(branchSignal);
                                 const { stdout } = await runShellCommand({
                                     command,
                                     stdin: stdinValue,
                                     env: branchEnv,
                                     cwd: branchCwd,
                                     signal: branchSignal,
-                                    ...(shouldForceKill ? { killSignal: "SIGKILL" } : {}),
+                                    forceTerminationSignal: ctx.forceTerminationSignal,
+                                    killSignal: timeoutKillSignal,
                                 });
                                 return {
                                     branchId: branch.id,
@@ -724,10 +916,16 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                                     stepId: branch.id,
                                     pipelineText,
                                     inputValue,
-                                    ctx: { ...ctx, signal: branchSignal },
+                                    ctx: {
+                                        ...ctx,
+                                        signal: branchSignal,
+                                        _onNonRetryableSideEffect: markNonRetryableSideEffect,
+                                    },
+                                    llmSpendLedger: ledgerFor(branch.id),
                                     env: branchEnv,
                                     cwd: branchCwd,
                                     requestInputEnabled: false,
+                                    onExecutionStart: () => markExecutionStarted(branchSignal),
                                 });
                                 return { branchId: branch.id, result: branchResult };
                             }
@@ -737,26 +935,27 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                         const timeoutPromise = parallel.timeout_ms
                             ? new Promise((_resolve, reject) => {
                                 parallelTimeoutId = setTimeout(() => {
-                                    branchAbortController.abort();
+                                    parallelTimeoutController.abort(new Error(`Parallel step ${step.id} timed out after ${parallel.timeout_ms}ms`));
                                     reject(new Error(`Parallel step ${step.id} timed out after ${parallel.timeout_ms}ms`));
                                 }, parallel.timeout_ms);
                             })
                             : null;
+                        const branchPromises = parallel.branches.map((branch) => runBranch(branch));
+                        const branchesSettled = Promise.allSettled(branchPromises);
                         try {
                             if (wait === "any") {
-                                const branchPromises = parallel.branches.map((branch) => runBranch(branch));
                                 const winner = (await (timeoutPromise
                                     ? Promise.race([...branchPromises, timeoutPromise])
                                     : Promise.race(branchPromises)));
-                                parallelBranchResults = { [winner.branchId]: winner.result };
-                                branchAbortController.abort();
+                                parallelBranchResults = createRecord();
+                                parallelBranchResults[winner.branchId] = winner.result;
+                                branchLedgers.get(winner.branchId)?.release();
                             }
                             else {
-                                const branchPromises = parallel.branches.map((branch) => runBranch(branch));
                                 const settled = (await (timeoutPromise
                                     ? Promise.race([Promise.allSettled(branchPromises), timeoutPromise])
-                                    : Promise.allSettled(branchPromises)));
-                                parallelBranchResults = {};
+                                    : branchesSettled));
+                                parallelBranchResults = createRecord();
                                 for (const entry of settled) {
                                     if (entry.status === "rejected") {
                                         throw new Error(`Parallel branch failed: ${entry.reason?.message ?? String(entry.reason)}`);
@@ -768,11 +967,18 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                         finally {
                             if (parallelTimeoutId !== undefined)
                                 clearTimeout(parallelTimeoutId);
+                            const branchesCancelled = wait === "any" || parallelSignal.aborted;
+                            if (branchesCancelled)
+                                branchAbortController.abort();
+                            if (branchesCancelled)
+                                await waitForCancelledParallelBranches(branchesSettled);
+                            else
+                                await branchesSettled;
                         }
-                        const merged = {};
-                        for (const [branchId, branchResult] of Object.entries(parallelBranchResults ?? {})) {
-                            merged[branchId] = branchResult.json;
-                        }
+                        const merged = Object.fromEntries(Object.entries(parallelBranchResults ?? {}).map(([branchId, branchResult]) => [
+                            branchId,
+                            branchResult.json,
+                        ]));
                         result = {
                             id: step.id,
                             json: merged,
@@ -800,10 +1006,25 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                         const childActive = new Set(activeWorkflows);
                         childActive.add(canonicalWorkflowPath);
                         const subArgs = resolveWorkflowStepArgs(step.workflow_args, resolvedArgs, results);
+                        stepSignal?.throwIfAborted();
                         const subResult = await runWorkflowFile({
                             filePath: resolvedWorkflowPath,
                             args: subArgs,
-                            ctx: { ...ctx, env, cwd, _activeWorkflows: childActive },
+                            ctx: {
+                                ...ctx,
+                                env,
+                                cwd,
+                                signal: stepSignal,
+                                _activeWorkflows: childActive,
+                                _llmSpendLedger: stepScopedLedger(llmSpendLedger, step.id),
+                                _onExecutionStarted: async () => {
+                                    await markExecutionStarted(stepSignal);
+                                },
+                                _onExecutionInterrupted: () => {
+                                    ctx._onExecutionInterrupted?.();
+                                },
+                                _onNonRetryableSideEffect: markNonRetryableSideEffect,
+                            },
                         });
                         if (subResult.status === "needs_approval" || subResult.status === "needs_input") {
                             const resumeToken = subResult.requiresApproval?.resumeToken ?? subResult.requiresInput?.resumeToken;
@@ -823,17 +1044,26 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                         const json = subResult.output.length === 1 ? subResult.output[0] : subResult.output;
                         const stdout = subResult.output.length ? serializeValueForStdout(json) : "";
                         result = { id: step.id, stdout, json };
+                        // A child ending in a rendered pipeline (`llm.invoke | json`) prints its items
+                        // and returns JSON parsed back out of that print, which no symbol survives.
+                        // Carry the child's own items across the composition boundary so this run
+                        // applies to them the same replay rule it applies to its own pipelines.
+                        const composedItems = pipelineSourceItems(subResult);
+                        if (composedItems)
+                            attachPipelineSourceItems(result, composedItems);
                     }
                     else if (execution.kind === "shell") {
                         const command = resolveTemplate(execution.value, resolvedArgs, results);
                         const stdinValue = resolveShellStdin(step.stdin, resolvedArgs, results);
+                        await markExecutionStarted(stepSignal);
                         const { stdout } = await runShellCommand({
                             command,
                             stdin: stdinValue,
                             env,
                             cwd,
                             signal: stepSignal,
-                            ...(step.timeout_ms ? { killSignal: "SIGKILL" } : {}),
+                            forceTerminationSignal: ctx.forceTerminationSignal,
+                            killSignal: () => stepTimeoutController?.signal.aborted ? "SIGKILL" : undefined,
                         });
                         result = { id: step.id, stdout, json: parseJson(stdout) };
                     }
@@ -847,7 +1077,12 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                             stepId: step.id,
                             pipelineText,
                             inputValue,
-                            ctx: { ...ctx, signal: stepSignal },
+                            ctx: {
+                                ...ctx,
+                                signal: stepSignal,
+                                _onNonRetryableSideEffect: markNonRetryableSideEffect,
+                            },
+                            llmSpendLedger,
                             env,
                             cwd,
                             resume: resumedPipelineInput?.stepId === step.id
@@ -857,14 +1092,23 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                                     onConsumed: resumedPipelineInput.onConsumed,
                                 }
                                 : undefined,
+                            onExecutionStart: () => markExecutionStarted(stepSignal),
                         });
                     }
                     else {
+                        if (execution.kind !== "none")
+                            await markExecutionStarted(stepSignal);
                         const inputValue = resolveInputValue(step.stdin, resolvedArgs, results);
                         result = createSyntheticStepResult(step.id, inputValue);
                     }
                     return { result, parallelBranchResults };
                 }
+                catch (error) {
+                    if (attemptHadNonRetryableSideEffect) {
+                        throw markStepErrorNonRetryable(error);
+                    }
+                    throw error;
+                }
                 finally {
                     if (timeoutId !== undefined)
                         clearTimeout(timeoutId);
@@ -878,8 +1122,19 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                         signal: ctx.signal,
                         shouldRetry: (error) => {
                             if (error instanceof WorkflowPipelineInputSuspension ||
-                                error instanceof RequestInputResumeError) {
+                                error instanceof RequestInputResumeError ||
+                                error instanceof CostLimitExceededError) {
+                                return false;
+                            }
+                            if (isStepErrorNonRetryable(error)) {
+                                return false;
+                            }
+                            if (ctx.signal?.aborted)
                                 return false;
+                            if (workflow.cost_limit?.action === "stop") {
+                                // Failed pipelines can discard output after the provider was paid.
+                                settleUnbilledCharges(costTracker, llmSpendLedger);
+                                costTracker.checkLimit(workflow.cost_limit, ctx.stderr);
                             }
                             const message = error?.message ?? String(error);
                             return !/halted (for approval inside|before completion at) pipeline/.test(message);
@@ -902,40 +1157,67 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                         subject: err.request.subject,
                         maxEnvelopeBytes: resolveToolEnvelopeMaxBytes(ctx.env),
                     });
-                    const stateKey = await saveWorkflowResumeState(ctx.env, {
-                        filePath: resolvedFilePath,
-                        resumeAtIndex: idx,
-                        steps: results,
-                        args: resolvedArgs,
-                        inputStepId: err.stepId,
-                        inputKind: "pipeline_command",
-                        inputSchema: err.request.responseSchema,
-                        inputSubject: err.request.subject,
-                        pipelineInput: err.pipelineInput,
-                        createdAt: new Date().toISOString(),
-                    });
-                    if (consumedResumeStateKey && consumedResumeStateKey !== stateKey) {
-                        await deleteStateJson({ env: ctx.env, key: consumedResumeStateKey });
+                    let stateKey;
+                    try {
+                        stateKey = await saveWorkflowResumeState(ctx.env, {
+                            filePath: resolvedFilePath,
+                            resumeAtIndex: idx,
+                            steps: results,
+                            args: resolvedArgs,
+                            inputStepId: err.stepId,
+                            inputKind: "pipeline_command",
+                            inputSchema: err.request.responseSchema,
+                            inputSubject: err.request.subject,
+                            pipelineInput: err.pipelineInput,
+                            ...(costTracker.hasUsage() ? { cost: costTracker.getSummary() } : null),
+                            ...(llmSpendLedger.outstanding().length
+                                ? { llmSpend: llmSpendLedger.outstanding() }
+                                : null),
+                            ...(llmSpendLedger.settled().length
+                                ? { llmBilled: llmSpendLedger.settled() }
+                                : null),
+                            supersededResumeStateKeys: collectSupersededWorkflowResumeStateKeys(consumedResumeStateKey, resumeState),
+                            createdAt: new Date().toISOString(),
+                        }, ctx.signal);
+                        const replaced = await replaceWorkflowResumeState({
+                            env: ctx.env,
+                            previousStateKey: consumedResumeStateKey,
+                            expectedPreviousState: resumeState,
+                            previousStateConsumed: resumeStateConsumed,
+                            replacementStateKey: stateKey,
+                            signal: ctx.signal,
+                        });
+                        if (!replaced) {
+                            throw new Error("Workflow resume state not found");
+                        }
+                        const resumeToken = encodeToken({
+                            protocolVersion: 1,
+                            v: 1,
+                            kind: "workflow-file",
+                            stateKey,
+                        });
+                        return {
+                            status: "needs_input",
+                            output: [],
+                            requiresInput: {
+                                ...inputRequest,
+                                resumeToken,
+                            },
+                        };
+                    }
+                    catch (stateError) {
+                        if (stateKey)
+                            await discardWorkflowResumeState(ctx.env, stateKey);
+                        throw stateError;
                     }
-                    const resumeToken = encodeToken({
-                        protocolVersion: 1,
-                        v: 1,
-                        kind: "workflow-file",
-                        stateKey,
-                    });
-                    return {
-                        status: "needs_input",
-                        output: [],
-                        requiresInput: {
-                            ...inputRequest,
-                            resumeToken,
-                        },
-                    };
                 }
                 if (err instanceof RequestInputResumeError) {
                     throw err;
                 }
-                if (ctx.signal?.aborted && (err?.name === "AbortError" || err?.code === "ABORT_ERR")) {
+                if (err instanceof CostLimitExceededError) {
+                    throw err;
+                }
+                if (ctx.signal?.aborted) {
                     throw err;
                 }
                 if (err?.message &&
@@ -956,20 +1238,36 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                     error: true,
                     errorMessage,
                 };
+                // The step failed after paying a provider: a call it made before failing is spend
+                // this run really made, and its items are gone with the failure. Settling here is
+                // what stops a budget already blown from letting the next step run its side
+                // effects, which is the whole point of `action: stop`.
+                settleUnbilledCharges(costTracker, llmSpendLedger);
+                if (workflow.cost_limit) {
+                    costTracker.checkLimit(workflow.cost_limit, ctx.stderr);
+                }
                 if (policy === "skip_rest") {
                     break;
                 }
                 continue;
             }
+            // One accounting group per step: its parallel branches and its own result settle
+            // together, so no branch can be billed for a charge another branch is about to claim.
+            const deferredCharges = [];
             if (parallelBranchResults) {
                 for (const [branchId, branchResult] of Object.entries(parallelBranchResults)) {
                     results[branchId] = branchResult;
-                    trackStepCost(costTracker, branchId, branchResult);
+                    trackStepCost(costTracker, branchId, branchResult, llmSpendLedger, deferredCharges);
                 }
             }
             results[step.id] = result;
             lastStepId = step.id;
-            trackStepCost(costTracker, step.id, result);
+            trackStepCost(costTracker, step.id, result, llmSpendLedger, deferredCharges);
+            settleDeferredCosts(costTracker, deferredCharges, llmSpendLedger);
+            // Before the next step runs: a charge no item carried is spend the step really made,
+            // and leaving it for the end would let `cost_limit` wave through every step after the
+            // one that blew the budget.
+            settleUnbilledCharges(costTracker, llmSpendLedger);
             if (workflow.cost_limit) {
                 costTracker.checkLimit(workflow.cost_limit, ctx.stderr);
             }
@@ -980,45 +1278,64 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                     results[step.id].initiatedBy = approvalIdentity.initiatedBy;
                 }
                 if (ctx.mode === "tool" || !isInteractive(ctx.stdin)) {
-                    const stateKey = await saveWorkflowResumeState(ctx.env, {
-                        filePath: resolvedFilePath,
-                        resumeAtIndex: idx + 1,
-                        steps: results,
-                        args: resolvedArgs,
-                        approvalStepId: step.id,
-                        approvalIdentity,
-                        createdAt: new Date().toISOString(),
-                    });
-                    let approvalId;
+                    let stateKey;
                     try {
-                        approvalId = await createApprovalIndex({ env: ctx.env, stateKey });
+                        stateKey = await saveWorkflowResumeState(ctx.env, {
+                            filePath: resolvedFilePath,
+                            resumeAtIndex: idx + 1,
+                            steps: results,
+                            args: resolvedArgs,
+                            approvalStepId: step.id,
+                            approvalIdentity,
+                            ...(costTracker.hasUsage() ? { cost: costTracker.getSummary() } : null),
+                            ...(llmSpendLedger.outstanding().length
+                                ? { llmSpend: llmSpendLedger.outstanding() }
+                                : null),
+                            ...(llmSpendLedger.settled().length
+                                ? { llmBilled: llmSpendLedger.settled() }
+                                : null),
+                            supersededResumeStateKeys: collectSupersededWorkflowResumeStateKeys(consumedResumeStateKey, resumeState),
+                            createdAt: new Date().toISOString(),
+                        }, ctx.signal);
+                        const approvalId = await createApprovalIndex({ env: ctx.env, stateKey });
+                        ctx.signal?.throwIfAborted();
+                        const replaced = await replaceWorkflowResumeState({
+                            env: ctx.env,
+                            previousStateKey: consumedResumeStateKey,
+                            expectedPreviousState: resumeState,
+                            previousStateConsumed: resumeStateConsumed,
+                            replacementStateKey: stateKey,
+                            signal: ctx.signal,
+                        });
+                        if (!replaced) {
+                            throw new Error("Workflow resume state not found");
+                        }
+                        const resumeToken = encodeToken({
+                            protocolVersion: 1,
+                            v: 1,
+                            kind: "workflow-file",
+                            stateKey,
+                        });
+                        return {
+                            status: "needs_approval",
+                            output: [],
+                            requiresApproval: {
+                                ...approval,
+                                resumeToken,
+                                ...(approvalId ? { approvalId } : null),
+                            },
+                        };
                     }
                     catch (err) {
-                        await deleteStateJson({ env: ctx.env, key: stateKey }).catch(() => { });
+                        if (stateKey)
+                            await discardWorkflowResumeState(ctx.env, stateKey);
                         throw err;
                     }
-                    if (consumedResumeStateKey && consumedResumeStateKey !== stateKey) {
-                        await deleteStateJson({ env: ctx.env, key: consumedResumeStateKey });
-                    }
-                    const resumeToken = encodeToken({
-                        protocolVersion: 1,
-                        v: 1,
-                        kind: "workflow-file",
-                        stateKey,
-                    });
-                    return {
-                        status: "needs_approval",
-                        output: [],
-                        requiresApproval: {
-                            ...approval,
-                            resumeToken,
-                            ...(approvalId ? { approvalId } : null),
-                        },
-                    };
                 }
                 ctx.stdout.write(`${approval.prompt} [y/N] `);
                 const answer = await readLineFromStream(ctx.stdin, {
                     timeoutMs: parseApprovalTimeoutMs(ctx.env),
+                    signal: ctx.signal,
                 });
                 if (!/^y(es)?$/i.test(String(answer).trim())) {
                     throw new Error("Not approved");
@@ -1034,17 +1351,81 @@ export async function runWorkflowFile({ filePath, args, ctx, resume, approved, r
                     results[step.id].approvedBy = approvedBy;
             }
         }
+        // Whatever a skipped or failed step left open, and anything from a run with no step left
+        // to settle it against. A paused run does not reach this line -- its charges travel in
+        // the resume state instead -- so this settles only calls with nowhere left to be billed.
+        settleUnbilledCharges(costTracker, llmSpendLedger);
+        if (workflow.cost_limit) {
+            costTracker.checkLimit(workflow.cost_limit, ctx.stderr);
+        }
         const output = lastStepId ? toOutputItems(results[lastStepId]) : [];
         if (consumedResumeStateKey) {
-            await deleteStateJson({ env: ctx.env, key: consumedResumeStateKey });
+            if (resumeStateConsumed) {
+                ctx.signal?.throwIfAborted();
+                await deleteStateJson({
+                    env: ctx.env,
+                    key: consumedResumeStateKey,
+                    signal: ctx.signal,
+                });
+                ctx.signal?.throwIfAborted();
+            }
+            else {
+                const deleted = await deleteResumeStateWithRollback({
+                    env: ctx.env,
+                    key: consumedResumeStateKey,
+                    expectedState: resumeState,
+                    signal: ctx.signal,
+                });
+                if (!deleted) {
+                    throw new Error("Workflow resume state not found");
+                }
+            }
+            await cleanupSupersededWorkflowResumeStates(ctx.env, resumeState?.supersededResumeStateKeys);
+        }
+        else {
+            ctx.signal?.throwIfAborted();
         }
         const runResult = { status: "ok", output };
+        // `output` is what the last step could serialize; a run composing this one also needs the
+        // items themselves, because that is where replay provenance lives.
+        const lastItems = lastStepId ? pipelineSourceItems(results[lastStepId]) : undefined;
+        if (lastItems)
+            attachPipelineSourceItems(runResult, lastItems);
         if (costTracker.hasUsage()) {
             runResult._meta = { cost: costTracker.getSummary() };
         }
         return runResult;
     }
+    catch (err) {
+        if (resumedExecutionStarted && consumedResumeStateKey) {
+            try {
+                if (ctx.signal?.aborted) {
+                    await deleteStateJsonWithBoundedResumeCleanup({
+                        env: ctx.env,
+                        key: consumedResumeStateKey,
+                    });
+                }
+                else {
+                    await deleteStateJson({ env: ctx.env, key: consumedResumeStateKey });
+                }
+            }
+            catch {
+                // Leave the tombstone in place if cleanup fails: it is the durable
+                // record that prevents the consumed token from replaying an effect.
+            }
+            // A consumed marker remains non-replayable even when its bounded cleanup
+            // times out, so its short approval ID can be retired independently.
+            await cleanupApprovalIndexByStateKey({
+                env: ctx.env,
+                stateKey: consumedResumeStateKey,
+            }).catch(() => { });
+        }
+        throw err;
+    }
     finally {
+        for (const { signal, onAbort } of executionSignalListeners) {
+            signal.removeEventListener("abort", onAbort);
+        }
         ctx._activeWorkflows?.delete(canonicalFilePath);
     }
 }
@@ -1129,7 +1510,7 @@ function dryRunWorkflow({ steps, resolvedArgs, results, startIndex, ctx, }) {
             if (step.pause_ms)
                 lines.push(`     pause_ms: ${step.pause_ms}`);
             lines.push(`     sub-steps: ${step.steps.length}`);
-            const loopScopedResults = { ...results };
+            const loopScopedResults = createRecord(results);
             loopScopedResults[dryItemVar] = { id: dryItemVar, json: { _placeholder: true } };
             loopScopedResults[dryIndexVar] = { id: dryIndexVar, json: 0 };
             for (let subIdx = 0; subIdx < step.steps.length; subIdx++) {
@@ -1172,6 +1553,16 @@ function dryRunWorkflow({ steps, resolvedArgs, results, startIndex, ctx, }) {
                 }
                 loopScopedResults[sub.id] = { id: sub.id };
             }
+            if (step.timeout_ms)
+                lines.push(`     timeout: ${step.timeout_ms}ms`);
+            if (step.on_error && step.on_error !== "stop")
+                lines.push(`     on_error: ${step.on_error}`);
+            if (step.retry && typeof step.retry === "object") {
+                const rc = resolveRetryConfig(step.retry);
+                if (rc.max > 1) {
+                    lines.push(`     retry: up to ${rc.max} attempts, ${rc.backoff} backoff (base: ${rc.delay_ms}ms${rc.jitter ? ", jitter" : ""})`);
+                }
+            }
             results[step.id] = { id: step.id };
             continue;
         }
@@ -1319,12 +1710,87 @@ export function decodeWorkflowResumePayload(payload) {
         throw new Error("Invalid workflow token");
     return data;
 }
-async function saveWorkflowResumeState(env, state) {
+async function saveWorkflowResumeState(env, state, signal) {
     const stateKey = `workflow_resume_${randomUUID()}`;
-    await writeStateJson({ env, key: stateKey, value: state });
-    return stateKey;
+    try {
+        signal?.throwIfAborted();
+        await writeStateJson({ env, key: stateKey, value: state, signal });
+        signal?.throwIfAborted();
+        return stateKey;
+    }
+    catch (err) {
+        if (signal?.aborted)
+            await discardWorkflowResumeState(env, stateKey);
+        throw err;
+    }
+}
+async function replaceWorkflowResumeState({ env, previousStateKey, expectedPreviousState, previousStateConsumed, replacementStateKey, signal, }) {
+    if (!previousStateKey || previousStateKey === replacementStateKey) {
+        signal?.throwIfAborted();
+        return true;
+    }
+    if (previousStateConsumed) {
+        signal?.throwIfAborted();
+        // The successor is already durable. Retire the predecessor's short ID here,
+        // rather than when the preceding unsafe execution is only being claimed.
+        await cleanupApprovalIndexByStateKey({ env, stateKey: previousStateKey }).catch(() => { });
+        return true;
+    }
+    if (!expectedPreviousState)
+        return false;
+    let claimId;
+    try {
+        const consumption = await consumeResumeState({
+            env,
+            key: previousStateKey,
+            expectedState: expectedPreviousState,
+            signal,
+        });
+        if (!consumption.consumed)
+            return false;
+        claimId = consumption.claimId;
+        // Keep the consumed marker until terminal cleanup. It is the atomic
+        // predecessor-to-successor handoff and prevents another resume from
+        // creating a competing successor after this call returns.
+        signal?.throwIfAborted();
+        await cleanupApprovalIndexByStateKey({ env, stateKey: previousStateKey }).catch(() => { });
+        return true;
+    }
+    catch (err) {
+        if (claimId && signal?.aborted) {
+            await restoreConsumedResumeState({
+                env,
+                key: previousStateKey,
+                expectedState: expectedPreviousState,
+                claimId,
+            }).catch(() => { });
+        }
+        throw err;
+    }
+}
+function collectSupersededWorkflowResumeStateKeys(previousStateKey, previousState) {
+    return [
+        ...(previousState?.supersededResumeStateKeys ?? []),
+        ...(previousStateKey ? [previousStateKey] : []),
+    ].filter((stateKey, index, all) => stateKey && all.indexOf(stateKey) === index);
+}
+async function cleanupSupersededWorkflowResumeStates(env, stateKeys) {
+    for (const stateKey of stateKeys ?? []) {
+        try {
+            if (isConsumedResumeState(await readStateJsonWithLock({ env, key: stateKey }))) {
+                await deleteStateJson({ env, key: stateKey });
+            }
+        }
+        catch {
+            // The completed successor makes retained ancestor markers safe.
+        }
+    }
+}
+async function discardWorkflowResumeState(env, stateKey) {
+    await cleanupApprovalIndexByStateKey({ env, stateKey }).catch(() => { });
+    await deleteStateJson({ env, key: stateKey }).catch(() => { });
 }
-function alternateWorkflowResumeStateKey(stateKey) {
+export function alternateWorkflowResumeStateKey(stateKey) {
     if (stateKey.includes("workflow-resume_")) {
         return stateKey.replace("workflow-resume_", "workflow_resume_");
     }
@@ -1333,8 +1799,8 @@ function alternateWorkflowResumeStateKey(stateKey) {
     }
     return null;
 }
-async function resolveWorkflowResumeStateKey(env, stateKey) {
-    const stored = await readStateJson({ env, key: stateKey });
+async function resolveWorkflowResumeStateKey(env, stateKey, signal) {
+    const stored = await readStateJsonWithLock({ env, key: stateKey, signal });
     if (stored && typeof stored === "object") {
         return stateKey;
     }
@@ -1342,18 +1808,24 @@ async function resolveWorkflowResumeStateKey(env, stateKey) {
     if (!altKey) {
         return stateKey;
     }
-    const altStored = await readStateJson({ env, key: altKey });
+    const altStored = await readStateJsonWithLock({ env, key: altKey, signal });
     if (altStored && typeof altStored === "object") {
         return altKey;
     }
     return stateKey;
 }
-async function loadWorkflowResumeState(env, stateKey) {
-    let stored = await readStateJson({ env, key: stateKey });
+async function loadWorkflowResumeState(env, stateKey, signal) {
+    let stored = await readStateJsonWithLock({ env, key: stateKey, signal });
+    if (isConsumedResumeState(stored)) {
+        throw new Error("Workflow resume state not found");
+    }
     if ((!stored || typeof stored !== "object") && typeof stateKey === "string") {
         const altKey = alternateWorkflowResumeStateKey(stateKey);
         if (altKey) {
-            stored = await readStateJson({ env, key: altKey });
+            stored = await readStateJsonWithLock({ env, key: altKey, signal });
+            if (isConsumedResumeState(stored)) {
+                throw new Error("Workflow resume state not found");
+            }
         }
     }
     if (!stored || typeof stored !== "object") {
@@ -1368,6 +1840,11 @@ async function loadWorkflowResumeState(env, stateKey) {
         throw new Error("Invalid workflow resume state");
     if (!data.args || typeof data.args !== "object")
         throw new Error("Invalid workflow resume state");
+    if (data.supersededResumeStateKeys !== undefined &&
+        (!Array.isArray(data.supersededResumeStateKeys) ||
+            data.supersededResumeStateKeys.some((stateKey) => typeof stateKey !== "string"))) {
+        throw new Error("Invalid workflow resume state");
+    }
     if (data.inputKind !== undefined &&
         !["workflow_step", "pipeline_command"].includes(data.inputKind)) {
         throw new Error("Invalid workflow resume state");
@@ -1418,7 +1895,7 @@ function validateWorkflowPipelineShape(pipeline) {
     }
 }
 function mergeEnv(base, workflowEnv, stepEnv, args, results) {
-    const env = { ...base };
+    const env = createRecord(base);
     // Expose resolved args as env vars so shell commands can safely reference them
     // without embedding raw values into the command string.
     // Example: $LOBSTER_ARG_TEXT
@@ -1449,7 +1926,14 @@ function normalizeArgEnvKey(key) {
         return null;
     // Keep it predictable for shells: uppercase and [A-Z0-9_]
     const up = trimmed.toUpperCase();
-    const normalized = up.replace(/[^A-Z0-9]+/g, "_").replace(/^_+|_+$/g, "");
+    let normalized = up.replace(/[^A-Z0-9]+/g, "_");
+    let start = 0;
+    while (normalized[start] === "_")
+        start++;
+    let end = normalized.length;
+    while (end > start && normalized[end - 1] === "_")
+        end--;
+    normalized = normalized.slice(start, end);
     return normalized || null;
 }
 function resolveCwd(cwd, args) {
@@ -1478,8 +1962,8 @@ function resolveTemplate(input, args, results) {
 }
 function resolveWorkflowStepArgs(workflowArgs, parentArgs, results) {
     if (!workflowArgs)
-        return {};
-    const resolved = {};
+        return createRecord();
+    const resolved = createRecord();
     for (const [key, value] of Object.entries(workflowArgs)) {
         if (typeof value === "string") {
             resolved[key] = resolveTemplate(value, parentArgs, results);
@@ -1492,7 +1976,7 @@ function resolveWorkflowStepArgs(workflowArgs, parentArgs, results) {
 }
 function resolveArgsTemplate(input, args) {
     return input.replace(/\$\{([A-Za-z0-9_-]+)\}/g, (match, key) => {
-        if (key in args)
+        if (Object.hasOwn(args, key))
             return String(args[key]);
         return match;
     });
@@ -1671,21 +2155,222 @@ function parseBoolLike(value) {
         return false;
     return undefined;
 }
-function trackStepCost(costTracker, stepId, result) {
-    const json = result.json;
+// The objects a step's pipeline produced, held apart from the serializable step result.
+// A rendered pipeline such as `llm.invoke | json` prints its items and returns an empty
+// stream, so the step's `json` is parsed back out of stdout and only carries what JSON can
+// express. A symbol key survives neither `JSON.stringify` nor `JSON.parse`, so this holds
+// exactly the items this process made, and no workflow input can put one here.
+const PIPELINE_SOURCE_ITEMS = Symbol("lobster.workflow.pipelineSourceItems");
+function attachPipelineSourceItems(target, items) {
+    Object.defineProperty(target, PIPELINE_SOURCE_ITEMS, {
+        value: items,
+        enumerable: false,
+        configurable: true,
+    });
+    return target;
+}
+function pipelineSourceItems(target) {
+    if (!target)
+        return undefined;
+    const items = target[PIPELINE_SOURCE_ITEMS];
+    return Array.isArray(items) ? items : undefined;
+}
+// The ledger of the run that invoked this one as a `workflow:` step. A composed run bills its
+// own steps and the composing run bills the output handed back to it, so a live call has to be
+// open in both ledgers for each of them to count it once. Read defensively: this arrives
+// through the same context object a caller supplies.
+function composingSpendLedger(ctx) {
+    const ledger = ctx._llmSpendLedger;
+    if (!ledger || typeof ledger !== "object")
+        return null;
+    return typeof ledger.record === "function" && typeof ledger.claim === "function" ? ledger : null;
+}
+/**
+ * One `wait: "any"` branch's view of the run ledger. The race returns as soon as a branch
+ * answers and leaves the others running, so a losing branch can pay a provider long after the
+ * step settled. Writing that charge straight to the run ledger makes the workflow's own total
+ * depend on how long the run happened to live afterwards: land before the run ends and the
+ * discarded branch's spend is billed, land after and it is billed nowhere.
+ *
+ * So a branch opens its charges here instead, and only the winner releases them into the run.
+ * The output of a losing branch is thrown away; its accounting goes with it. Reads pass
+ * through, because they settle charges the run opened before the race and are made by the
+ * workflow after a branch has returned, never from inside one.
+ */
+function branchSpendBuffer(ledger) {
+    const held = [];
+    let released = false;
+    return {
+        record: (cacheKey, charge) => {
+            if (released)
+                ledger.record(cacheKey, charge);
+            else
+                held.push({ cacheKey, charge });
+        },
+        claim: (cacheKey, cost) => ledger.claim(cacheKey, cost),
+        billCopy: (cacheKey, model, usage) => ledger.billCopy(cacheKey, model, usage),
+        outstanding: () => ledger.outstanding(),
+        restore: (charges) => ledger.restore(charges),
+        settled: () => ledger.settled(),
+        restoreSettled: (charges) => ledger.restoreSettled(charges),
+        release: () => {
+            if (released)
+                return;
+            released = true;
+            for (const { cacheKey, charge } of held)
+                ledger.record(cacheKey, charge);
+            held.length = 0;
+        },
+    };
+}
+/**
+ * The run ledger seen by one step, so a charge opened inside it knows where it came from. A
+ * composed run's charges reach the composing run through this too, which is why the step id is
+ * overwritten rather than kept: in the parent's totals the spend belongs to the `workflow:`
+ * step, not to whatever the child called its own.
+ */
+function stepScopedLedger(ledger, stepId) {
+    return {
+        record: (cacheKey, charge) => ledger.record(cacheKey, { ...charge, stepId }),
+        claim: (cacheKey, cost) => ledger.claim(cacheKey, cost),
+        billCopy: (cacheKey, model, usage) => ledger.billCopy(cacheKey, model, usage),
+        outstanding: () => ledger.outstanding(),
+        restore: (charges) => ledger.restore(charges),
+        settled: () => ledger.settled(),
+        restoreSettled: (charges) => ledger.restoreSettled(charges),
+    };
+}
+/**
+ * Bills the calls that ended the run unclaimed. An item is the usual carrier of usage, but a
+ * renderer can consume the pipeline before a later stage emits items of its own, an `ask` gate
+ * can swallow the only item, and a composed run can end on a step that carries none. The
+ * charge outlives all three, and dropping it would leave real spend out of `_meta.cost` and
+ * out of `cost_limit`.
+ */
+function settleUnbilledCharges(costTracker, llmSpendLedger) {
+    for (const charge of llmSpendLedger.outstanding()) {
+        if (!charge.usage)
+            continue;
+        if (!llmSpendLedger.claim(charge.cacheKey))
+            continue;
+        costTracker.recordUsage(charge.stepId ?? "llm", charge.model ?? null, charge.usage);
+    }
+}
+function trackStepCost(costTracker, stepId, result, llmSpendLedger, deferredCharges) {
+    // Bill the items the pipeline produced rather than the JSON re-read from its stdout: a
+    // renderer serializes them, and replay provenance does not survive that round trip. Steps
+    // without a pipeline (shell commands) have no such items and are read from `json` as before.
+    const json = pipelineSourceItems(result) ?? result.json;
     if (!json || typeof json !== "object")
         return;
+    // Replays wait for the live results accounted with them. Branches of one parallel step are
+    // billed in declaration order, so a cached replay declared before an identical `--refresh`
+    // branch would otherwise claim the charge that branch had just opened — and the live branch,
+    // billed whatever its own claim returns, would be billed for it too. One provider call, two
+    // charges, and a `cost_limit` tripped on spend that never happened.
+    const deferred = deferredCharges ?? [];
     const items = Array.isArray(json) ? json : [json];
     for (const item of items) {
         if (!item || typeof item !== "object")
             continue;
-        const usage = item.usage;
+        const record = item;
+        const usage = record.usage;
         if (!usage || typeof usage !== "object")
             continue;
-        const modelValue = item.model;
+        const modelValue = record.model;
         const model = typeof modelValue === "string" ? modelValue : null;
+        // Provenance the LLM commands attach in-process, never a field in the JSON: a step that
+        // prints a replay-shaped object stays billed, so `cost_limit` cannot be evaded. A
+        // projection can drop the item's own mark while carrying its usage record across, so the
+        // record is read too — both marks are symbol keys built in this process.
+        const provenance = llmProvenanceOf(record) ?? llmProvenanceOf(usage);
+        if (provenance) {
+            if (provenance.replayed) {
+                deferred.push({
+                    kind: "replay",
+                    stepId,
+                    cacheKey: provenance.cacheKey,
+                    model,
+                    usage: usage,
+                });
+                continue;
+            }
+            // Settles the charge this live call opened -- the one that cost what this item cost,
+            // not merely the first under the key. A step retried after an attempt that failed
+            // after paying leaves more than one charge open, and they did not cost the same.
+            const settled = llmSpendLedger.claim(provenance.cacheKey, {
+                model,
+                usage: usage,
+            });
+            // Bill what the provider was really asked for, not what the item says by the time it
+            // gets here. A pipeline can drop the model on the way -- `| pick usage` hands on the
+            // marked usage record without it -- or rewrite it, and the step is then priced at the
+            // wrong rate or at nothing at all while `cost_limit` reads the difference as room left.
+            // The charge the call opened is the same witness a replay is already billed from. A
+            // charge restored from resume state written before it carried a cost falls back.
+            if (settled?.usage) {
+                costTracker.recordUsage(stepId, settled.model ?? null, settled.usage);
+                continue;
+            }
+        }
+        else {
+            // No mark, but numbers: a stage that JSON round-trips the stream hands on a copy that
+            // kept `model` and `usage` and lost everything this process attached -- sometimes the
+            // cache key with it. Held back like a replay, and for the same reason: a copy that
+            // lost its key is read by cost alone, and until the marked results have settled their
+            // own charges an identical parallel call's charge is still open to be mistaken for
+            // the one this copy came from.
+            deferred.push({
+                kind: "copy",
+                stepId,
+                cacheKey: typeof record.cacheKey === "string" && record.cacheKey ? record.cacheKey : null,
+                model,
+                usage: usage,
+            });
+            continue;
+        }
         costTracker.recordUsage(stepId, model, usage);
     }
+    if (!deferredCharges)
+        settleDeferredCosts(costTracker, deferred, llmSpendLedger);
+}
+/**
+ * Bills what a step held back, now that every marked live result accounted alongside them has
+ * settled its own charge. Replays go first: one carries the mark of the call it stands in for,
+ * which is better evidence than the cost a copy carries on its own.
+ *
+ * A replay carries the usage of the call that produced it, so counting it again charges a run
+ * for tokens no provider was asked to spend. It is billed only when the call it replays is
+ * still unpaid for: a step is costed once it succeeds, so a step that failed after its LLM
+ * call and was retried leaves its real spend riding on the replay the retry produced.
+ *
+ * A copy is billed either way — it is a record of a call, and nothing in a JSON payload can be
+ * trusted to say otherwise — but the ledger still decides whether it settles a charge, so the
+ * step settlement does not add a second record of the same call.
+ */
+function settleDeferredCosts(costTracker, deferred, llmSpendLedger) {
+    for (const replay of deferred) {
+        if (replay.kind !== "replay" || replay.cacheKey === null)
+            continue;
+        const settled = llmSpendLedger.claim(replay.cacheKey);
+        if (!settled)
+            continue;
+        // Bill what the call this settles actually cost, not what the replay carries. Identical
+        // calls that raced on a cold cache are separate charges under one key, and every replay
+        // of them repeats whichever single answer was stored -- so costing each replay by its
+        // own copy reports all of those calls at the price of the one that won the cache write.
+        // A charge restored from resume state written before it carried a cost falls back.
+        const cost = typeof settled === "object" && settled.usage ? settled : null;
+        costTracker.recordUsage(replay.stepId, cost ? (cost.model ?? null) : replay.model, cost?.usage ?? replay.usage);
+    }
+    for (const copy of deferred) {
+        if (copy.kind !== "copy")
+            continue;
+        if (llmSpendLedger.billCopy(copy.cacheKey, copy.model, copy.usage)) {
+            costTracker.recordUsage(copy.stepId, copy.model, copy.usage);
+        }
+    }
+    deferred.length = 0;
 }
 function parseJson(stdout) {
     const trimmed = stdout.trim();
@@ -1713,7 +2398,7 @@ function toOutputItems(result) {
     return [];
 }
 function cloneResults(results) {
-    const out = {};
+    const out = createRecord();
     for (const [key, value] of Object.entries(results)) {
         out[key] = { ...value };
     }
@@ -2141,39 +2826,22 @@ function parseQuotedConditionString(expression, startIndex, quoteChar) {
     }
     throw new Error(`Unsupported condition: ${expression}`);
 }
-async function runShellCommand({ command, stdin, env, cwd, signal, killSignal, }) {
-    const { spawn } = await import("node:child_process");
-    return await new Promise((resolve, reject) => {
-        const shell = resolveInlineShellCommand({ command, env });
-        const child = spawn(shell.command, shell.argv, {
-            env,
-            cwd,
-            signal,
-            killSignal,
-            stdio: ["pipe", "pipe", "pipe"],
-        });
-        let stdout = "";
-        let stderr = "";
-        child.stdout.setEncoding("utf8");
-        child.stderr.setEncoding("utf8");
-        child.stdout.on("data", (d) => {
-            stdout += d;
-        });
-        child.stderr.on("data", (d) => {
-            stderr += d;
-        });
-        if (typeof stdin === "string") {
-            child.stdin.setDefaultEncoding("utf8");
-            child.stdin.write(stdin);
-        }
-        child.stdin.end();
-        child.on("error", reject);
-        child.on("close", (code) => {
-            if (code === 0)
-                return resolve({ stdout, stderr });
-            reject(new Error(`workflow command failed (${code}): ${stderr.trim() || stdout.trim() || command}`));
-        });
+async function runShellCommand({ command, stdin, env, cwd, signal, forceTerminationSignal, killSignal, }) {
+    signal?.throwIfAborted();
+    signal?.throwIfAborted();
+    const { stdout, stderr, code } = await runAbortableProcess({
+        shellCommand: command,
+        env,
+        cwd,
+        stdin,
+        signal,
+        forceTerminationSignal,
+        killSignal,
+        notFoundMessage: "workflow shell not found; check LOBSTER_SHELL or ComSpec",
     });
+    if (code === 0)
+        return { stdout, stderr };
+    throw new Error(`workflow command failed (${code}): ${stderr.trim() || stdout.trim() || command}`);
 }
 function getStepExecution(step) {
     if (step.parallel && typeof step.parallel === "object" && !Array.isArray(step.parallel)) {
@@ -2191,7 +2859,7 @@ function getStepExecution(step) {
     }
     return { kind: "none" };
 }
-async function runPipelineStep({ stepId, pipelineText, inputValue, ctx, env, cwd, resume, requestInputEnabled = true, }) {
+async function runPipelineStep({ stepId, pipelineText, inputValue, ctx, llmSpendLedger, env, cwd, resume, requestInputEnabled = true, onExecutionStart, }) {
     let pipeline;
     try {
         const currentPipeline = parsePipeline(pipelineText);
@@ -2228,7 +2896,11 @@ async function runPipelineStep({ stepId, pipelineText, inputValue, ctx, env, cwd
         mode: ctx.mode,
         cwd,
         signal: ctx.signal,
+        forceTerminationSignal: ctx.forceTerminationSignal,
+        haltAfterStageOnAbort: true,
         llmAdapters: ctx.llmAdapters,
+        // Scoped so a charge nothing bills can still name the step that opened it.
+        llmSpendLedger: llmSpendLedger ? stepScopedLedger(llmSpendLedger, stepId) : undefined,
         input: resume ? resume.pipelineInput.items : inputValueToPipelineItems(inputValue),
         requestInputEnabled,
         requestInputResume: resume
@@ -2238,8 +2910,11 @@ async function runPipelineStep({ stepId, pipelineText, inputValue, ctx, env, cwd
                 onConsumed: resume.onConsumed,
             }
             : undefined,
+        onExecutionStart,
+        onNonRetryableSideEffect: ctx._onNonRetryableSideEffect,
     });
     stdout.end();
+    ctx.signal?.throwIfAborted();
     if (result.halted) {
         const haltedName = result.haltedAt?.stage?.name ?? "unknown";
         if (result.items.length === 1 && result.items[0]?.type === "approval_request") {
@@ -2273,11 +2948,16 @@ async function runPipelineStep({ stepId, pipelineText, inputValue, ctx, env, cwd
             ? result.items[0]
             : result.items
         : parseJson(renderedStdout);
-    return {
+    const stepResult = {
         id: stepId,
         stdout: normalizedStdout,
         json,
     };
+    // Only when a renderer consumed the pipeline: otherwise `json` already is those items.
+    if (!result.items.length && result.renderedItems.length) {
+        attachPipelineSourceItems(stepResult, result.renderedItems);
+    }
+    return stepResult;
 }
 function createSyntheticStepResult(stepId, value) {
     if (value === null || value === undefined) {