@agwab/pi-workflow 0.2.1 → 0.4.0

package/dist/engine.js

@@ -4,11 +4,12 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 import { Worker } from "node:worker_threads";
 import { compileWorkflow } from "./compiler.js";
 import { loadWorkflowSpec } from "./schema.js";
-import { createRunRecord, createTaskRunRecord, compiledWorkflowPath, fromProjectPath, indexSupervisorErrorPath, isTerminalWorkflowStatus, isTerminalTaskStatus, listRunRecords, readIndex, readJson, readRunRecord, resetTaskForResume, setTaskTerminal, supervisorPath, toProjectPath, updateIndex, withRunLease, workflowRunPath, writeJsonAtomic, writeRunRecord, writeCompiledRunArtifact, writeStaticRunArtifacts, } from "./store.js";
+import { createRunRecord, createTaskRunRecord, compiledWorkflowPath, fromProjectPath, indexSupervisorErrorPath, isBlockedTaskResumableForResume, isTerminalWorkflowStatus, isTerminalTaskStatus, listRunRecords, readIndex, readJson, readRunRecord, resetTaskForResume, setTaskTerminal, supervisorPath, toProjectPath, updateIndex, withRunLease, workflowRunPath, writeJsonAtomic, writeRunRecord, writeCompiledRunArtifact, writeStaticRunArtifacts, } from "./store.js";
 import { resolveWorkflowBackend } from "./backend.js";
 import { ensureManagedWorktree } from "./worktree.js";
 import { resolveWorkflowHelperRef } from "./workflow-helpers.js";
 import { buildAvailableToolView } from "./tool-metadata.js";
+import { summarizeWorkflowTelemetry } from "./workflow-artifacts.js";
 import { workflowBundleFingerprint, workflowBundleSpecPath, } from "./workflow-source-context-runtime.js";
 import { readSimpleJsonPath, } from "./workflow-runtime.js";
 import { dynamicRunDir, hashDynamicRequest, readDynamicEvents, } from "./dynamic-events.js";
@@ -18,10 +19,11 @@ import { assertDynamicRuntimeBudgetAvailable, dynamicRuntimeBudgetExceededMessag
 import { assertDynamicGeneratedMetadataMatches, assertDynamicGenerationBudgetAvailable, buildDynamicGeneratedCompiledTask, dynamicGeneratedInsertIndex, isDynamicCompiledTaskPayload, normalizeDynamicAgentRequest, readDynamicGeneratedTaskResult, } from "./dynamic-generated-task-runtime.js";
 import { optionalEventString, runDynamicHelperCall, runDynamicNestedWorkflowCall, } from "./dynamic-controller-calls.js";
 import { normalizeDynamicFanoutPlanRequest, runDynamicDecisionLoopStatusPersistCall, runDynamicDecisionPersistCall, runDynamicFanoutPlanPersistCall, runDynamicResultReadCall, runDynamicStateIndexPersistCall, } from "./dynamic-control-ops.js";
-import { assertRunTaskPositionalAlignment, buildForeachGeneratedTasks, dependenciesReady, markDagDependentsSkipped, nextTaskRecordIndex, reconcileDynamicGeneratedRunRecords, recoverStaleRunningDynamicControllers, replaceDependencyList, sourceStageIdsForFrom, stageSourcePolicy, updateDownstreamDependencies, } from "./engine-run-graph.js";
+import { assertRunTaskPositionalAlignment, buildForeachGeneratedTasks, dependenciesReady, foreachStreamingEnabled, foreachStreamingMinChunk, markDagDependentsSkipped, nextTaskRecordIndex, reconcileDynamicGeneratedRunRecords, reconcileForeachGeneratedRunRecords, recoverStaleRunningDynamicControllers, replaceDependencyList, sourceStageIdsForFrom, stageSourcePolicy, updateDownstreamDependencies, } from "./engine-run-graph.js";
 import { reconcileLoopTaskMaterialization, scheduleLoop, } from "./loop-runtime.js";
 import { executeSupportTask, normalizeDynamicControllerOutput, prepareArtifactGraphRetryTask, prepareDagTask, readArtifactGraphControl, readArtifactGraphSupportSources, readSupportSources, writeArtifactGraphDynamicResult, } from "./artifact-graph-runtime.js";
 import { DIRECT_DYNAMIC_RUNTIME_VERSION, ensureDirectDynamicRuntimeBundle, } from "./dynamic-runtime-bundle.js";
+import { hasFatalPartialOutputIssue, readWorkflowPartialOutputLedger, writeWorkflowPartialOutputLedgerFromFile, } from "./workflow-partial-output.js";
 import { WORKFLOW_RUN_TYPE, } from "./types.js";
 export { buildRunSourceContext } from "./workflow-source-context-runtime.js";
 export { evaluateLoopUntilCondition } from "./loop-runtime.js";
@@ -37,6 +39,8 @@ const DYNAMIC_CONTROLLER_ENGINE_CAPABILITIES = Object.freeze({
 const DYNAMIC_CONTROLLER_ENGINE_INTEGRITY_ERROR_MESSAGE = "incompatible or stale pi-workflow engine: dynamic controller context is missing runDecisionLoop (rebuild dist / reload workflow engine)";
 const supervisorTimers = new Map();
 const supervisorRunMtimes = new Map();
+const supervisorErrorCounts = new Map();
+const MAX_SUPERVISOR_CONSECUTIVE_ERRORS = 3;
 export async function runWorkflowSpec(specPath, cwd, options = {}) {
     const loaded = await loadWorkflowSpec(specPath, cwd);
     return runLoadedWorkflowSpec(cwd, loaded.specPath, loaded.spec, options);
@@ -83,7 +87,7 @@ async function runLoadedWorkflowSpec(cwd, specPath, spec, options, provenance) {
     };
     const scheduled = (await scheduleRun(cwd, run.runId, compiled, scheduleOptions)) ??
         (await readRunRecord(cwd, run.runId));
-    if (scheduled.status === "running")
+    if (shouldWatchRun(scheduled))
         watchRun(cwd, scheduled.runId, scheduleOptions);
     return scheduled;
 }
@@ -95,11 +99,19 @@ export async function refreshRun(cwd, runIdOrPrefix) {
     });
     return refreshed ?? current;
 }
+function hasActiveSchedulerWork(run) {
+    return (run.status === "running" ||
+        run.taskSummary.running > 0 ||
+        run.taskSummary.pending > 0);
+}
+function shouldWatchRun(run) {
+    return hasActiveSchedulerWork(run);
+}
 export async function waitForRun(cwd, runIdOrPrefix, timeoutMs, options = {}) {
     const timeout = clampTimeout(timeoutMs);
     const deadline = Date.now() + timeout;
     let run = await refreshRun(cwd, runIdOrPrefix);
-    while (run.status === "running") {
+    while (hasActiveSchedulerWork(run)) {
         const beforeScheduleRemaining = deadline - Date.now();
         if (beforeScheduleRemaining <= 0)
             throw new Error(`Flow run still running after ${timeout}ms: ${run.runId}`);
@@ -107,7 +119,7 @@ export async function waitForRun(cwd, runIdOrPrefix, timeoutMs, options = {}) {
         run = await refreshRun(cwd, run.runId);
         const remaining = deadline - Date.now();
         if (remaining <= 0) {
-            if (run.status !== "running")
+            if (!hasActiveSchedulerWork(run))
                 return run;
             throw new Error(`Flow run still running after ${timeout}ms: ${run.runId}`);
         }
@@ -116,6 +128,45 @@ export async function waitForRun(cwd, runIdOrPrefix, timeoutMs, options = {}) {
     }
     return run;
 }
+function assertBlockedRunResumable(run) {
+    if (run.status !== "blocked")
+        return;
+    const blockers = run.tasks.filter((task) => task.status === "blocked" && !isBlockedTaskResumableForResume(task));
+    if (blockers.length === 0)
+        return;
+    const details = blockers
+        .slice(0, 3)
+        .map((task) => `${task.specId} statusDetail=${task.statusDetail}`)
+        .join(", ");
+    throw new Error(`Cannot resume blocked run ${run.runId}: non-resumable blocked task(s): ${details}. Resolve the attention/approval blocker before resuming.`);
+}
+export async function stopRun(cwd, runIdOrPrefix) {
+    const current = await readRunRecord(cwd, runIdOrPrefix);
+    const stopped = await withRunLease(cwd, current.runId, async () => {
+        const run = await readRunRecord(cwd, current.runId);
+        if (isTerminalWorkflowStatus(run.status)) {
+            throw new Error(`stop requires a non-terminal run; ${run.runId} is ${run.status}`);
+        }
+        await resolveWorkflowBackend(run)
+            .cleanupRun(cwd, run)
+            .catch(() => undefined);
+        const interruptedTaskIds = [];
+        for (const task of run.tasks) {
+            if (setTaskTerminal(task, "interrupted", "workflow_stopped", {
+                exitCode: 130,
+                lastMessage: "Workflow stopped by user request",
+            })) {
+                interruptedTaskIds.push(task.taskId);
+            }
+        }
+        await writeRunRecord(cwd, run);
+        unwatchRun(cwd, run.runId);
+        return { run, interruptedTaskIds };
+    });
+    if (!stopped)
+        throw new Error(`Could not acquire workflow run lease for ${current.runId}`);
+    return stopped;
+}
 export async function resumeRun(cwd, runIdOrPrefix, options = {}) {
     const current = await readRunRecord(cwd, runIdOrPrefix);
     if (current.status !== "failed" &&
@@ -123,6 +174,7 @@ export async function resumeRun(cwd, runIdOrPrefix, options = {}) {
         current.status !== "blocked") {
         throw new Error(`resume requires a failed, interrupted, or resumable blocked run; ${current.runId} is ${current.status}`);
     }
+    assertBlockedRunResumable(current);
     const compiledFlow = await readCompiledWorkflow(cwd, current.runId);
     const hasLoopTasks = compiledFlow?.tasks.some((task) => task.kind === "loop" ||
         task.loopPlaceholder !== undefined ||
@@ -133,6 +185,10 @@ export async function resumeRun(cwd, runIdOrPrefix, options = {}) {
     const resetTaskIds = [];
     const updated = await withRunLease(cwd, current.runId, async () => {
         const run = await readRunRecord(cwd, current.runId);
+        assertBlockedRunResumable(run);
+        await resolveWorkflowBackend(run)
+            .cleanupRun(cwd, run)
+            .catch(() => undefined);
         for (const task of run.tasks) {
             if (resetTaskForResume(task))
                 resetTaskIds.push(task.taskId);
@@ -147,7 +203,7 @@ export async function resumeRun(cwd, runIdOrPrefix, options = {}) {
         throw new Error(`No failed, interrupted, skipped, or resumable blocked tasks to resume in ${current.runId}`);
     const scheduled = (await scheduleRun(cwd, current.runId, undefined, options)) ??
         (await readRunRecord(cwd, current.runId));
-    if (scheduled.status === "running")
+    if (shouldWatchRun(scheduled))
         watchRun(cwd, scheduled.runId, options);
     return { run: scheduled, resetTaskIds };
 }
@@ -155,7 +211,7 @@ export async function resumeSupervisors(cwd, options = {}) {
     try {
         const runs = await listRunRecords(cwd);
         for (const run of runs) {
-            if (run.status === "running") {
+            if (hasActiveSchedulerWork(run)) {
                 await scheduleRun(cwd, run.runId, undefined, options).catch((error) => recordSupervisorError(cwd, run.runId, error));
                 watchRun(cwd, run.runId, options);
             }
@@ -166,6 +222,15 @@ export async function resumeSupervisors(cwd, options = {}) {
         await recordSupervisorError(cwd, "index", error);
     }
 }
+function unwatchRun(cwd, runId) {
+    const key = `${cwd}\0${runId}`;
+    const existing = supervisorTimers.get(key);
+    if (existing)
+        clearInterval(existing);
+    supervisorTimers.delete(key);
+    supervisorRunMtimes.delete(key);
+    supervisorErrorCounts.delete(key);
+}
 export function watchRun(cwd, runId, options = {}) {
     const key = `${cwd}\0${runId}`;
     if (supervisorTimers.has(key))
@@ -179,7 +244,8 @@ export function watchRun(cwd, runId, options = {}) {
             const currentMtime = afterMtime ?? beforeMtime;
             if (currentMtime !== undefined)
                 supervisorRunMtimes.set(key, currentMtime);
-            if (refreshed.status === "running") {
+            supervisorErrorCounts.delete(key);
+            if (hasActiveSchedulerWork(refreshed)) {
                 const unchanged = previousMtime !== undefined &&
                     currentMtime !== undefined &&
                     currentMtime <= previousMtime;
@@ -187,13 +253,18 @@ export function watchRun(cwd, runId, options = {}) {
                     await scheduleRun(cwd, runId, undefined, options);
                 return;
             }
-            const existing = supervisorTimers.get(key);
-            if (existing)
-                clearInterval(existing);
-            supervisorTimers.delete(key);
-            supervisorRunMtimes.delete(key);
+            unwatchRun(cwd, runId);
         })().catch((error) => {
-            void recordSupervisorError(cwd, runId, error);
+            if (isMissingRunError(error)) {
+                unwatchRun(cwd, runId);
+                return;
+            }
+            const failures = (supervisorErrorCounts.get(key) ?? 0) + 1;
+            supervisorErrorCounts.set(key, failures);
+            void recordSupervisorError(cwd, runId, error).finally(() => {
+                if (failures >= MAX_SUPERVISOR_CONSECUTIVE_ERRORS)
+                    unwatchRun(cwd, runId);
+            });
         });
     }, POLL_INTERVAL_MS);
     timer.unref?.();
@@ -204,16 +275,38 @@ async function readRunMtimeMs(cwd, runId) {
         return (await stat(workflowRunPath(cwd, runId))).mtimeMs;
     }
     catch (error) {
-        if (error.code === "ENOENT")
+        if (isEnoentError(error))
             return undefined;
         throw error;
     }
 }
+function isEnoentError(error) {
+    return error?.code === "ENOENT";
+}
+function isMissingRunError(error) {
+    return (isEnoentError(error) ||
+        (error instanceof Error && /^Flow run not found: /.test(error.message)));
+}
+function assertScheduleLeaseActive(signal) {
+    if (!signal?.aborted)
+        return;
+    const reason = signal.reason;
+    if (reason instanceof Error)
+        throw reason;
+    throw new Error(reason === undefined
+        ? "Lost supervisor lease"
+        : `Lost supervisor lease: ${String(reason)}`);
+}
 export async function scheduleRun(cwd, runId, compiled, options = {}) {
-    return withRunLease(cwd, runId, async () => {
+    return withRunLease(cwd, runId, async (leaseSignal) => {
+        assertScheduleLeaseActive(leaseSignal);
         let run = await readRunRecord(cwd, runId);
         run = await resolveWorkflowBackend(run).refreshRun(cwd, run);
-        if (run.taskSummary.blocked > 0 || isTerminalWorkflowStatus(run.status))
+        if (isTerminalWorkflowStatus(run.status))
+            return run;
+        if (run.taskSummary.blocked > 0 &&
+            run.taskSummary.pending === 0 &&
+            run.taskSummary.running === 0)
             return run;
         const compiledFlow = compiled ?? (await readCompiledWorkflow(cwd, run.runId));
         if (!compiledFlow)
@@ -221,7 +314,8 @@ export async function scheduleRun(cwd, runId, compiled, options = {}) {
         if (compiledFlow.type !== WORKFLOW_RUN_TYPE) {
             throw new Error(`unsupported compiled workflow type: ${compiledFlow.type}`);
         }
-        await scheduleDag(cwd, run, compiledFlow, options);
+        await scheduleDag(cwd, run, compiledFlow, options, leaseSignal);
+        assertScheduleLeaseActive(leaseSignal);
         run = await readRunRecord(cwd, run.runId);
         return run;
     });
@@ -233,13 +327,13 @@ export async function formatStatus(cwd) {
         const refreshed = (await readIndex(cwd).catch(() => cached)) ?? cached;
         if (refreshed.runs.length === 0)
             return "No workflow runs found.";
-        return formatIndex(refreshed);
+        return formatIndex(cwd, refreshed);
     }
     await reconcileActiveRuns(cwd);
     const rebuilt = await updateIndex(cwd).catch(() => readIndex(cwd));
     if (!rebuilt || rebuilt.runs.length === 0)
         return "No workflow runs found.";
-    return formatIndex(rebuilt);
+    return formatIndex(cwd, rebuilt);
 }
 export async function formatRunDetails(cwd, runIdOrPrefix) {
     const run = await refreshRun(cwd, runIdOrPrefix);
@@ -270,10 +364,12 @@ export async function formatLogs(cwd, runIdOrPrefix, taskId = "task-1", lineCoun
     return `${run.runId}/${task.taskId} output=${task.files.output}\n${tail || "(empty log)"}`;
 }
 export function formatRun(run, detail = "summary") {
+    const telemetry = summarizeWorkflowTelemetry(run);
     const lines = [
         `${run.runId} [${run.status}] type=${run.type} backend=${run.backend.type}/${run.backend.mode}`,
         `created=${run.createdAt} updated=${run.updatedAt}`,
         `tasks=${run.taskSummary.completed}/${run.taskSummary.total} completed, running=${run.taskSummary.running}, pending=${run.taskSummary.pending}, blocked=${run.taskSummary.blocked}, failed=${run.taskSummary.failed}, interrupted=${run.taskSummary.interrupted}`,
+        `completion=${telemetry.completion.health}, outputRetries=${telemetry.retryCounts.output}, launchRetries=${telemetry.retryCounts.launch}, resumeEvents=${telemetry.resumeCounts.events}, contextLimitFailures=${telemetry.completion.contextLimitFailures}`,
     ];
     for (const task of run.tasks) {
         lines.push(formatTask(task, detail));
@@ -283,13 +379,13 @@ export function formatRun(run, detail = "summary") {
 async function reconcileActiveRuns(cwd) {
     const runs = await listRunRecords(cwd);
     for (const run of runs) {
-        if (run.status === "running")
+        if (hasActiveSchedulerWork(run))
             await refreshRun(cwd, run.runId).catch((error) => recordSupervisorError(cwd, run.runId, error));
     }
 }
 async function reconcileIndexedActiveRuns(cwd, index) {
     for (const run of index.runs) {
-        if (run.status === "running")
+        if (hasActiveSchedulerWork(run))
             await refreshRun(cwd, run.runId).catch((error) => recordSupervisorError(cwd, run.runId, error));
     }
 }
@@ -306,11 +402,18 @@ async function recordSupervisorError(cwd, runId, error) {
         error: error instanceof Error ? error.message : String(error),
     }).catch(() => undefined);
 }
-async function scheduleDag(cwd, run, compiledFlow, options = {}) {
+async function scheduleDag(cwd, run, compiledFlow, options = {}, leaseSignal) {
+    assertScheduleLeaseActive(leaseSignal);
     if (compiledFlow.type === WORKFLOW_RUN_TYPE) {
         const loopReconciled = await reconcileLoopTaskMaterialization(cwd, run, compiledFlow);
         if (loopReconciled)
             return;
+        const foreachReconciled = reconcileForeachGeneratedRunRecords(cwd, run, compiledFlow);
+        if (foreachReconciled) {
+            await writeJsonAtomic(compiledWorkflowPath(cwd, run.runId), compiledFlow);
+            await writeRunRecord(cwd, run);
+            return;
+        }
         const dynamicReconciled = reconcileDynamicGeneratedRunRecords(cwd, run, compiledFlow);
         const staleDynamicRecovered = recoverStaleRunningDynamicControllers(run, compiledFlow);
         if (dynamicReconciled || staleDynamicRecovered)
@@ -326,6 +429,7 @@ async function scheduleDag(cwd, run, compiledFlow, options = {}) {
     let running = run.tasks.filter((task) => task.status === "running").length;
     const bySpecId = new Map(run.tasks.map((task) => [task.specId, task]));
     for (let index = 0; index < run.tasks.length && running < maxConcurrency; index += 1) {
+        assertScheduleLeaseActive(leaseSignal);
         const task = run.tasks[index];
         const compiledTask = compiledFlow.tasks[index];
         if (!task || !compiledTask || task.status !== "pending")
@@ -345,6 +449,8 @@ async function scheduleDag(cwd, run, compiledFlow, options = {}) {
             const changed = await materializeForeachTask(cwd, run, compiledFlow, index, compiledTask);
             if (changed)
                 return;
+            if (foreachStreamingEnabled(compiledTask))
+                continue;
         }
         if (compiledTask.stageMaxConcurrency !== undefined) {
             const runningInStage = run.tasks.filter((candidate) => candidate.stageId === compiledTask.stageId &&
@@ -353,8 +459,10 @@ async function scheduleDag(cwd, run, compiledFlow, options = {}) {
                 Math.max(1, Math.min(MAX_CONCURRENCY, compiledTask.stageMaxConcurrency)))
                 continue;
         }
+        assertScheduleLeaseActive(leaseSignal);
         const launched = await launchPendingTaskAt(cwd, run, compiledFlow, index, options);
-        if (launched)
+        assertScheduleLeaseActive(leaseSignal);
+        if (launched && run.tasks[index]?.status === "running")
             running += 1;
     }
 }
@@ -433,10 +541,12 @@ async function materializeForeachTask(cwd, run, compiledFlow, index, template) {
         return false;
     const sourceStageIds = sourceStageIdsForFrom(template.foreach.from);
     const sourceTasks = run.tasks.filter((task) => sourceStageIds.includes(task.stageId ?? ""));
+    const streaming = foreachStreamingEnabled(template);
     const extracted = await extractArtifactGraphForeachItems(cwd, {
         from: template.foreach.from,
         sourcePolicy: stageSourcePolicy(compiledFlow, template.stageId),
         maxItems: template.foreach.maxItems,
+        streaming,
     }, sourceTasks);
     if (extracted.error) {
         setTaskTerminal(templateRunTask, "blocked", "foreach_expansion_blocked", {
@@ -455,7 +565,30 @@ async function materializeForeachTask(cwd, run, compiledFlow, index, template) {
         return true;
     }
     const placeholderSpecId = template.id;
+    if (streaming) {
+        return await materializeStreamingForeachTask({
+            cwd,
+            run,
+            compiledFlow,
+            index,
+            templateRunTask,
+            placeholderSpecId,
+            sourceTaskSpecIds: sourceTasks.map((task) => task.specId),
+            itemMetas: extracted.itemMetas ?? [],
+            generatedTasks: generated.tasks,
+            waitingForSources: extracted.waitingForSources ?? false,
+            minChunk: foreachStreamingMinChunk(template),
+        });
+    }
     const generatedSpecIds = generated.tasks.map((task) => task.id);
+    const hasDownstreamDependents = compiledFlow.tasks.some((task, taskIndex) => taskIndex !== index && (task.dependsOn ?? []).includes(placeholderSpecId));
+    if (generatedSpecIds.length === 0 && !hasDownstreamDependents) {
+        setTaskTerminal(templateRunTask, "completed", "foreach_empty", {
+            lastMessage: "foreach produced 0 item(s)",
+        });
+        await writeRunRecord(cwd, run);
+        return true;
+    }
     compiledFlow.tasks.splice(index, 1, ...generated.tasks);
     updateDownstreamDependencies(compiledFlow, placeholderSpecId, generatedSpecIds);
     const nextIndex = nextTaskRecordIndex(run);
@@ -470,16 +603,183 @@ async function materializeForeachTask(cwd, run, compiledFlow, index, template) {
     await writeRunRecord(cwd, run);
     return true;
 }
+async function materializeStreamingForeachTask(input) {
+    const sourceTaskSpecIdSet = new Set(input.sourceTaskSpecIds);
+    const generatedTasksWithItemDeps = input.generatedTasks.map((task, index) => {
+        const itemMeta = input.itemMetas[index];
+        if (!itemMeta)
+            return task;
+        return {
+            ...task,
+            dependsOn: replaceSourceDependenciesWithItemSource(task.dependsOn ?? [], sourceTaskSpecIdSet, itemMeta, {
+                keepPartialSourceDependency: partialGeneratedTaskNeedsCompletedSourceContext(task),
+            }),
+            foreachGenerated: {
+                ...(task.foreachGenerated ?? {
+                    placeholderSpecId: input.placeholderSpecId,
+                }),
+                itemHash: itemMeta.itemHash,
+                itemSourceSpecId: itemMeta.sourceSpecId,
+                itemSourceKind: itemMeta.sourceKind,
+                itemRef: itemMeta.itemRef,
+            },
+        };
+    });
+    const existingGeneratedTasks = input.compiledFlow.tasks.filter((task) => task.foreachGenerated?.placeholderSpecId === input.placeholderSpecId);
+    const existingGeneratedSpecIds = existingGeneratedTasks.map((task) => task.id);
+    const existingGeneratedTaskBySpecId = new Map(existingGeneratedTasks.map((task) => [task.id, task]));
+    for (const task of generatedTasksWithItemDeps) {
+        const existing = existingGeneratedTaskBySpecId.get(task.id);
+        const existingHash = existing?.foreachGenerated?.itemHash;
+        const nextHash = task.foreachGenerated?.itemHash;
+        if (existing && existingHash && nextHash && existingHash !== nextHash) {
+            setTaskTerminal(input.templateRunTask, "blocked", "foreach_expansion_blocked", {
+                lastMessage: `foreach streaming item ${task.id} changed after materialization`,
+            });
+            await writeRunRecord(input.cwd, input.run);
+            return true;
+        }
+    }
+    const existingGeneratedSpecIdSet = new Set(existingGeneratedSpecIds);
+    const finalGeneratedSpecIdSet = new Set(generatedTasksWithItemDeps.map((task) => task.id));
+    if (!input.waitingForSources) {
+        const withdrawn = existingGeneratedTasks.find((task) => task.foreachGenerated?.itemSourceKind === "partial" &&
+            !finalGeneratedSpecIdSet.has(task.id));
+        if (withdrawn) {
+            setTaskTerminal(input.templateRunTask, "blocked", "foreach_expansion_blocked", {
+                lastMessage: `foreach streaming item ${withdrawn.id} was published as partial output but is missing from final control`,
+            });
+            await writeRunRecord(input.cwd, input.run);
+            return true;
+        }
+    }
+    const newGeneratedTasks = generatedTasksWithItemDeps.filter((task) => !existingGeneratedSpecIdSet.has(task.id));
+    const allGeneratedSpecIds = [
+        ...existingGeneratedSpecIds,
+        ...newGeneratedTasks.map((task) => task.id),
+    ];
+    const shouldHoldForMinChunk = input.waitingForSources &&
+        newGeneratedTasks.length > 0 &&
+        newGeneratedTasks.length < input.minChunk;
+    if (shouldHoldForMinChunk)
+        return false;
+    let changed = false;
+    if (newGeneratedTasks.length > 0) {
+        let compiledInsertIndex = input.index + 1;
+        while (input.compiledFlow.tasks[compiledInsertIndex]?.foreachGenerated
+            ?.placeholderSpecId === input.placeholderSpecId) {
+            compiledInsertIndex += 1;
+        }
+        input.compiledFlow.tasks.splice(compiledInsertIndex, 0, ...newGeneratedTasks);
+        let runInsertIndex = input.index + 1;
+        while (input.run.tasks[runInsertIndex]?.foreachGenerated?.placeholderSpecId ===
+            input.placeholderSpecId) {
+            runInsertIndex += 1;
+        }
+        const nextIndex = nextTaskRecordIndex(input.run);
+        const generatedRunTasks = newGeneratedTasks.map((task, offset) => createTaskRunRecord(input.cwd, input.run.runId, task, nextIndex + offset));
+        input.run.tasks.splice(runInsertIndex, 0, ...generatedRunTasks);
+        changed = true;
+    }
+    const dependencyTargets = [input.placeholderSpecId, ...allGeneratedSpecIds];
+    for (const task of input.compiledFlow.tasks) {
+        if (!task.dependsOn)
+            continue;
+        const replaced = replaceDependencyList(task.dependsOn, input.placeholderSpecId, dependencyTargets);
+        if (JSON.stringify(task.dependsOn) !== JSON.stringify(replaced)) {
+            task.dependsOn = replaced;
+            changed = true;
+        }
+    }
+    for (const task of input.run.tasks) {
+        if (!task.dependsOn)
+            continue;
+        const replaced = replaceDependencyList(task.dependsOn, input.placeholderSpecId, dependencyTargets);
+        if (JSON.stringify(task.dependsOn) !== JSON.stringify(replaced)) {
+            task.dependsOn = replaced;
+            changed = true;
+        }
+    }
+    if (!input.waitingForSources) {
+        const statusDetail = allGeneratedSpecIds.length === 0
+            ? "foreach_empty"
+            : "foreach_streaming_complete";
+        const lastMessage = allGeneratedSpecIds.length === 0
+            ? "foreach produced 0 item(s)"
+            : `foreach streaming materialized ${allGeneratedSpecIds.length} item(s)`;
+        setTaskTerminal(input.templateRunTask, "completed", statusDetail, {
+            lastMessage,
+        });
+        changed = true;
+    }
+    else if (newGeneratedTasks.length > 0) {
+        input.templateRunTask.statusDetail = "foreach_streaming_waiting";
+        input.templateRunTask.lastMessage = `foreach streaming materialized ${allGeneratedSpecIds.length} item(s); waiting for more source tasks`;
+        changed = true;
+    }
+    if (!changed)
+        return false;
+    await writeJsonAtomic(compiledWorkflowPath(input.cwd, input.run.runId), input.compiledFlow);
+    await writeRunRecord(input.cwd, input.run);
+    return true;
+}
+function replaceSourceDependenciesWithItemSource(dependsOn, sourceTaskSpecIds, itemMeta, options = {}) {
+    const replaced = [];
+    let inserted = false;
+    const shouldReplaceWithSource = itemMeta.sourceKind !== "partial" ||
+        options.keepPartialSourceDependency === true;
+    for (const dep of dependsOn) {
+        if (!sourceTaskSpecIds.has(dep)) {
+            replaced.push(dep);
+            continue;
+        }
+        if (!shouldReplaceWithSource)
+            continue;
+        if (!inserted) {
+            replaced.push(itemMeta.sourceSpecId);
+            inserted = true;
+        }
+    }
+    if (!inserted && shouldReplaceWithSource) {
+        replaced.push(itemMeta.sourceSpecId);
+    }
+    return [...new Set(replaced)];
+}
+function partialGeneratedTaskNeedsCompletedSourceContext(task) {
+    const artifactGraph = task.artifactGraph;
+    if (artifactGraph?.artifactAccess === "none")
+        return false;
+    return Boolean(artifactGraph?.sourceProjection !== undefined ||
+        (artifactGraph?.requiredReads?.length ?? 0) > 0);
+}
 async function extractArtifactGraphForeachItems(cwd, stage, sourceTasks) {
     const items = [];
+    const itemMetas = [];
     const path = stage.from?.path;
     if (typeof path !== "string" || !path.startsWith("$.")) {
         return {
             error: "foreach.from.path must be a control JSONPath like $.items",
         };
     }
+    let waitingForSources = false;
     for (const task of sourceTasks) {
         if (task.status !== "completed") {
+            if (stage.streaming && !isTerminalTaskStatus(task.status)) {
+                const partial = await extractPartialForeachItems(cwd, task, path);
+                if (partial.error)
+                    return { error: partial.error };
+                for (const item of partial.items) {
+                    items.push(item.item);
+                    itemMetas.push({
+                        sourceSpecId: task.specId,
+                        sourceKind: "partial",
+                        itemHash: item.itemHash,
+                        itemRef: `${task.specId}:${item.itemRef}`,
+                    });
+                }
+                waitingForSources = true;
+                continue;
+            }
             if (stage.sourcePolicy !== "partial")
                 return { error: `${task.taskId} did not complete` };
             continue;
@@ -495,7 +795,15 @@ async function extractArtifactGraphForeachItems(cwd, stage, sourceTasks) {
                 }
                 continue;
             }
-            items.push(...value);
+            for (const [index, item] of value.entries()) {
+                items.push(item);
+                itemMetas.push({
+                    sourceSpecId: task.specId,
+                    sourceKind: "control",
+                    itemHash: hashDynamicRequest(item),
+                    itemRef: `${task.specId}:control:${path}[${index}]`,
+                });
+            }
         }
         catch (error) {
             if (stage.sourcePolicy !== "partial") {
@@ -510,7 +818,27 @@ async function extractArtifactGraphForeachItems(cwd, stage, sourceTasks) {
             error: `foreach extracted ${items.length} items, exceeding maxItems=${stage.maxItems}`,
         };
     }
-    return { items };
+    return { items, itemMetas, waitingForSources };
+}
+async function extractPartialForeachItems(cwd, task, path) {
+    const partialPaths = task.artifactGraph?.output.partial?.paths ?? [];
+    if (!partialPaths.includes(path))
+        return { items: [] };
+    const taskDir = dirname(fromProjectPath(cwd, task.files.result));
+    let ledger = await readWorkflowPartialOutputLedger(taskDir).catch(() => undefined);
+    if (!ledger) {
+        ledger = await writeWorkflowPartialOutputLedgerFromFile({
+            taskDir,
+            outputFile: fromProjectPath(cwd, task.files.output),
+            allowedPaths: partialPaths,
+        }).catch(() => undefined);
+    }
+    if (!ledger)
+        return { items: [] };
+    const fatal = hasFatalPartialOutputIssue(ledger);
+    if (fatal)
+        return { items: [], error: fatal.message };
+    return { items: ledger.items.filter((item) => item.path === path) };
 }
 async function launchPendingTaskAt(cwd, run, compiledFlow, index, options = {}) {
     const task = run.tasks[index];
@@ -526,11 +854,14 @@ async function launchPendingTaskAt(cwd, run, compiledFlow, index, options = {})
         await writeRunRecord(cwd, run);
         return false;
     }
-    let launchTask = await prepareDagTask(cwd, run, compiledFlow, index);
-    if (task.outputRetry) {
-        launchTask = await prepareArtifactGraphRetryTask(cwd, task, launchTask);
-    }
+    let launchTask;
+    let prepareComplete = false;
     try {
+        launchTask = await prepareDagTask(cwd, run, compiledFlow, index);
+        if (task.outputRetry) {
+            launchTask = await prepareArtifactGraphRetryTask(cwd, task, launchTask);
+        }
+        prepareComplete = true;
         if (launchTask.kind === "support") {
             return await executeSupportTask(cwd, run, task, launchTask);
         }
@@ -547,11 +878,13 @@ async function launchPendingTaskAt(cwd, run, compiledFlow, index, options = {})
         return launch.kind === "launched";
     }
     catch (error) {
-        const statusDetail = launchTask.kind === "support"
-            ? "support_failed"
-            : launchTask.safety.requiresWorktree
-                ? "worktree_failed"
-                : "launch_failed";
+        const statusDetail = !prepareComplete
+            ? "prepare_failed"
+            : launchTask?.kind === "support"
+                ? "support_failed"
+                : launchTask?.safety.requiresWorktree
+                    ? "worktree_failed"
+                    : "launch_failed";
         setTaskTerminal(task, "failed", statusDetail, {
             lastMessage: error instanceof Error ? error.message : String(error),
         });
@@ -1815,21 +2148,36 @@ function uniqueStrings(values) {
 async function readCompiledWorkflow(cwd, runId) {
     return readJson(compiledWorkflowPath(cwd, runId));
 }
-function formatIndex(index) {
-    return index.runs
-        .map((run) => {
+async function formatIndex(cwd, index) {
+    const blocks = await Promise.all(index.runs.map(async (run) => {
         const lines = [
             `${run.runId} [${run.status}] type=${run.type} updated=${run.updatedAt}`,
             `tasks=${run.taskSummary.completed}/${run.taskSummary.total} completed, running=${run.taskSummary.running}, pending=${run.taskSummary.pending}, blocked=${run.taskSummary.blocked}, failed=${run.taskSummary.failed}, skipped=${run.taskSummary.skipped}, interrupted=${run.taskSummary.interrupted}`,
         ];
-        for (const task of run.tasks) {
+        for (const task of await indexTasksForStatus(cwd, run)) {
             const message = task.lastMessage ? ` — ${task.lastMessage}` : "";
             const kind = task.kind && task.kind !== "main" ? ` ${task.kind}` : "";
             lines.push(`- ${task.taskId}${kind} ${task.agent} [${task.status}/${task.statusDetail}]${message}`);
         }
         return lines.join("\n");
-    })
-        .join("\n\n");
+    }));
+    return blocks.join("\n\n");
+}
+async function indexTasksForStatus(cwd, run) {
+    if (Array.isArray(run.tasks))
+        return run.tasks;
+    const fullRun = await readRunRecord(cwd, run.runId).catch(() => undefined);
+    return (fullRun?.tasks.map((task) => ({
+        taskId: task.taskId,
+        displayName: task.displayName,
+        agent: task.agent,
+        kind: task.kind,
+        stageId: task.stageId,
+        backendHandle: task.backendHandle,
+        status: task.status,
+        statusDetail: task.statusDetail,
+        lastMessage: task.lastMessage,
+    })) ?? []);
 }
 function formatTask(task, detail) {
     const elapsed = task.elapsedMs !== undefined