@agwab/pi-workflow 0.2.1 → 0.4.0

package/src/engine.ts

@@ -11,6 +11,7 @@ import {
 	compiledWorkflowPath,
 	fromProjectPath,
 	indexSupervisorErrorPath,
+	isBlockedTaskResumableForResume,
 	isTerminalWorkflowStatus,
 	isTerminalTaskStatus,
 	listRunRecords,
@@ -33,6 +34,7 @@ import { resolveWorkflowBackend } from "./backend.js";
 import { ensureManagedWorktree } from "./worktree.js";
 import { resolveWorkflowHelperRef } from "./workflow-helpers.js";
 import { buildAvailableToolView } from "./tool-metadata.js";
+import { summarizeWorkflowTelemetry } from "./workflow-artifacts.js";
 import {
 	workflowBundleFingerprint,
 	workflowBundleSpecPath,
@@ -93,9 +95,12 @@ import {
 	assertRunTaskPositionalAlignment,
 	buildForeachGeneratedTasks,
 	dependenciesReady,
+	foreachStreamingEnabled,
+	foreachStreamingMinChunk,
 	markDagDependentsSkipped,
 	nextTaskRecordIndex,
 	reconcileDynamicGeneratedRunRecords,
+	reconcileForeachGeneratedRunRecords,
 	recoverStaleRunningDynamicControllers,
 	replaceDependencyList,
 	sourceStageIdsForFrom,
@@ -120,6 +125,12 @@ import {
 	DIRECT_DYNAMIC_RUNTIME_VERSION,
 	ensureDirectDynamicRuntimeBundle,
 } from "./dynamic-runtime-bundle.js";
+import {
+	hasFatalPartialOutputIssue,
+	readWorkflowPartialOutputLedger,
+	writeWorkflowPartialOutputLedgerFromFile,
+	type WorkflowPartialOutputItem,
+} from "./workflow-partial-output.js";
 import {
 	type CompiledDynamicWorkflowTask,
 	type CompiledTask,
@@ -147,6 +158,8 @@ const DYNAMIC_CONTROLLER_ENGINE_INTEGRITY_ERROR_MESSAGE =
 	"incompatible or stale pi-workflow engine: dynamic controller context is missing runDecisionLoop (rebuild dist / reload workflow engine)";
 const supervisorTimers = new Map<string, ReturnType<typeof setInterval>>();
 const supervisorRunMtimes = new Map<string, number>();
+const supervisorErrorCounts = new Map<string, number>();
+const MAX_SUPERVISOR_CONSECUTIVE_ERRORS = 3;
 
 export interface WorkflowRunOptions {
 	task?: string;
@@ -228,7 +241,7 @@ async function runLoadedWorkflowSpec(
 	const scheduled =
 		(await scheduleRun(cwd, run.runId, compiled, scheduleOptions)) ??
 		(await readRunRecord(cwd, run.runId));
-	if (scheduled.status === "running")
+	if (shouldWatchRun(scheduled))
 		watchRun(cwd, scheduled.runId, scheduleOptions);
 	return scheduled;
 }
@@ -245,6 +258,22 @@ export async function refreshRun(
 	return refreshed ?? current;
 }
 
+function hasActiveSchedulerWork(
+	run: Pick<WorkflowRunRecord, "status" | "taskSummary">,
+): boolean {
+	return (
+		run.status === "running" ||
+		run.taskSummary.running > 0 ||
+		run.taskSummary.pending > 0
+	);
+}
+
+function shouldWatchRun(
+	run: Pick<WorkflowRunRecord, "status" | "taskSummary">,
+): boolean {
+	return hasActiveSchedulerWork(run);
+}
+
 export async function waitForRun(
 	cwd: string,
 	runIdOrPrefix: string,
@@ -255,7 +284,7 @@ export async function waitForRun(
 	const deadline = Date.now() + timeout;
 	let run = await refreshRun(cwd, runIdOrPrefix);
 
-	while (run.status === "running") {
+	while (hasActiveSchedulerWork(run)) {
 		const beforeScheduleRemaining = deadline - Date.now();
 		if (beforeScheduleRemaining <= 0)
 			throw new Error(
@@ -265,7 +294,7 @@ export async function waitForRun(
 		run = await refreshRun(cwd, run.runId);
 		const remaining = deadline - Date.now();
 		if (remaining <= 0) {
-			if (run.status !== "running") return run;
+			if (!hasActiveSchedulerWork(run)) return run;
 			throw new Error(
 				`Flow run still running after ${timeout}ms: ${run.runId}`,
 			);
@@ -282,6 +311,64 @@ export interface ResumeRunSummary {
 	resetTaskIds: string[];
 }
 
+export interface StopRunSummary {
+	run: WorkflowRunRecord;
+	interruptedTaskIds: string[];
+}
+
+function assertBlockedRunResumable(run: WorkflowRunRecord): void {
+	if (run.status !== "blocked") return;
+	const blockers = run.tasks.filter(
+		(task) =>
+			task.status === "blocked" && !isBlockedTaskResumableForResume(task),
+	);
+	if (blockers.length === 0) return;
+	const details = blockers
+		.slice(0, 3)
+		.map((task) => `${task.specId} statusDetail=${task.statusDetail}`)
+		.join(", ");
+	throw new Error(
+		`Cannot resume blocked run ${run.runId}: non-resumable blocked task(s): ${details}. Resolve the attention/approval blocker before resuming.`,
+	);
+}
+
+export async function stopRun(
+	cwd: string,
+	runIdOrPrefix: string,
+): Promise<StopRunSummary> {
+	const current = await readRunRecord(cwd, runIdOrPrefix);
+	const stopped = await withRunLease(cwd, current.runId, async () => {
+		const run = await readRunRecord(cwd, current.runId);
+		if (isTerminalWorkflowStatus(run.status)) {
+			throw new Error(
+				`stop requires a non-terminal run; ${run.runId} is ${run.status}`,
+			);
+		}
+		await resolveWorkflowBackend(run)
+			.cleanupRun(cwd, run)
+			.catch(() => undefined);
+		const interruptedTaskIds: string[] = [];
+		for (const task of run.tasks) {
+			if (
+				setTaskTerminal(task, "interrupted", "workflow_stopped", {
+					exitCode: 130,
+					lastMessage: "Workflow stopped by user request",
+				})
+			) {
+				interruptedTaskIds.push(task.taskId);
+			}
+		}
+		await writeRunRecord(cwd, run);
+		unwatchRun(cwd, run.runId);
+		return { run, interruptedTaskIds };
+	});
+	if (!stopped)
+		throw new Error(
+			`Could not acquire workflow run lease for ${current.runId}`,
+		);
+	return stopped;
+}
+
 export async function resumeRun(
 	cwd: string,
 	runIdOrPrefix: string,
@@ -297,6 +384,7 @@ export async function resumeRun(
 			`resume requires a failed, interrupted, or resumable blocked run; ${current.runId} is ${current.status}`,
 		);
 	}
+	assertBlockedRunResumable(current);
 	const compiledFlow = await readCompiledWorkflow(cwd, current.runId);
 	const hasLoopTasks =
 		compiledFlow?.tasks.some(
@@ -314,6 +402,10 @@ export async function resumeRun(
 	const resetTaskIds: string[] = [];
 	const updated = await withRunLease(cwd, current.runId, async () => {
 		const run = await readRunRecord(cwd, current.runId);
+		assertBlockedRunResumable(run);
+		await resolveWorkflowBackend(run)
+			.cleanupRun(cwd, run)
+			.catch(() => undefined);
 		for (const task of run.tasks) {
 			if (resetTaskForResume(task)) resetTaskIds.push(task.taskId);
 		}
@@ -332,7 +424,7 @@ export async function resumeRun(
 	const scheduled =
 		(await scheduleRun(cwd, current.runId, undefined, options)) ??
 		(await readRunRecord(cwd, current.runId));
-	if (scheduled.status === "running") watchRun(cwd, scheduled.runId, options);
+	if (shouldWatchRun(scheduled)) watchRun(cwd, scheduled.runId, options);
 	return { run: scheduled, resetTaskIds };
 }
 
@@ -343,7 +435,7 @@ export async function resumeSupervisors(
 	try {
 		const runs = await listRunRecords(cwd);
 		for (const run of runs) {
-			if (run.status === "running") {
+			if (hasActiveSchedulerWork(run)) {
 				await scheduleRun(cwd, run.runId, undefined, options).catch((error) =>
 					recordSupervisorError(cwd, run.runId, error),
 				);
@@ -358,6 +450,15 @@ export async function resumeSupervisors(
 	}
 }
 
+function unwatchRun(cwd: string, runId: string): void {
+	const key = `${cwd}\0${runId}`;
+	const existing = supervisorTimers.get(key);
+	if (existing) clearInterval(existing);
+	supervisorTimers.delete(key);
+	supervisorRunMtimes.delete(key);
+	supervisorErrorCounts.delete(key);
+}
+
 export function watchRun(
 	cwd: string,
 	runId: string,
@@ -375,8 +476,9 @@ export function watchRun(
 			const currentMtime = afterMtime ?? beforeMtime;
 			if (currentMtime !== undefined)
 				supervisorRunMtimes.set(key, currentMtime);
+			supervisorErrorCounts.delete(key);
 
-			if (refreshed.status === "running") {
+			if (hasActiveSchedulerWork(refreshed)) {
 				const unchanged =
 					previousMtime !== undefined &&
 					currentMtime !== undefined &&
@@ -385,12 +487,18 @@ export function watchRun(
 				return;
 			}
 
-			const existing = supervisorTimers.get(key);
-			if (existing) clearInterval(existing);
-			supervisorTimers.delete(key);
-			supervisorRunMtimes.delete(key);
+			unwatchRun(cwd, runId);
 		})().catch((error) => {
-			void recordSupervisorError(cwd, runId, error);
+			if (isMissingRunError(error)) {
+				unwatchRun(cwd, runId);
+				return;
+			}
+			const failures = (supervisorErrorCounts.get(key) ?? 0) + 1;
+			supervisorErrorCounts.set(key, failures);
+			void recordSupervisorError(cwd, runId, error).finally(() => {
+				if (failures >= MAX_SUPERVISOR_CONSECUTIVE_ERRORS)
+					unwatchRun(cwd, runId);
+			});
 		});
 	}, POLL_INTERVAL_MS);
 
@@ -405,21 +513,49 @@ async function readRunMtimeMs(
 	try {
 		return (await stat(workflowRunPath(cwd, runId))).mtimeMs;
 	} catch (error) {
-		if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+		if (isEnoentError(error)) return undefined;
 		throw error;
 	}
 }
 
+function isEnoentError(error: unknown): boolean {
+	return (error as NodeJS.ErrnoException | undefined)?.code === "ENOENT";
+}
+
+function isMissingRunError(error: unknown): boolean {
+	return (
+		isEnoentError(error) ||
+		(error instanceof Error && /^Flow run not found: /.test(error.message))
+	);
+}
+
+function assertScheduleLeaseActive(signal?: AbortSignal): void {
+	if (!signal?.aborted) return;
+	const reason = (signal as AbortSignal & { reason?: unknown }).reason;
+	if (reason instanceof Error) throw reason;
+	throw new Error(
+		reason === undefined
+			? "Lost supervisor lease"
+			: `Lost supervisor lease: ${String(reason)}`,
+	);
+}
+
 export async function scheduleRun(
 	cwd: string,
 	runId: string,
 	compiled?: CompiledWorkflow,
 	options: WorkflowScheduleOptions = {},
 ): Promise<WorkflowRunRecord | undefined> {
-	return withRunLease(cwd, runId, async () => {
+	return withRunLease(cwd, runId, async (leaseSignal) => {
+		assertScheduleLeaseActive(leaseSignal);
 		let run = await readRunRecord(cwd, runId);
 		run = await resolveWorkflowBackend(run).refreshRun(cwd, run);
-		if (run.taskSummary.blocked > 0 || isTerminalWorkflowStatus(run.status))
+		if (isTerminalWorkflowStatus(run.status)) return run;
+		if (
+			run.taskSummary.blocked > 0 &&
+			run.taskSummary.pending === 0 &&
+			run.taskSummary.running === 0
+		)
 			return run;
 
 		const compiledFlow =
@@ -431,7 +567,8 @@ export async function scheduleRun(
 				`unsupported compiled workflow type: ${compiledFlow.type}`,
 			);
 		}
-		await scheduleDag(cwd, run, compiledFlow, options);
+		await scheduleDag(cwd, run, compiledFlow, options, leaseSignal);
+		assertScheduleLeaseActive(leaseSignal);
 
 		run = await readRunRecord(cwd, run.runId);
 		return run;
@@ -444,13 +581,13 @@ export async function formatStatus(cwd: string): Promise<string> {
 		await reconcileIndexedActiveRuns(cwd, cached);
 		const refreshed = (await readIndex(cwd).catch(() => cached)) ?? cached;
 		if (refreshed.runs.length === 0) return "No workflow runs found.";
-		return formatIndex(refreshed);
+		return formatIndex(cwd, refreshed);
 	}
 
 	await reconcileActiveRuns(cwd);
 	const rebuilt = await updateIndex(cwd).catch(() => readIndex(cwd));
 	if (!rebuilt || rebuilt.runs.length === 0) return "No workflow runs found.";
-	return formatIndex(rebuilt);
+	return formatIndex(cwd, rebuilt);
 }
 
 export async function formatRunDetails(
@@ -502,10 +639,12 @@ export function formatRun(
 	run: WorkflowRunRecord,
 	detail: "summary" | "full" = "summary",
 ): string {
+	const telemetry = summarizeWorkflowTelemetry(run);
 	const lines = [
 		`${run.runId} [${run.status}] type=${run.type} backend=${run.backend.type}/${run.backend.mode}`,
 		`created=${run.createdAt} updated=${run.updatedAt}`,
 		`tasks=${run.taskSummary.completed}/${run.taskSummary.total} completed, running=${run.taskSummary.running}, pending=${run.taskSummary.pending}, blocked=${run.taskSummary.blocked}, failed=${run.taskSummary.failed}, interrupted=${run.taskSummary.interrupted}`,
+		`completion=${telemetry.completion.health}, outputRetries=${telemetry.retryCounts.output}, launchRetries=${telemetry.retryCounts.launch}, resumeEvents=${telemetry.resumeCounts.events}, contextLimitFailures=${telemetry.completion.contextLimitFailures}`,
 	];
 
 	for (const task of run.tasks) {
@@ -518,7 +657,7 @@ export function formatRun(
 async function reconcileActiveRuns(cwd: string): Promise<void> {
 	const runs = await listRunRecords(cwd);
 	for (const run of runs) {
-		if (run.status === "running")
+		if (hasActiveSchedulerWork(run))
 			await refreshRun(cwd, run.runId).catch((error) =>
 				recordSupervisorError(cwd, run.runId, error),
 			);
@@ -530,7 +669,7 @@ async function reconcileIndexedActiveRuns(
 	index: WorkflowIndexRecord,
 ): Promise<void> {
 	for (const run of index.runs) {
-		if (run.status === "running")
+		if (hasActiveSchedulerWork(run))
 			await refreshRun(cwd, run.runId).catch((error) =>
 				recordSupervisorError(cwd, run.runId, error),
 			);
@@ -561,7 +700,9 @@ async function scheduleDag(
 	run: WorkflowRunRecord,
 	compiledFlow: CompiledWorkflow,
 	options: WorkflowScheduleOptions = {},
+	leaseSignal?: AbortSignal,
 ): Promise<void> {
+	assertScheduleLeaseActive(leaseSignal);
 	if (compiledFlow.type === WORKFLOW_RUN_TYPE) {
 		const loopReconciled = await reconcileLoopTaskMaterialization(
 			cwd,
@@ -569,6 +710,16 @@ async function scheduleDag(
 			compiledFlow,
 		);
 		if (loopReconciled) return;
+		const foreachReconciled = reconcileForeachGeneratedRunRecords(
+			cwd,
+			run,
+			compiledFlow,
+		);
+		if (foreachReconciled) {
+			await writeJsonAtomic(compiledWorkflowPath(cwd, run.runId), compiledFlow);
+			await writeRunRecord(cwd, run);
+			return;
+		}
 		const dynamicReconciled = reconcileDynamicGeneratedRunRecords(
 			cwd,
 			run,
@@ -601,6 +752,7 @@ async function scheduleDag(
 		index < run.tasks.length && running < maxConcurrency;
 		index += 1
 	) {
+		assertScheduleLeaseActive(leaseSignal);
 		const task = run.tasks[index];
 		const compiledTask = compiledFlow.tasks[index];
 		if (!task || !compiledTask || task.status !== "pending") continue;
@@ -632,6 +784,7 @@ async function scheduleDag(
 				compiledTask,
 			);
 			if (changed) return;
+			if (foreachStreamingEnabled(compiledTask)) continue;
 		}
 
 		if (compiledTask.stageMaxConcurrency !== undefined) {
@@ -647,6 +800,7 @@ async function scheduleDag(
 				continue;
 		}
 
+		assertScheduleLeaseActive(leaseSignal);
 		const launched = await launchPendingTaskAt(
 			cwd,
 			run,
@@ -654,7 +808,8 @@ async function scheduleDag(
 			index,
 			options,
 		);
-		if (launched) running += 1;
+		assertScheduleLeaseActive(leaseSignal);
+		if (launched && run.tasks[index]?.status === "running") running += 1;
 	}
 }
 
@@ -780,12 +935,14 @@ async function materializeForeachTask(
 	const sourceTasks = run.tasks.filter((task) =>
 		sourceStageIds.includes(task.stageId ?? ""),
 	);
+	const streaming = foreachStreamingEnabled(template);
 	const extracted = await extractArtifactGraphForeachItems(
 		cwd,
 		{
 			from: template.foreach.from,
 			sourcePolicy: stageSourcePolicy(compiledFlow, template.stageId),
 			maxItems: template.foreach.maxItems,
+			streaming,
 		},
 		sourceTasks,
 	);
@@ -813,7 +970,33 @@ async function materializeForeachTask(
 	}
 
 	const placeholderSpecId = template.id;
+	if (streaming) {
+		return await materializeStreamingForeachTask({
+			cwd,
+			run,
+			compiledFlow,
+			index,
+			templateRunTask,
+			placeholderSpecId,
+			sourceTaskSpecIds: sourceTasks.map((task) => task.specId),
+			itemMetas: extracted.itemMetas ?? [],
+			generatedTasks: generated.tasks,
+			waitingForSources: extracted.waitingForSources ?? false,
+			minChunk: foreachStreamingMinChunk(template),
+		});
+	}
 	const generatedSpecIds = generated.tasks.map((task) => task.id);
+	const hasDownstreamDependents = compiledFlow.tasks.some(
+		(task, taskIndex) =>
+			taskIndex !== index && (task.dependsOn ?? []).includes(placeholderSpecId),
+	);
+	if (generatedSpecIds.length === 0 && !hasDownstreamDependents) {
+		setTaskTerminal(templateRunTask, "completed", "foreach_empty", {
+			lastMessage: "foreach produced 0 item(s)",
+		});
+		await writeRunRecord(cwd, run);
+		return true;
+	}
 	compiledFlow.tasks.splice(index, 1, ...generated.tasks);
 	updateDownstreamDependencies(
 		compiledFlow,
@@ -840,20 +1023,279 @@ async function materializeForeachTask(
 	return true;
 }
 
+async function materializeStreamingForeachTask(input: {
+	cwd: string;
+	run: WorkflowRunRecord;
+	compiledFlow: CompiledWorkflow;
+	index: number;
+	templateRunTask: WorkflowTaskRunRecord;
+	placeholderSpecId: string;
+	sourceTaskSpecIds: string[];
+	itemMetas: ForeachExtractedItemMeta[];
+	generatedTasks: CompiledTask[];
+	waitingForSources: boolean;
+	minChunk: number;
+}): Promise<boolean> {
+	const sourceTaskSpecIdSet = new Set(input.sourceTaskSpecIds);
+	const generatedTasksWithItemDeps = input.generatedTasks.map((task, index) => {
+		const itemMeta = input.itemMetas[index];
+		if (!itemMeta) return task;
+		return {
+			...task,
+			dependsOn: replaceSourceDependenciesWithItemSource(
+				task.dependsOn ?? [],
+				sourceTaskSpecIdSet,
+				itemMeta,
+				{
+					keepPartialSourceDependency:
+						partialGeneratedTaskNeedsCompletedSourceContext(task),
+				},
+			),
+			foreachGenerated: {
+				...(task.foreachGenerated ?? {
+					placeholderSpecId: input.placeholderSpecId,
+				}),
+				itemHash: itemMeta.itemHash,
+				itemSourceSpecId: itemMeta.sourceSpecId,
+				itemSourceKind: itemMeta.sourceKind,
+				itemRef: itemMeta.itemRef,
+			},
+		};
+	});
+	const existingGeneratedTasks = input.compiledFlow.tasks.filter(
+		(task) =>
+			task.foreachGenerated?.placeholderSpecId === input.placeholderSpecId,
+	);
+	const existingGeneratedSpecIds = existingGeneratedTasks.map(
+		(task) => task.id,
+	);
+	const existingGeneratedTaskBySpecId = new Map(
+		existingGeneratedTasks.map((task) => [task.id, task]),
+	);
+	for (const task of generatedTasksWithItemDeps) {
+		const existing = existingGeneratedTaskBySpecId.get(task.id);
+		const existingHash = existing?.foreachGenerated?.itemHash;
+		const nextHash = task.foreachGenerated?.itemHash;
+		if (existing && existingHash && nextHash && existingHash !== nextHash) {
+			setTaskTerminal(
+				input.templateRunTask,
+				"blocked",
+				"foreach_expansion_blocked",
+				{
+					lastMessage: `foreach streaming item ${task.id} changed after materialization`,
+				},
+			);
+			await writeRunRecord(input.cwd, input.run);
+			return true;
+		}
+	}
+	const existingGeneratedSpecIdSet = new Set(existingGeneratedSpecIds);
+	const finalGeneratedSpecIdSet = new Set(
+		generatedTasksWithItemDeps.map((task) => task.id),
+	);
+	if (!input.waitingForSources) {
+		const withdrawn = existingGeneratedTasks.find(
+			(task) =>
+				task.foreachGenerated?.itemSourceKind === "partial" &&
+				!finalGeneratedSpecIdSet.has(task.id),
+		);
+		if (withdrawn) {
+			setTaskTerminal(
+				input.templateRunTask,
+				"blocked",
+				"foreach_expansion_blocked",
+				{
+					lastMessage: `foreach streaming item ${withdrawn.id} was published as partial output but is missing from final control`,
+				},
+			);
+			await writeRunRecord(input.cwd, input.run);
+			return true;
+		}
+	}
+	const newGeneratedTasks = generatedTasksWithItemDeps.filter(
+		(task) => !existingGeneratedSpecIdSet.has(task.id),
+	);
+	const allGeneratedSpecIds = [
+		...existingGeneratedSpecIds,
+		...newGeneratedTasks.map((task) => task.id),
+	];
+	const shouldHoldForMinChunk =
+		input.waitingForSources &&
+		newGeneratedTasks.length > 0 &&
+		newGeneratedTasks.length < input.minChunk;
+	if (shouldHoldForMinChunk) return false;
+
+	let changed = false;
+	if (newGeneratedTasks.length > 0) {
+		let compiledInsertIndex = input.index + 1;
+		while (
+			input.compiledFlow.tasks[compiledInsertIndex]?.foreachGenerated
+				?.placeholderSpecId === input.placeholderSpecId
+		) {
+			compiledInsertIndex += 1;
+		}
+		input.compiledFlow.tasks.splice(
+			compiledInsertIndex,
+			0,
+			...newGeneratedTasks,
+		);
+
+		let runInsertIndex = input.index + 1;
+		while (
+			input.run.tasks[runInsertIndex]?.foreachGenerated?.placeholderSpecId ===
+			input.placeholderSpecId
+		) {
+			runInsertIndex += 1;
+		}
+		const nextIndex = nextTaskRecordIndex(input.run);
+		const generatedRunTasks = newGeneratedTasks.map((task, offset) =>
+			createTaskRunRecord(input.cwd, input.run.runId, task, nextIndex + offset),
+		);
+		input.run.tasks.splice(runInsertIndex, 0, ...generatedRunTasks);
+		changed = true;
+	}
+
+	const dependencyTargets = [input.placeholderSpecId, ...allGeneratedSpecIds];
+	for (const task of input.compiledFlow.tasks) {
+		if (!task.dependsOn) continue;
+		const replaced = replaceDependencyList(
+			task.dependsOn,
+			input.placeholderSpecId,
+			dependencyTargets,
+		);
+		if (JSON.stringify(task.dependsOn) !== JSON.stringify(replaced)) {
+			task.dependsOn = replaced;
+			changed = true;
+		}
+	}
+	for (const task of input.run.tasks) {
+		if (!task.dependsOn) continue;
+		const replaced = replaceDependencyList(
+			task.dependsOn,
+			input.placeholderSpecId,
+			dependencyTargets,
+		);
+		if (JSON.stringify(task.dependsOn) !== JSON.stringify(replaced)) {
+			task.dependsOn = replaced;
+			changed = true;
+		}
+	}
+
+	if (!input.waitingForSources) {
+		const statusDetail =
+			allGeneratedSpecIds.length === 0
+				? "foreach_empty"
+				: "foreach_streaming_complete";
+		const lastMessage =
+			allGeneratedSpecIds.length === 0
+				? "foreach produced 0 item(s)"
+				: `foreach streaming materialized ${allGeneratedSpecIds.length} item(s)`;
+		setTaskTerminal(input.templateRunTask, "completed", statusDetail, {
+			lastMessage,
+		});
+		changed = true;
+	} else if (newGeneratedTasks.length > 0) {
+		input.templateRunTask.statusDetail = "foreach_streaming_waiting";
+		input.templateRunTask.lastMessage = `foreach streaming materialized ${allGeneratedSpecIds.length} item(s); waiting for more source tasks`;
+		changed = true;
+	}
+
+	if (!changed) return false;
+	await writeJsonAtomic(
+		compiledWorkflowPath(input.cwd, input.run.runId),
+		input.compiledFlow,
+	);
+	await writeRunRecord(input.cwd, input.run);
+	return true;
+}
+
+function replaceSourceDependenciesWithItemSource(
+	dependsOn: string[],
+	sourceTaskSpecIds: Set<string>,
+	itemMeta: ForeachExtractedItemMeta,
+	options: { keepPartialSourceDependency?: boolean } = {},
+): string[] {
+	const replaced: string[] = [];
+	let inserted = false;
+	const shouldReplaceWithSource =
+		itemMeta.sourceKind !== "partial" ||
+		options.keepPartialSourceDependency === true;
+	for (const dep of dependsOn) {
+		if (!sourceTaskSpecIds.has(dep)) {
+			replaced.push(dep);
+			continue;
+		}
+		if (!shouldReplaceWithSource) continue;
+		if (!inserted) {
+			replaced.push(itemMeta.sourceSpecId);
+			inserted = true;
+		}
+	}
+	if (!inserted && shouldReplaceWithSource) {
+		replaced.push(itemMeta.sourceSpecId);
+	}
+	return [...new Set(replaced)];
+}
+
+function partialGeneratedTaskNeedsCompletedSourceContext(
+	task: CompiledTask,
+): boolean {
+	const artifactGraph = task.artifactGraph;
+	if (artifactGraph?.artifactAccess === "none") return false;
+	return Boolean(
+		artifactGraph?.sourceProjection !== undefined ||
+			(artifactGraph?.requiredReads?.length ?? 0) > 0,
+	);
+}
+
+interface ForeachExtractedItemMeta {
+	sourceSpecId: string;
+	sourceKind: "control" | "partial";
+	itemHash: string;
+	itemRef: string;
+}
+
 async function extractArtifactGraphForeachItems(
 	cwd: string,
-	stage: { from: unknown; sourcePolicy?: string; maxItems?: number },
+	stage: {
+		from: unknown;
+		sourcePolicy?: string;
+		maxItems?: number;
+		streaming?: boolean;
+	},
 	sourceTasks: WorkflowTaskRunRecord[],
-): Promise<{ items?: unknown[]; error?: string }> {
+): Promise<{
+	items?: unknown[];
+	itemMetas?: ForeachExtractedItemMeta[];
+	error?: string;
+	waitingForSources?: boolean;
+}> {
 	const items: unknown[] = [];
+	const itemMetas: ForeachExtractedItemMeta[] = [];
 	const path = (stage.from as any)?.path;
 	if (typeof path !== "string" || !path.startsWith("$.")) {
 		return {
 			error: "foreach.from.path must be a control JSONPath like $.items",
 		};
 	}
+	let waitingForSources = false;
 	for (const task of sourceTasks) {
 		if (task.status !== "completed") {
+			if (stage.streaming && !isTerminalTaskStatus(task.status)) {
+				const partial = await extractPartialForeachItems(cwd, task, path);
+				if (partial.error) return { error: partial.error };
+				for (const item of partial.items) {
+					items.push(item.item);
+					itemMetas.push({
+						sourceSpecId: task.specId,
+						sourceKind: "partial",
+						itemHash: item.itemHash,
+						itemRef: `${task.specId}:${item.itemRef}`,
+					});
+				}
+				waitingForSources = true;
+				continue;
+			}
 			if (stage.sourcePolicy !== "partial")
 				return { error: `${task.taskId} did not complete` };
 			continue;
@@ -869,7 +1311,15 @@ async function extractArtifactGraphForeachItems(
 				}
 				continue;
 			}
-			items.push(...value);
+			for (const [index, item] of value.entries()) {
+				items.push(item);
+				itemMetas.push({
+					sourceSpecId: task.specId,
+					sourceKind: "control",
+					itemHash: hashDynamicRequest(item),
+					itemRef: `${task.specId}:control:${path}[${index}]`,
+				});
+			}
 		} catch (error) {
 			if (stage.sourcePolicy !== "partial") {
 				return {
@@ -883,7 +1333,31 @@ async function extractArtifactGraphForeachItems(
 			error: `foreach extracted ${items.length} items, exceeding maxItems=${stage.maxItems}`,
 		};
 	}
-	return { items };
+	return { items, itemMetas, waitingForSources };
+}
+
+async function extractPartialForeachItems(
+	cwd: string,
+	task: WorkflowTaskRunRecord,
+	path: string,
+): Promise<{ items: WorkflowPartialOutputItem[]; error?: string }> {
+	const partialPaths = task.artifactGraph?.output.partial?.paths ?? [];
+	if (!partialPaths.includes(path)) return { items: [] };
+	const taskDir = dirname(fromProjectPath(cwd, task.files.result));
+	let ledger = await readWorkflowPartialOutputLedger(taskDir).catch(
+		() => undefined,
+	);
+	if (!ledger) {
+		ledger = await writeWorkflowPartialOutputLedgerFromFile({
+			taskDir,
+			outputFile: fromProjectPath(cwd, task.files.output),
+			allowedPaths: partialPaths,
+		}).catch(() => undefined);
+	}
+	if (!ledger) return { items: [] };
+	const fatal = hasFatalPartialOutputIssue(ledger);
+	if (fatal) return { items: [], error: fatal.message };
+	return { items: ledger.items.filter((item) => item.path === path) };
 }
 
 async function launchPendingTaskAt(
@@ -906,12 +1380,15 @@ async function launchPendingTaskAt(
 		return false;
 	}
 
-	let launchTask = await prepareDagTask(cwd, run, compiledFlow, index);
-	if (task.outputRetry) {
-		launchTask = await prepareArtifactGraphRetryTask(cwd, task, launchTask);
-	}
-
+	let launchTask: CompiledWorkflow["tasks"][number] | undefined;
+	let prepareComplete = false;
 	try {
+		launchTask = await prepareDagTask(cwd, run, compiledFlow, index);
+		if (task.outputRetry) {
+			launchTask = await prepareArtifactGraphRetryTask(cwd, task, launchTask);
+		}
+		prepareComplete = true;
+
 		if (launchTask.kind === "support") {
 			return await executeSupportTask(cwd, run, task, launchTask);
 		}
@@ -939,10 +1416,11 @@ async function launchPendingTaskAt(
 		if (launch.kind === "fatal") throw new Error(launch.message);
 		return launch.kind === "launched";
 	} catch (error) {
-		const statusDetail =
-			launchTask.kind === "support"
+		const statusDetail = !prepareComplete
+			? "prepare_failed"
+			: launchTask?.kind === "support"
 				? "support_failed"
-				: launchTask.safety.requiresWorktree
+				: launchTask?.safety.requiresWorktree
 					? "worktree_failed"
 					: "launch_failed";
 		setTaskTerminal(task, "failed", statusDetail, {
@@ -2692,14 +3170,17 @@ async function readCompiledWorkflow(
 	return readJson<CompiledWorkflow>(compiledWorkflowPath(cwd, runId));
 }
 
-function formatIndex(index: WorkflowIndexRecord): string {
-	return index.runs
-		.map((run) => {
+async function formatIndex(
+	cwd: string,
+	index: WorkflowIndexRecord,
+): Promise<string> {
+	const blocks = await Promise.all(
+		index.runs.map(async (run) => {
 			const lines = [
 				`${run.runId} [${run.status}] type=${run.type} updated=${run.updatedAt}`,
 				`tasks=${run.taskSummary.completed}/${run.taskSummary.total} completed, running=${run.taskSummary.running}, pending=${run.taskSummary.pending}, blocked=${run.taskSummary.blocked}, failed=${run.taskSummary.failed}, skipped=${run.taskSummary.skipped}, interrupted=${run.taskSummary.interrupted}`,
 			];
-			for (const task of run.tasks) {
+			for (const task of await indexTasksForStatus(cwd, run)) {
 				const message = task.lastMessage ? ` — ${task.lastMessage}` : "";
 				const kind = task.kind && task.kind !== "main" ? ` ${task.kind}` : "";
 				lines.push(
@@ -2707,8 +3188,34 @@ function formatIndex(index: WorkflowIndexRecord): string {
 				);
 			}
 			return lines.join("\n");
-		})
-		.join("\n\n");
+		}),
+	);
+	return blocks.join("\n\n");
+}
+
+type WorkflowIndexTaskEntry = NonNullable<
+	WorkflowIndexRecord["runs"][number]["tasks"]
+>[number];
+
+async function indexTasksForStatus(
+	cwd: string,
+	run: WorkflowIndexRecord["runs"][number],
+): Promise<WorkflowIndexTaskEntry[]> {
+	if (Array.isArray(run.tasks)) return run.tasks;
+	const fullRun = await readRunRecord(cwd, run.runId).catch(() => undefined);
+	return (
+		fullRun?.tasks.map((task) => ({
+			taskId: task.taskId,
+			displayName: task.displayName,
+			agent: task.agent,
+			kind: task.kind,
+			stageId: task.stageId,
+			backendHandle: task.backendHandle,
+			status: task.status,
+			statusDetail: task.statusDetail,
+			lastMessage: task.lastMessage,
+		})) ?? []
+	);
 }
 
 function formatTask(