@agwab/pi-workflow 0.2.0 → 0.3.0

package/src/engine.ts

@@ -1,5 +1,5 @@
-import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
-import { dirname, extname, join, relative, resolve } from "node:path";
+import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { dirname, extname, join, resolve } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { Worker } from "node:worker_threads";
 
@@ -23,7 +23,7 @@ import {
 	toProjectPath,
 	updateIndex,
 	withRunLease,
-	workflowRunDir,
+	workflowRunPath,
 	writeJsonAtomic,
 	writeRunRecord,
 	writeCompiledRunArtifact,
@@ -40,6 +40,7 @@ import {
 import {
 	readSimpleJsonPath,
 	type WorkflowModelInfo,
+	type WorkflowRuntimeDefaults,
 } from "./workflow-runtime.js";
 import {
 	dynamicRunDir,
@@ -74,7 +75,6 @@ import {
 	isDynamicCompiledTaskPayload,
 	normalizeDynamicAgentRequest,
 	readDynamicGeneratedTaskResult,
-	type DynamicAgentRequest,
 } from "./dynamic-generated-task-runtime.js";
 import {
 	optionalEventString,
@@ -96,6 +96,7 @@ import {
 	markDagDependentsSkipped,
 	nextTaskRecordIndex,
 	reconcileDynamicGeneratedRunRecords,
+	reconcileForeachGeneratedRunRecords,
 	recoverStaleRunningDynamicControllers,
 	replaceDependencyList,
 	sourceStageIdsForFrom,
@@ -116,10 +117,6 @@ import {
 	readSupportSources,
 	writeArtifactGraphDynamicResult,
 } from "./artifact-graph-runtime.js";
-import {
-	isDynamicOutputProfile,
-	type DynamicOutputProfile,
-} from "./dynamic-profiles.js";
 import {
 	DIRECT_DYNAMIC_RUNTIME_VERSION,
 	ensureDirectDynamicRuntimeBundle,
@@ -128,7 +125,6 @@ import {
 	type CompiledDynamicWorkflowTask,
 	type CompiledTask,
 	type CompiledWorkflow,
-	type ThinkingLevel,
 	WORKFLOW_RUN_TYPE,
 	type WorkflowIndexRecord,
 	type WorkflowRunRecord,
@@ -151,10 +147,14 @@ const DYNAMIC_CONTROLLER_ENGINE_CAPABILITIES = Object.freeze({
 const DYNAMIC_CONTROLLER_ENGINE_INTEGRITY_ERROR_MESSAGE =
 	"incompatible or stale pi-workflow engine: dynamic controller context is missing runDecisionLoop (rebuild dist / reload workflow engine)";
 const supervisorTimers = new Map<string, ReturnType<typeof setInterval>>();
+const supervisorRunMtimes = new Map<string, number>();
+const supervisorErrorCounts = new Map<string, number>();
+const MAX_SUPERVISOR_CONSECUTIVE_ERRORS = 3;
 
 export interface WorkflowRunOptions {
 	task?: string;
-	runtimeDefaults?: { model?: string; thinking?: ThinkingLevel };
+	runtimeOverrides?: WorkflowRuntimeDefaults;
+	runtimeDefaults?: WorkflowRuntimeDefaults;
 	availableModels?: WorkflowModelInfo[];
 	dynamicUi?: DynamicWorkflowUi;
 	runId?: string;
@@ -163,6 +163,7 @@ export interface WorkflowRunOptions {
 
 interface WorkflowScheduleOptions {
 	dynamicUi?: DynamicWorkflowUi;
+	availableModels?: WorkflowModelInfo[];
 }
 
 export async function runWorkflowSpec(
@@ -207,6 +208,7 @@ async function runLoadedWorkflowSpec(
 		cwd,
 		specPath,
 		task: options.task,
+		runtimeOverrides: options.runtimeOverrides,
 		runtimeDefaults: options.runtimeDefaults,
 		availableModels: options.availableModels,
 	});
@@ -222,12 +224,15 @@ async function runLoadedWorkflowSpec(
 		await writeRunRecord(cwd, run);
 	});
 
+	const scheduleOptions = {
+		dynamicUi: options.dynamicUi,
+		availableModels: options.availableModels,
+	};
 	const scheduled =
-		(await scheduleRun(cwd, run.runId, compiled, {
-			dynamicUi: options.dynamicUi,
-		})) ?? (await readRunRecord(cwd, run.runId));
-	if (scheduled.status === "running")
-		watchRun(cwd, scheduled.runId, { dynamicUi: options.dynamicUi });
+		(await scheduleRun(cwd, run.runId, compiled, scheduleOptions)) ??
+		(await readRunRecord(cwd, run.runId));
+	if (shouldWatchRun(scheduled))
+		watchRun(cwd, scheduled.runId, scheduleOptions);
 	return scheduled;
 }
 
@@ -243,6 +248,22 @@ export async function refreshRun(
 	return refreshed ?? current;
 }
 
+function hasActiveSchedulerWork(
+	run: Pick<WorkflowRunRecord, "status" | "taskSummary">,
+): boolean {
+	return (
+		run.status === "running" ||
+		run.taskSummary.running > 0 ||
+		run.taskSummary.pending > 0
+	);
+}
+
+function shouldWatchRun(
+	run: Pick<WorkflowRunRecord, "status" | "taskSummary">,
+): boolean {
+	return hasActiveSchedulerWork(run);
+}
+
 export async function waitForRun(
 	cwd: string,
 	runIdOrPrefix: string,
@@ -253,7 +274,7 @@ export async function waitForRun(
 	const deadline = Date.now() + timeout;
 	let run = await refreshRun(cwd, runIdOrPrefix);
 
-	while (run.status === "running") {
+	while (hasActiveSchedulerWork(run)) {
 		const beforeScheduleRemaining = deadline - Date.now();
 		if (beforeScheduleRemaining <= 0)
 			throw new Error(
@@ -263,7 +284,7 @@ export async function waitForRun(
 		run = await refreshRun(cwd, run.runId);
 		const remaining = deadline - Date.now();
 		if (remaining <= 0) {
-			if (run.status !== "running") return run;
+			if (!hasActiveSchedulerWork(run)) return run;
 			throw new Error(
 				`Flow run still running after ${timeout}ms: ${run.runId}`,
 			);
@@ -280,6 +301,48 @@ export interface ResumeRunSummary {
 	resetTaskIds: string[];
 }
 
+export interface StopRunSummary {
+	run: WorkflowRunRecord;
+	interruptedTaskIds: string[];
+}
+
+export async function stopRun(
+	cwd: string,
+	runIdOrPrefix: string,
+): Promise<StopRunSummary> {
+	const current = await readRunRecord(cwd, runIdOrPrefix);
+	const stopped = await withRunLease(cwd, current.runId, async () => {
+		const run = await readRunRecord(cwd, current.runId);
+		if (isTerminalWorkflowStatus(run.status)) {
+			throw new Error(
+				`stop requires a non-terminal run; ${run.runId} is ${run.status}`,
+			);
+		}
+		await resolveWorkflowBackend(run)
+			.cleanupRun(cwd, run)
+			.catch(() => undefined);
+		const interruptedTaskIds: string[] = [];
+		for (const task of run.tasks) {
+			if (
+				setTaskTerminal(task, "interrupted", "workflow_stopped", {
+					exitCode: 130,
+					lastMessage: "Workflow stopped by user request",
+				})
+			) {
+				interruptedTaskIds.push(task.taskId);
+			}
+		}
+		await writeRunRecord(cwd, run);
+		unwatchRun(cwd, run.runId);
+		return { run, interruptedTaskIds };
+	});
+	if (!stopped)
+		throw new Error(
+			`Could not acquire workflow run lease for ${current.runId}`,
+		);
+	return stopped;
+}
+
 export async function resumeRun(
 	cwd: string,
 	runIdOrPrefix: string,
@@ -312,6 +375,9 @@ export async function resumeRun(
 	const resetTaskIds: string[] = [];
 	const updated = await withRunLease(cwd, current.runId, async () => {
 		const run = await readRunRecord(cwd, current.runId);
+		await resolveWorkflowBackend(run)
+			.cleanupRun(cwd, run)
+			.catch(() => undefined);
 		for (const task of run.tasks) {
 			if (resetTaskForResume(task)) resetTaskIds.push(task.taskId);
 		}
@@ -330,7 +396,7 @@ export async function resumeRun(
 	const scheduled =
 		(await scheduleRun(cwd, current.runId, undefined, options)) ??
 		(await readRunRecord(cwd, current.runId));
-	if (scheduled.status === "running") watchRun(cwd, scheduled.runId, options);
+	if (shouldWatchRun(scheduled)) watchRun(cwd, scheduled.runId, options);
 	return { run: scheduled, resetTaskIds };
 }
 
@@ -341,7 +407,7 @@ export async function resumeSupervisors(
 	try {
 		const runs = await listRunRecords(cwd);
 		for (const run of runs) {
-			if (run.status === "running") {
+			if (hasActiveSchedulerWork(run)) {
 				await scheduleRun(cwd, run.runId, undefined, options).catch((error) =>
 					recordSupervisorError(cwd, run.runId, error),
 				);
@@ -356,6 +422,15 @@ export async function resumeSupervisors(
 	}
 }
 
+function unwatchRun(cwd: string, runId: string): void {
+	const key = `${cwd}\0${runId}`;
+	const existing = supervisorTimers.get(key);
+	if (existing) clearInterval(existing);
+	supervisorTimers.delete(key);
+	supervisorRunMtimes.delete(key);
+	supervisorErrorCounts.delete(key);
+}
+
 export function watchRun(
 	cwd: string,
 	runId: string,
@@ -366,17 +441,36 @@ export function watchRun(
 
 	const timer = setInterval(() => {
 		void (async () => {
+			const previousMtime = supervisorRunMtimes.get(key);
+			const beforeMtime = await readRunMtimeMs(cwd, runId);
 			const refreshed = await refreshRun(cwd, runId);
-			if (refreshed.status === "running") {
-				await scheduleRun(cwd, runId, undefined, options);
+			const afterMtime = await readRunMtimeMs(cwd, runId);
+			const currentMtime = afterMtime ?? beforeMtime;
+			if (currentMtime !== undefined)
+				supervisorRunMtimes.set(key, currentMtime);
+			supervisorErrorCounts.delete(key);
+
+			if (hasActiveSchedulerWork(refreshed)) {
+				const unchanged =
+					previousMtime !== undefined &&
+					currentMtime !== undefined &&
+					currentMtime <= previousMtime;
+				if (!unchanged) await scheduleRun(cwd, runId, undefined, options);
 				return;
 			}
 
-			const existing = supervisorTimers.get(key);
-			if (existing) clearInterval(existing);
-			supervisorTimers.delete(key);
+			unwatchRun(cwd, runId);
 		})().catch((error) => {
-			void recordSupervisorError(cwd, runId, error);
+			if (isMissingRunError(error)) {
+				unwatchRun(cwd, runId);
+				return;
+			}
+			const failures = (supervisorErrorCounts.get(key) ?? 0) + 1;
+			supervisorErrorCounts.set(key, failures);
+			void recordSupervisorError(cwd, runId, error).finally(() => {
+				if (failures >= MAX_SUPERVISOR_CONSECUTIVE_ERRORS)
+					unwatchRun(cwd, runId);
+			});
 		});
 	}, POLL_INTERVAL_MS);
 
@@ -384,6 +478,29 @@ export function watchRun(
 	supervisorTimers.set(key, timer);
 }
 
+async function readRunMtimeMs(
+	cwd: string,
+	runId: string,
+): Promise<number | undefined> {
+	try {
+		return (await stat(workflowRunPath(cwd, runId))).mtimeMs;
+	} catch (error) {
+		if (isEnoentError(error)) return undefined;
+		throw error;
+	}
+}
+
+function isEnoentError(error: unknown): boolean {
+	return (error as NodeJS.ErrnoException | undefined)?.code === "ENOENT";
+}
+
+function isMissingRunError(error: unknown): boolean {
+	return (
+		isEnoentError(error) ||
+		(error instanceof Error && /^Flow run not found: /.test(error.message))
+	);
+}
+
 export async function scheduleRun(
 	cwd: string,
 	runId: string,
@@ -393,7 +510,12 @@ export async function scheduleRun(
 	return withRunLease(cwd, runId, async () => {
 		let run = await readRunRecord(cwd, runId);
 		run = await resolveWorkflowBackend(run).refreshRun(cwd, run);
-		if (run.taskSummary.blocked > 0 || isTerminalWorkflowStatus(run.status))
+		if (isTerminalWorkflowStatus(run.status)) return run;
+		if (
+			run.taskSummary.blocked > 0 &&
+			run.taskSummary.pending === 0 &&
+			run.taskSummary.running === 0
+		)
 			return run;
 
 		const compiledFlow =
@@ -492,7 +614,7 @@ export function formatRun(
 async function reconcileActiveRuns(cwd: string): Promise<void> {
 	const runs = await listRunRecords(cwd);
 	for (const run of runs) {
-		if (run.status === "running")
+		if (hasActiveSchedulerWork(run))
 			await refreshRun(cwd, run.runId).catch((error) =>
 				recordSupervisorError(cwd, run.runId, error),
 			);
@@ -504,7 +626,7 @@ async function reconcileIndexedActiveRuns(
 	index: WorkflowIndexRecord,
 ): Promise<void> {
 	for (const run of index.runs) {
-		if (run.status === "running")
+		if (hasActiveSchedulerWork(run))
 			await refreshRun(cwd, run.runId).catch((error) =>
 				recordSupervisorError(cwd, run.runId, error),
 			);
@@ -543,6 +665,16 @@ async function scheduleDag(
 			compiledFlow,
 		);
 		if (loopReconciled) return;
+		const foreachReconciled = reconcileForeachGeneratedRunRecords(
+			cwd,
+			run,
+			compiledFlow,
+		);
+		if (foreachReconciled) {
+			await writeJsonAtomic(compiledWorkflowPath(cwd, run.runId), compiledFlow);
+			await writeRunRecord(cwd, run);
+			return;
+		}
 		const dynamicReconciled = reconcileDynamicGeneratedRunRecords(
 			cwd,
 			run,
@@ -628,7 +760,7 @@ async function scheduleDag(
 			index,
 			options,
 		);
-		if (launched) running += 1;
+		if (launched && run.tasks[index]?.status === "running") running += 1;
 	}
 }
 
@@ -788,6 +920,17 @@ async function materializeForeachTask(
 
 	const placeholderSpecId = template.id;
 	const generatedSpecIds = generated.tasks.map((task) => task.id);
+	const hasDownstreamDependents = compiledFlow.tasks.some(
+		(task, taskIndex) =>
+			taskIndex !== index && (task.dependsOn ?? []).includes(placeholderSpecId),
+	);
+	if (generatedSpecIds.length === 0 && !hasDownstreamDependents) {
+		setTaskTerminal(templateRunTask, "completed", "foreach_empty", {
+			lastMessage: "foreach produced 0 item(s)",
+		});
+		await writeRunRecord(cwd, run);
+		return true;
+	}
 	compiledFlow.tasks.splice(index, 1, ...generated.tasks);
 	updateDownstreamDependencies(
 		compiledFlow,
@@ -880,12 +1023,15 @@ async function launchPendingTaskAt(
 		return false;
 	}
 
-	let launchTask = await prepareDagTask(cwd, run, compiledFlow, index);
-	if (task.outputRetry) {
-		launchTask = await prepareArtifactGraphRetryTask(cwd, task, launchTask);
-	}
-
+	let launchTask: CompiledWorkflow["tasks"][number] | undefined;
+	let prepareComplete = false;
 	try {
+		launchTask = await prepareDagTask(cwd, run, compiledFlow, index);
+		if (task.outputRetry) {
+			launchTask = await prepareArtifactGraphRetryTask(cwd, task, launchTask);
+		}
+		prepareComplete = true;
+
 		if (launchTask.kind === "support") {
 			return await executeSupportTask(cwd, run, task, launchTask);
 		}
@@ -913,10 +1059,11 @@ async function launchPendingTaskAt(
 		if (launch.kind === "fatal") throw new Error(launch.message);
 		return launch.kind === "launched";
 	} catch (error) {
-		const statusDetail =
-			launchTask.kind === "support"
+		const statusDetail = !prepareComplete
+			? "prepare_failed"
+			: launchTask?.kind === "support"
 				? "support_failed"
-				: launchTask.safety.requiresWorktree
+				: launchTask?.safety.requiresWorktree
 					? "worktree_failed"
 					: "launch_failed";
 		setTaskTerminal(task, "failed", statusDetail, {
@@ -1047,6 +1194,7 @@ async function executeDynamicControllerTask(
 			sources,
 			dynamic: compiledTask.dynamic,
 			dynamicUi: options.dynamicUi,
+			availableModels: options.availableModels,
 		});
 		await assertDynamicGeneratedTasksSettled({
 			cwd,
@@ -1056,6 +1204,7 @@ async function executeDynamicControllerTask(
 			controllerTask: task,
 			controllerCompiledTask: compiledTask,
 			dynamic: compiledTask.dynamic,
+			availableModels: options.availableModels,
 		});
 		await recordActiveRuntime();
 		const unrunBranchBlockers = await dynamicUnrunBranchBlockers(
@@ -1188,6 +1337,7 @@ async function runDynamicControllerWorker(input: {
 	sources: Record<string, unknown>;
 	dynamic: CompiledDynamicWorkflowTask;
 	dynamicUi?: DynamicWorkflowUi;
+	availableModels?: WorkflowModelInfo[];
 }): Promise<unknown> {
 	const resolved = await resolveWorkflowHelperRef(
 		input.dynamic.uses,
@@ -1810,65 +1960,6 @@ function requiredDynamicString(
 	return value.trim();
 }
 
-function optionalDynamicString(
-	value: unknown,
-	field: string,
-): string | undefined {
-	if (value === undefined) return undefined;
-	return requiredDynamicString(value, field);
-}
-
-function optionalDynamicStringArray(
-	value: unknown,
-	field: string,
-): string[] | undefined {
-	if (value === undefined) return undefined;
-	if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
-		throw new Error(`ctx.agent() ${field} must be an array of strings`);
-	}
-	return value.map((item) => item.trim()).filter(Boolean);
-}
-
-function isPlainDynamicRecord(
-	value: unknown,
-): value is Record<string, unknown> {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-function optionalDynamicPositiveInteger(
-	value: unknown,
-	field: string,
-): number | undefined {
-	if (value === undefined) return undefined;
-	if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
-		throw new Error(`ctx.agent() ${field} must be a positive integer`);
-	}
-	return value;
-}
-
-function requiredDynamicOutputProfile(
-	value: unknown,
-	field: string,
-	api: string,
-): DynamicOutputProfile {
-	const profile = requiredDynamicString(value, field, api);
-	if (!isDynamicOutputProfile(profile)) {
-		throw new Error(`${api} ${field} has an unsupported output profile`);
-	}
-	return profile;
-}
-
-function requiredDynamicNonNegativeInteger(
-	value: unknown,
-	field: string,
-	api: string,
-): number {
-	if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
-		throw new Error(`${api} ${field} must be a non-negative integer`);
-	}
-	return value;
-}
-
 function requiredDynamicPositiveInteger(
 	value: unknown,
 	field: string,
@@ -1880,17 +1971,6 @@ function requiredDynamicPositiveInteger(
 	return value;
 }
 
-function optionalDynamicStringField(value: unknown): string | undefined {
-	return typeof value === "string" && value.trim() ? value.trim() : undefined;
-}
-
-function optionalDynamicOutputProfile(
-	value: unknown,
-): DynamicOutputProfile | undefined {
-	if (value === undefined) return undefined;
-	return requiredDynamicOutputProfile(value, "outputProfile", "ctx.agent()");
-}
-
 async function currentDynamicBudgetRemaining(input: {
 	cwd: string;
 	run: WorkflowRunRecord;
@@ -2263,6 +2343,7 @@ async function assertDynamicGeneratedTasksSettled(input: {
 	controllerTask: WorkflowTaskRunRecord;
 	controllerCompiledTask: CompiledTask;
 	dynamic: CompiledDynamicWorkflowTask;
+	availableModels?: WorkflowModelInfo[];
 }): Promise<void> {
 	const state = await readOrRebuildDynamicState(input.cwd, input.run.runId);
 	const generatedTaskIds =
@@ -2289,6 +2370,7 @@ async function repairMissingDynamicGeneratedTask(
 		controllerTask: WorkflowTaskRunRecord;
 		controllerCompiledTask: CompiledTask;
 		dynamic: CompiledDynamicWorkflowTask;
+		availableModels?: WorkflowModelInfo[];
 	},
 	specId: string,
 ): Promise<WorkflowTaskRunRecord> {
@@ -2325,6 +2407,7 @@ async function repairMissingDynamicGeneratedTask(
 				branchId: optionalEventString(event.payload.branchId),
 				request,
 				dynamic: input.dynamic,
+				availableModels: input.availableModels,
 			});
 	assertDynamicGeneratedMetadataMatches(compiledTask, {
 		controllerSpecId: input.controllerTask.specId,
@@ -2374,6 +2457,7 @@ async function runDynamicAgentRequest(input: {
 	request: unknown;
 	generatedTaskIds: string[];
 	isSettled?: () => boolean;
+	availableModels?: WorkflowModelInfo[];
 }): Promise<unknown> {
 	await assertDynamicRuntimeBudgetAvailable({
 		cwd: input.cwd,
@@ -2459,6 +2543,7 @@ async function runDynamicAgentRequest(input: {
 					branchId: generationBranchId,
 					request: generationRequest,
 					dynamic: input.dynamic,
+					availableModels: input.availableModels,
 				});
 		assertDynamicGeneratedMetadataMatches(compiledTask, {
 			controllerSpecId: input.controllerTask.specId,