@agwab/pi-workflow 0.2.1 → 0.4.0

package/src/compiler.ts

@@ -3,6 +3,8 @@ import { dirname, resolve } from "node:path";
 
 import { loadAgentByName } from "./agents.js";
 import { DYNAMIC_OUTPUT_PROFILES } from "./dynamic-profiles.js";
+import { stringifyPromptJson } from "./prompt-json.js";
+import { compileRole } from "./roles.js";
 import {
 	classifyToolCapability,
 	effectiveToolClassification,
@@ -178,7 +180,13 @@ function lowerArtifactGraphFrom(from: ArtifactGraphStageSpec["from"]): unknown {
 		!Array.isArray(from) &&
 		typeof from.source === "string"
 	) {
-		return { stage: from.source, path: from.path };
+		return {
+			stage: from.source,
+			path: from.path,
+			...((from as { streaming?: unknown }).streaming !== undefined
+				? { streaming: (from as { streaming?: unknown }).streaming }
+				: {}),
+		};
 	}
 	return from;
 }
@@ -209,11 +217,26 @@ function appendWorkflowOutputInstructions(
 			: "Use schema `stage-control-v1` unless the workflow asks for a more specific control schema.",
 		"Put detailed prose, reasoning, and evidence discussion in <analysis> only.",
 		"Put structured evidence pointers in <refs> as a JSON array; use [] if none.",
+		...partialOutputInstructions(stage.output?.partial?.paths),
 	]
 		.filter(Boolean)
 		.join("\n\n");
 }
 
+function partialOutputInstructions(
+	paths: readonly string[] | undefined,
+): string[] {
+	if (!paths || paths.length === 0) return [];
+	return [
+		"# Workflow Partial Output Protocol (optional)",
+		`If a complete stable array item is ready before your final answer for one of these control paths (${paths.join(", ")}), you may emit a partial-control section before the final output:`,
+		'<partial-control>{"schema":"workflow-partial-output-v1","path":"$.items","items":[{"id":"stable-id","...":"..."}]}</partial-control>',
+		"Use the actual declared path, not the example path, and include only items that are final/stable enough to appear unchanged in your final <control> at that path.",
+		"Every partial item must be the exact JSON object that will appear in the final array and must include a stable non-empty string `id`; never revise or withdraw a published partial item.",
+		"If an item might change, do not publish it partially; wait for the final workflow output. The final answer must still include the normal <control>, <analysis>, and <refs> sections exactly once.",
+	];
+}
+
 function artifactGraphTaskMetadata(
 	stage: ArtifactGraphStageSpec,
 	specDir: string,
@@ -230,8 +253,12 @@ function artifactGraphTaskMetadata(
 				? resolve(specDir, controlSchema)
 				: undefined,
 			maxDigestChars: stage.output?.maxDigestChars,
+			partial: stage.output?.partial
+				? { paths: [...stage.output.partial.paths] }
+				: undefined,
 		},
 		requiredReads: stage.inputPolicy?.requiredReads ?? [],
+		artifactAccess: stage.inputPolicy?.artifactAccess ?? "enabled",
 		sourceProjection: stage.sourceProjection,
 	};
 }
@@ -663,15 +690,19 @@ async function compileArtifactGraphPlan(
 		return defaultAgent;
 	};
 	const roleEntries = Object.entries(spec.roles ?? {});
-	const roles = roleEntries.map(([name, role]: [string, any]) => ({
-		name,
-		fromAgent: role.fromAgent,
-		content: role.prompt ?? "",
-		maxChars: role.maxChars ?? 8000,
-		truncated: false,
-		includedSections: [],
-		excludedSections: [],
-	}));
+	const roles = await Promise.all(
+		roleEntries.map(async ([name, role]: [string, any]) => {
+			const sourceAgent = role.fromAgent
+				? await loadWorkflowAgent(
+						role.fromAgent,
+						options.cwd,
+						agentCache,
+						`$.roles.${name}.fromAgent`,
+					)
+				: undefined;
+			return compileRole(name, role, sourceAgent);
+		}),
+	);
 	const roleText = roles.length
 		? `# Role Context\n\n${roles.map((r) => `## Role: ${r.name}\n${r.content}`).join("\n\n")}`
 		: "";
@@ -681,7 +712,7 @@ async function compileArtifactGraphPlan(
 		typeof workflowInput === "object" &&
 		!Array.isArray(workflowInput) &&
 		Object.keys(workflowInput).length > 0
-			? `# Workflow Input\n\n${JSON.stringify(workflowInput, null, 2)}`
+			? `# Workflow Input\n\n${stringifyPromptJson(workflowInput)}`
 			: "";
 	const runtimeOverrides = options.runtimeOverrides;
 	const runtimeDefaults = options.runtimeDefaults;
@@ -831,15 +862,29 @@ async function compileArtifactGraphPlan(
 			/\$\{item\}/g,
 			"the relevant item from the dependency context",
 		);
-		const compiledPrompt = [
+		const instructionText = `# Instructions\n\n${normalizedPrompt}`;
+		const stageText = `# Workflow Stage\n\nstage=${stage.id}\ntype=${runtimeStageKind}`;
+		const taskText =
 			injectRuntimeTaskInPrompt && options.task
 				? `# Task\n\n${options.task}`
-				: undefined,
-			workflowInputText || undefined,
-			`# Workflow Stage\n\nstage=${stage.id}\ntype=${runtimeStageKind}`,
-			`# Instructions\n\n${normalizedPrompt}`,
-			roleText || undefined,
-		]
+				: undefined;
+		const compiledPrompt = (
+			runtimeStageKind === "foreach"
+				? [
+						taskText,
+						workflowInputText || undefined,
+						stageText,
+						roleText || undefined,
+						instructionText,
+					]
+				: [
+						taskText,
+						workflowInputText || undefined,
+						stageText,
+						instructionText,
+						roleText || undefined,
+					]
+		)
 			.filter(Boolean)
 			.join("\n\n");
 		const toolSelection = resolveToolSelection(

package/src/dynamic-generated-task-runtime.ts

@@ -7,6 +7,7 @@ import {
 } from "./dynamic-profiles.js";
 import { readOrRebuildDynamicState } from "./dynamic-state.js";
 import { sanitizeTaskId } from "./engine-run-graph.js";
+import { compactStrings } from "./strings.js";
 import { fromProjectPath, isTerminalTaskStatus, readJson } from "./store.js";
 import {
 	classifyToolCapability,
@@ -307,6 +308,7 @@ export async function buildDynamicGeneratedCompiledTask(input: {
 				maxDigestChars: DYNAMIC_OUTPUT_MAX_DIGEST_CHARS,
 			},
 			requiredReads: input.request.requiredReads,
+			artifactAccess: "enabled",
 			sourceProjection: dynamicInputSourceProjection(input.request.inputs),
 		},
 		dynamicGenerated: {
@@ -966,5 +968,5 @@ function dynamicOutputProfileInstructions(
 }
 
 function uniqueStrings(values: readonly string[]): string[] {
-	return [...new Set(values.filter((value) => value.trim().length > 0))];
+	return compactStrings(values, { trim: false, dropWhitespaceOnly: true });
 }

package/src/dynamic-profiles.ts

@@ -45,6 +45,6 @@ export function isTerminalDynamicOutputProfile(
 	return typeof value === "string" && TERMINAL_OUTPUT_PROFILE_SET.has(value);
 }
 
-export function dynamicOutputProfileValues(): string[] {
+export function dynamicOutputProfileValues(): DynamicOutputProfile[] {
 	return [...DYNAMIC_OUTPUT_PROFILES];
 }

package/src/engine-run-graph.ts

@@ -127,6 +127,151 @@ export function reconcileDynamicGeneratedRunRecords(
 	return changed;
 }
 
+export function reconcileForeachGeneratedRunRecords(
+	cwd: string,
+	run: WorkflowRunRecord,
+	compiledFlow: CompiledWorkflow,
+): boolean {
+	let changed = false;
+	const compiledSpecIds = new Set(
+		compiledFlow.tasks.map((task) => compiledTaskSpecId(task)),
+	);
+	const compiledTaskBySpecId = new Map(
+		compiledFlow.tasks.map((task) => [compiledTaskSpecId(task), task]),
+	);
+	const placeholderToGeneratedSpecIds = new Map<string, string[]>();
+	const streamingPlaceholderSpecIds = new Set<string>();
+
+	for (const compiledTask of compiledFlow.tasks) {
+		const specId = compiledTaskSpecId(compiledTask);
+		if (compiledTask.foreach && foreachStreamingEnabled(compiledTask)) {
+			streamingPlaceholderSpecIds.add(specId);
+		}
+		const placeholderSpecId = foreachGeneratedPlaceholderSpecId(
+			compiledTask,
+			compiledFlow,
+			specId,
+		);
+		if (!placeholderSpecId) continue;
+		if (
+			compiledTask.foreachGenerated?.placeholderSpecId !== placeholderSpecId
+		) {
+			compiledTask.foreachGenerated = { placeholderSpecId };
+			changed = true;
+		}
+		const generated =
+			placeholderToGeneratedSpecIds.get(placeholderSpecId) ?? [];
+		generated.push(specId);
+		placeholderToGeneratedSpecIds.set(placeholderSpecId, generated);
+	}
+
+	if (placeholderToGeneratedSpecIds.size === 0) return changed;
+
+	const filteredRunTasks: WorkflowTaskRunRecord[] = [];
+	const seenGeneratedSpecIds = new Set<string>();
+	for (const task of run.tasks) {
+		const generatedSpecIds = placeholderToGeneratedSpecIds.get(task.specId);
+		let placeholderSpecId = foreachGeneratedPlaceholderSpecId(
+			task,
+			compiledFlow,
+			task.specId,
+		);
+		if (generatedSpecIds && !placeholderSpecId) {
+			const compiledTask = compiledTaskBySpecId.get(task.specId);
+			if (
+				compiledTask?.foreach &&
+				streamingPlaceholderSpecIds.has(task.specId)
+			) {
+				filteredRunTasks.push(task);
+				continue;
+			}
+			if (generatedSpecIds.includes(task.specId)) {
+				placeholderSpecId = task.specId;
+				task.foreachGenerated = { placeholderSpecId };
+				changed = true;
+			} else {
+				changed = true;
+				continue;
+			}
+		}
+		if (placeholderSpecId && !compiledSpecIds.has(task.specId)) {
+			changed = true;
+			continue;
+		}
+		if (placeholderSpecId && seenGeneratedSpecIds.has(task.specId)) {
+			changed = true;
+			continue;
+		}
+		if (placeholderSpecId) {
+			seenGeneratedSpecIds.add(task.specId);
+			if (task.foreachGenerated?.placeholderSpecId !== placeholderSpecId) {
+				task.foreachGenerated = { placeholderSpecId };
+				changed = true;
+			}
+		}
+		filteredRunTasks.push(task);
+	}
+
+	const runTaskBySpecId = new Map<string, WorkflowTaskRunRecord>();
+	for (const task of filteredRunTasks) {
+		if (!runTaskBySpecId.has(task.specId))
+			runTaskBySpecId.set(task.specId, task);
+	}
+
+	const reordered: WorkflowTaskRunRecord[] = [];
+	const usedSpecIds = new Set<string>();
+	let nextIndex = nextTaskRecordIndex({ ...run, tasks: filteredRunTasks });
+	for (const compiledTask of compiledFlow.tasks) {
+		const specId = compiledTaskSpecId(compiledTask);
+		const existing = runTaskBySpecId.get(specId);
+		if (existing) {
+			const placeholderSpecId =
+				compiledTask.foreachGenerated?.placeholderSpecId;
+			if (
+				placeholderSpecId &&
+				existing.foreachGenerated?.placeholderSpecId !== placeholderSpecId
+			) {
+				existing.foreachGenerated = { placeholderSpecId };
+				changed = true;
+			}
+			reordered.push(existing);
+			usedSpecIds.add(specId);
+			continue;
+		}
+		if (!compiledTask.foreachGenerated) continue;
+		const created = createTaskRunRecord(
+			cwd,
+			run.runId,
+			compiledTask,
+			nextIndex,
+		);
+		nextIndex += 1;
+		reordered.push(created);
+		usedSpecIds.add(specId);
+		changed = true;
+	}
+
+	for (const task of filteredRunTasks) {
+		if (!usedSpecIds.has(task.specId)) reordered.push(task);
+	}
+
+	if (!sameTaskRecordOrder(run.tasks, reordered)) changed = true;
+	for (const task of reordered) {
+		if (!task.dependsOn) continue;
+		const replaced = replaceForeachGeneratedDependencies(
+			task.dependsOn,
+			placeholderToGeneratedSpecIds,
+			streamingPlaceholderSpecIds,
+		);
+		if (!sameStringList(task.dependsOn, replaced)) {
+			task.dependsOn = replaced;
+			changed = true;
+		}
+	}
+	if (changed) run.tasks = reordered;
+	return changed;
+}
+
 export function assertRunTaskPositionalAlignment(
 	run: WorkflowRunRecord,
 	compiledFlow: CompiledWorkflow,
@@ -209,6 +354,57 @@ export function compiledTaskSpecId(task: CompiledTask): string {
 	return typeof specId === "string" && specId.trim() !== "" ? specId : task.id;
 }
 
+function foreachGeneratedPlaceholderSpecId(
+	task: CompiledTask | WorkflowTaskRunRecord,
+	compiledFlow: CompiledWorkflow,
+	specId: string,
+): string | undefined {
+	const explicit = task.foreachGenerated?.placeholderSpecId;
+	if (typeof explicit === "string" && explicit.trim() !== "") return explicit;
+	if ((task as CompiledTask).foreach) return undefined;
+	if (task.kind !== "foreach" || !task.stageId) return undefined;
+	const placeholderSpecId = foreachPlaceholderSpecId(
+		compiledFlow,
+		task.stageId,
+	);
+	if (!placeholderSpecId || specId === placeholderSpecId) return undefined;
+	return placeholderSpecId;
+}
+
+function foreachPlaceholderSpecId(
+	compiledFlow: CompiledWorkflow,
+	stageId: string,
+): string | undefined {
+	const stage = ((compiledFlow as any).stages ?? []).find(
+		(candidate: any) => candidate?.id === stageId,
+	);
+	if (stage?.type !== "foreach") return undefined;
+	return `${stageId}.item`;
+}
+
+function replaceForeachGeneratedDependencies(
+	dependsOn: string[],
+	placeholderToGeneratedSpecIds: Map<string, string[]>,
+	keepPlaceholderSpecIds = new Set<string>(),
+): string[] {
+	const replaced: string[] = [];
+	for (const dep of dependsOn) {
+		const generatedSpecIds = placeholderToGeneratedSpecIds.get(dep);
+		if (generatedSpecIds) {
+			if (keepPlaceholderSpecIds.has(dep)) replaced.push(dep);
+			replaced.push(...generatedSpecIds);
+		} else replaced.push(dep);
+	}
+	return [...new Set(replaced)];
+}
+
+function sameStringList(left: string[], right: string[]): boolean {
+	return (
+		left.length === right.length &&
+		left.every((value, index) => value === right[index])
+	);
+}
+
 function isLoopGeneratedCompiledTask(
 	task: CompiledTask,
 	loopIds: Set<string>,
@@ -303,6 +499,29 @@ export function dependenciesReady(
 	if (deps.length === 0) return true;
 	const partial =
 		stageSourcePolicy(compiledFlow, compiledTask.stageId ?? "") === "partial";
+	if (foreachStreamingEnabled(compiledTask)) {
+		let completedDependencyReady = false;
+		let runningDependencyMayHavePartialItems = false;
+		let allKnownDependenciesTerminal = true;
+		for (const dep of deps) {
+			const status = bySpecId.get(dep)?.status;
+			if (status === "completed") {
+				completedDependencyReady = true;
+				continue;
+			}
+			if (status && isTerminalTaskStatus(status)) {
+				if (!partial) return false;
+				continue;
+			}
+			if (status === "running") runningDependencyMayHavePartialItems = true;
+			allKnownDependenciesTerminal = false;
+		}
+		return (
+			completedDependencyReady ||
+			runningDependencyMayHavePartialItems ||
+			allKnownDependenciesTerminal
+		);
+	}
 	return deps.every((dep) => {
 		const status = bySpecId.get(dep)?.status;
 		if (status === "completed") return true;
@@ -311,6 +530,22 @@ export function dependenciesReady(
 	});
 }
 
+export function foreachStreamingEnabled(compiledTask: CompiledTask): boolean {
+	const streaming = (compiledTask.foreach?.from as any)?.streaming;
+	return Boolean(
+		streaming &&
+			typeof streaming === "object" &&
+			(streaming as { enabled?: unknown }).enabled === true,
+	);
+}
+
+export function foreachStreamingMinChunk(compiledTask: CompiledTask): number {
+	const value = (compiledTask.foreach?.from as any)?.streaming?.minChunk;
+	return typeof value === "number" && Number.isFinite(value) && value > 0
+		? Math.floor(value)
+		: 1;
+}
+
 export function buildForeachGeneratedTasks(
 	template: CompiledTask,
 	runtimeTask: string | undefined,
@@ -330,15 +565,16 @@ export function buildForeachGeneratedTasks(
 		const itemText = formatForeachItem(item);
 		const instructions = template.foreach!.prompt.replace(
 			/\$\{item\}/g,
-			itemText,
+			escapeReplacementText(itemText),
 		);
 		const compiledPrompt = [
 			template.foreach!.injectRuntimeTask && runtimeTask
 				? `# Task\n\n${runtimeTask}`
 				: undefined,
-			`# Workflow Stage\n\nstage=${template.stageId}\ntype=foreach\nitem=${taskId}`,
-			`# Instructions\n\n${instructions}`,
+			`# Workflow Stage\n\nstage=${template.stageId}\ntype=foreach`,
 			template.foreach!.roleText || undefined,
+			`# Workflow Item\n\nitem=${taskId}`,
+			`# Instructions\n\n${instructions}`,
 		]
 			.filter(Boolean)
 			.join("\n\n");
@@ -352,6 +588,7 @@ export function buildForeachGeneratedTasks(
 			compiledPrompt,
 			dependsOn: [...(template.dependsOn ?? [])],
 			foreach: undefined,
+			foreachGenerated: { placeholderSpecId: template.id },
 		} as CompiledTask);
 	}
 	return { tasks };
@@ -382,6 +619,10 @@ function formatForeachItem(item: unknown): string {
 	return typeof item === "string" ? item : JSON.stringify(item);
 }
 
+function escapeReplacementText(value: string): string {
+	return value.replace(/\$/g, "$$$$");
+}
+
 export function sourceStageIdsForFrom(from: unknown): string[] {
 	if (Array.isArray(from))
 		return from.filter((item): item is string => typeof item === "string");
@@ -453,7 +694,8 @@ export function markDagDependentsSkipped(
 				return (
 					status === "failed" ||
 					status === "interrupted" ||
-					status === "skipped"
+					status === "skipped" ||
+					status === "blocked"
 				);
 			});
 			if (!failedDep) continue;