@agwab/pi-workflow 0.3.0 → 0.4.0

package/src/artifact-graph-schema.ts

@@ -1,6 +1,7 @@
 import { isAbsolute } from "node:path";
 
 import { DYNAMIC_OUTPUT_PROFILES } from "./dynamic-profiles.js";
+import { compactStrings } from "./strings.js";
 import {
 	APPROVAL_MODES,
 	FAST_MODES,
@@ -71,10 +72,16 @@ const OUTPUT_KEYS = new Set([
 	"analysis",
 	"refs",
 	"maxDigestChars",
+	"partial",
 ]);
+const OUTPUT_PARTIAL_KEYS = new Set(["paths"]);
 const REQUIRED_FLAG_KEYS = new Set(["required"]);
 const REFS_OUTPUT_KEYS = new Set(["required", "minItems"]);
-const INPUT_POLICY_KEYS = new Set(["requiredReads", "enforcement"]);
+const INPUT_POLICY_KEYS = new Set([
+	"requiredReads",
+	"enforcement",
+	"artifactAccess",
+]);
 const SOURCE_PROJECTION_KEYS = new Set(["include", "maxChars"]);
 const SUPPORT_KEYS = new Set(["uses", "options"]);
 const DYNAMIC_STAGE_FORBIDDEN_KEYS = new Set([
@@ -185,7 +192,8 @@ const UNTIL_KEYS = new Set([
 	"all",
 	"any",
 ]);
-const FOREACH_FROM_KEYS = new Set(["source", "path"]);
+const FOREACH_FROM_KEYS = new Set(["source", "path", "streaming"]);
+const FOREACH_STREAMING_KEYS = new Set(["enabled", "minChunk"]);
 const NORMAL_ARTIFACT_KINDS = new Set<WorkflowArtifactKind>([
 	"control",
 	"analysis",
@@ -296,9 +304,54 @@ function validateStageArray(
 	for (const [index, item] of value.entries()) {
 		validateStage(item, `${path}[${index}]`, ids, sourceIds, issues);
 	}
+	validateStreamingProducerDeclarations(value, path, issues);
 	validateStageDependencyGraph(value, path, issues);
 }
 
+function validateStreamingProducerDeclarations(
+	stages: readonly unknown[],
+	path: string,
+	issues: ValidationIssue[],
+): void {
+	const byId = new Map<string, Record<string, unknown>>();
+	for (const stage of stages) {
+		if (isRecord(stage) && typeof stage.id === "string") {
+			byId.set(stage.id, stage);
+		}
+	}
+	for (const [index, stage] of stages.entries()) {
+		if (!isRecord(stage) || !isRecord(stage.from)) continue;
+		const from = stage.from;
+		if (!isRecord(from.streaming) || from.streaming.enabled !== true) continue;
+		const source = typeof from.source === "string" ? from.source : undefined;
+		const controlPath = typeof from.path === "string" ? from.path : undefined;
+		if (!source || !controlPath) continue;
+		const sourceStage = byId.get(source);
+		const partialPaths = outputPartialPaths(sourceStage);
+		if (!partialPaths.includes(controlPath)) {
+			issues.push({
+				path: `${path}[${index}].from.streaming`,
+				message: `source stage "${source}" must declare output.partial.paths including "${controlPath}" to use streaming`,
+			});
+		}
+	}
+}
+
+function outputPartialPaths(
+	stage: Record<string, unknown> | undefined,
+): string[] {
+	const output = isRecord(stage?.output) ? stage.output : undefined;
+	const partial = isRecord(output?.partial) ? output.partial : undefined;
+	return Array.isArray(partial?.paths)
+		? compactStrings(partial.paths, {
+				trim: false,
+				unique: false,
+				dropEmpty: false,
+				dropWhitespaceOnly: false,
+			})
+		: [];
+}
+
 function validateStageDependencyGraph(
 	stages: readonly unknown[],
 	path: string,
@@ -372,7 +425,12 @@ function extractDependencyRefs(value: unknown): string[] {
 	if (value === undefined) return [];
 	if (typeof value === "string") return [value];
 	if (Array.isArray(value))
-		return value.filter((item): item is string => typeof item === "string");
+		return compactStrings(value, {
+			trim: false,
+			unique: false,
+			dropEmpty: false,
+			dropWhitespaceOnly: false,
+		});
 	if (isRecord(value) && typeof value.source === "string")
 		return [value.source];
 	return [];
@@ -517,6 +575,7 @@ function validateStage(
 		`${path}.inputPolicy`,
 		sourceIds,
 		issues,
+		stage.sourceProjection,
 	);
 	validateOutput(stage.output, `${path}.output`, issues);
 	validateSupportStage(stage, type, path, issues);
@@ -611,6 +670,23 @@ function validateControlPathRef(
 			message: "must be a control JSONPath starting with $.",
 		});
 	}
+	if (value.streaming !== undefined) {
+		validateForeachStreaming(value.streaming, `${path}.streaming`, issues);
+	}
+}
+
+function validateForeachStreaming(
+	value: unknown,
+	path: string,
+	issues: ValidationIssue[],
+): void {
+	const streaming = recordAt(value, path, issues);
+	if (!streaming) return;
+	rejectUnknownKeys(streaming, FOREACH_STREAMING_KEYS, path, issues);
+	if (streaming.enabled !== true) {
+		issues.push({ path: `${path}.enabled`, message: "must be true" });
+	}
+	optionalPositiveInteger(streaming.minChunk, `${path}.minChunk`, issues);
 }
 
 function validateKnownStageRef(
@@ -649,6 +725,42 @@ function validateOutput(
 	);
 	validateRequiredFlagObject(output.analysis, `${path}.analysis`, issues);
 	validateRefsOutputObject(output.refs, `${path}.refs`, issues);
+	validatePartialOutput(output.partial, `${path}.partial`, issues);
+}
+
+function validatePartialOutput(
+	value: unknown,
+	path: string,
+	issues: ValidationIssue[],
+): void {
+	if (value === undefined) return;
+	const partial = recordAt(value, path, issues);
+	if (!partial) return;
+	rejectUnknownKeys(partial, OUTPUT_PARTIAL_KEYS, path, issues);
+	if (!Array.isArray(partial.paths)) {
+		issues.push({ path: `${path}.paths`, message: "must be an array" });
+		return;
+	}
+	if (partial.paths.length === 0) {
+		issues.push({ path: `${path}.paths`, message: "must not be empty" });
+	}
+	const seen = new Set<string>();
+	for (const [index, item] of partial.paths.entries()) {
+		const itemPath = `${path}.paths[${index}]`;
+		if (typeof item !== "string" || item.trim() === "") {
+			issues.push({ path: itemPath, message: "must be a non-empty string" });
+			continue;
+		}
+		if (!item.startsWith("$.")) {
+			issues.push({
+				path: itemPath,
+				message: "must be a control JSONPath starting with $.",
+			});
+		}
+		if (seen.has(item))
+			issues.push({ path: itemPath, message: `duplicate value "${item}"` });
+		seen.add(item);
+	}
 }
 
 function validateControlSchemaRef(
@@ -694,6 +806,7 @@ function validateInputPolicy(
 	path: string,
 	siblingIds: ReadonlySet<string>,
 	issues: ValidationIssue[],
+	sourceProjection: unknown,
 ): void {
 	if (value === undefined) return;
 	const policy = recordAt(value, path, issues);
@@ -708,6 +821,33 @@ function validateInputPolicy(
 	if (policy.enforcement !== undefined && policy.enforcement !== "fail") {
 		issues.push({ path: `${path}.enforcement`, message: 'must be "fail"' });
 	}
+	if (
+		policy.artifactAccess !== undefined &&
+		policy.artifactAccess !== "enabled" &&
+		policy.artifactAccess !== "none"
+	) {
+		issues.push({
+			path: `${path}.artifactAccess`,
+			message: 'must be "enabled" or "none"',
+		});
+	}
+	if (policy.artifactAccess === "none") {
+		if (
+			Array.isArray(policy.requiredReads) &&
+			policy.requiredReads.length > 0
+		) {
+			issues.push({
+				path: `${path}.requiredReads`,
+				message: 'must be empty when artifactAccess is "none"',
+			});
+		}
+		if (sourceProjection !== undefined) {
+			issues.push({
+				path: `${path}.artifactAccess`,
+				message: 'cannot be "none" when sourceProjection is declared',
+			});
+		}
+	}
 }
 
 function validateRequiredReads(

package/src/cli.mjs

@@ -53,6 +53,7 @@ const tasks = Array.isArray(run.tasks) ? run.tasks : [];
 const selected = options.has("--failures")
   ? tasks.filter((task) => ["failed", "blocked", "interrupted"].includes(task.status))
   : tasks;
+const reliability = summarizeReliability(tasks);
 
 const lines = [
   `runId: ${run.runId}`,
@@ -60,6 +61,8 @@ const lines = [
   `type: ${run.type}`,
   `status: ${run.status}`,
   `tasks: ${tasks.length}`,
+  `completion: ${reliability.health}`,
+  `retries: output=${reliability.outputRetries}, launch=${reliability.launchRetries}, resumes=${reliability.resumeEvents}, contextLimitFailures=${reliability.contextLimitFailures}`,
 ];
 
 for (const task of selected) {
@@ -184,3 +187,52 @@ async function readJson(path) {
 function indent(text, prefix) {
   return text.split(/\r?\n/).map((line) => `${prefix}${line}`).join("\n");
 }
+
+function summarizeReliability(tasks) {
+  let outputRetries = 0;
+  let launchRetries = 0;
+  let resumeEvents = 0;
+  let contextLimitFailures = 0;
+  for (const task of tasks) {
+    outputRetries += positiveCount(task.outputRetry?.attempts);
+    launchRetries += positiveCount(task.launchRetry?.attempts);
+    if (hasContextLimitFailure(task)) contextLimitFailures += 1;
+    for (const event of Array.isArray(task.resumeEvents) ? task.resumeEvents : []) {
+      resumeEvents += 1;
+      outputRetries += positiveCount(event.outputRetryAttempts);
+      launchRetries += positiveCount(event.launchRetryAttempts);
+      if (hasContextLimitFailure(event)) contextLimitFailures += 1;
+    }
+  }
+  const allCompleted = tasks.length > 0 && tasks.every((task) => task.status === "completed");
+  const repairEvents = outputRetries + launchRetries + resumeEvents;
+  const health = !allCompleted
+    ? "incomplete"
+    : repairEvents === 0 && contextLimitFailures === 0
+      ? "clean"
+      : "repaired";
+  return { health, outputRetries, launchRetries, resumeEvents, contextLimitFailures };
+}
+
+function positiveCount(value) {
+  return Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0;
+}
+
+function hasContextLimitFailure(value) {
+  return [
+    value?.statusDetail,
+    value?.fromStatusDetail,
+    value?.lastMessage,
+    value?.outputRetry?.reason,
+    value?.outputRetry?.message,
+    value?.launchRetry?.reason,
+    value?.launchRetry?.message,
+    value?.outputRetryReason,
+    value?.launchRetryReason,
+  ].some(isContextLimitText);
+}
+
+function isContextLimitText(value) {
+  const text = String(value ?? "").toLowerCase();
+  return text.includes("context_or_request_too_large") || /context (window|length)|maximum context|request too large|token limit/.test(text);
+}

package/src/compiler.ts

@@ -3,6 +3,7 @@ import { dirname, resolve } from "node:path";
 
 import { loadAgentByName } from "./agents.js";
 import { DYNAMIC_OUTPUT_PROFILES } from "./dynamic-profiles.js";
+import { stringifyPromptJson } from "./prompt-json.js";
 import { compileRole } from "./roles.js";
 import {
 	classifyToolCapability,
@@ -179,7 +180,13 @@ function lowerArtifactGraphFrom(from: ArtifactGraphStageSpec["from"]): unknown {
 		!Array.isArray(from) &&
 		typeof from.source === "string"
 	) {
-		return { stage: from.source, path: from.path };
+		return {
+			stage: from.source,
+			path: from.path,
+			...((from as { streaming?: unknown }).streaming !== undefined
+				? { streaming: (from as { streaming?: unknown }).streaming }
+				: {}),
+		};
 	}
 	return from;
 }
@@ -210,11 +217,26 @@ function appendWorkflowOutputInstructions(
 			: "Use schema `stage-control-v1` unless the workflow asks for a more specific control schema.",
 		"Put detailed prose, reasoning, and evidence discussion in <analysis> only.",
 		"Put structured evidence pointers in <refs> as a JSON array; use [] if none.",
+		...partialOutputInstructions(stage.output?.partial?.paths),
 	]
 		.filter(Boolean)
 		.join("\n\n");
 }
 
+function partialOutputInstructions(
+	paths: readonly string[] | undefined,
+): string[] {
+	if (!paths || paths.length === 0) return [];
+	return [
+		"# Workflow Partial Output Protocol (optional)",
+		`If a complete stable array item is ready before your final answer for one of these control paths (${paths.join(", ")}), you may emit a partial-control section before the final output:`,
+		'<partial-control>{"schema":"workflow-partial-output-v1","path":"$.items","items":[{"id":"stable-id","...":"..."}]}</partial-control>',
+		"Use the actual declared path, not the example path, and include only items that are final/stable enough to appear unchanged in your final <control> at that path.",
+		"Every partial item must be the exact JSON object that will appear in the final array and must include a stable non-empty string `id`; never revise or withdraw a published partial item.",
+		"If an item might change, do not publish it partially; wait for the final workflow output. The final answer must still include the normal <control>, <analysis>, and <refs> sections exactly once.",
+	];
+}
+
 function artifactGraphTaskMetadata(
 	stage: ArtifactGraphStageSpec,
 	specDir: string,
@@ -231,8 +253,12 @@ function artifactGraphTaskMetadata(
 				? resolve(specDir, controlSchema)
 				: undefined,
 			maxDigestChars: stage.output?.maxDigestChars,
+			partial: stage.output?.partial
+				? { paths: [...stage.output.partial.paths] }
+				: undefined,
 		},
 		requiredReads: stage.inputPolicy?.requiredReads ?? [],
+		artifactAccess: stage.inputPolicy?.artifactAccess ?? "enabled",
 		sourceProjection: stage.sourceProjection,
 	};
 }
@@ -686,7 +712,7 @@ async function compileArtifactGraphPlan(
 		typeof workflowInput === "object" &&
 		!Array.isArray(workflowInput) &&
 		Object.keys(workflowInput).length > 0
-			? `# Workflow Input\n\n${JSON.stringify(workflowInput, null, 2)}`
+			? `# Workflow Input\n\n${stringifyPromptJson(workflowInput)}`
 			: "";
 	const runtimeOverrides = options.runtimeOverrides;
 	const runtimeDefaults = options.runtimeDefaults;
@@ -836,15 +862,29 @@ async function compileArtifactGraphPlan(
 			/\$\{item\}/g,
 			"the relevant item from the dependency context",
 		);
-		const compiledPrompt = [
+		const instructionText = `# Instructions\n\n${normalizedPrompt}`;
+		const stageText = `# Workflow Stage\n\nstage=${stage.id}\ntype=${runtimeStageKind}`;
+		const taskText =
 			injectRuntimeTaskInPrompt && options.task
 				? `# Task\n\n${options.task}`
-				: undefined,
-			workflowInputText || undefined,
-			`# Workflow Stage\n\nstage=${stage.id}\ntype=${runtimeStageKind}`,
-			`# Instructions\n\n${normalizedPrompt}`,
-			roleText || undefined,
-		]
+				: undefined;
+		const compiledPrompt = (
+			runtimeStageKind === "foreach"
+				? [
+						taskText,
+						workflowInputText || undefined,
+						stageText,
+						roleText || undefined,
+						instructionText,
+					]
+				: [
+						taskText,
+						workflowInputText || undefined,
+						stageText,
+						instructionText,
+						roleText || undefined,
+					]
+		)
 			.filter(Boolean)
 			.join("\n\n");
 		const toolSelection = resolveToolSelection(

package/src/dynamic-decision.ts

@@ -283,6 +283,17 @@ export function validateDynamicDecision(
 	};
 }
 
+export function assertValidDynamicDecision(
+	value: unknown,
+	context: DynamicDecisionValidationContext = {},
+): NormalizedDynamicDecision {
+	const result = validateDynamicDecision(value, context);
+	if (!result.ok || !result.decision) {
+		throw new Error(`invalid dynamic decision: ${result.errors.join("; ")}`);
+	}
+	return result.decision;
+}
+
 export function hashDynamicDecision(value: unknown): string {
 	return createHash("sha256")
 		.update(stableStringify(toJsonNormalizedValue(value)))

package/src/dynamic-generated-task-runtime.ts

@@ -7,6 +7,7 @@ import {
 } from "./dynamic-profiles.js";
 import { readOrRebuildDynamicState } from "./dynamic-state.js";
 import { sanitizeTaskId } from "./engine-run-graph.js";
+import { compactStrings } from "./strings.js";
 import { fromProjectPath, isTerminalTaskStatus, readJson } from "./store.js";
 import {
 	classifyToolCapability,
@@ -307,6 +308,7 @@ export async function buildDynamicGeneratedCompiledTask(input: {
 				maxDigestChars: DYNAMIC_OUTPUT_MAX_DIGEST_CHARS,
 			},
 			requiredReads: input.request.requiredReads,
+			artifactAccess: "enabled",
 			sourceProjection: dynamicInputSourceProjection(input.request.inputs),
 		},
 		dynamicGenerated: {
@@ -966,5 +968,5 @@ function dynamicOutputProfileInstructions(
 }
 
 function uniqueStrings(values: readonly string[]): string[] {
-	return [...new Set(values.filter((value) => value.trim().length > 0))];
+	return compactStrings(values, { trim: false, dropWhitespaceOnly: true });
 }

package/src/dynamic-profiles.ts

@@ -44,3 +44,7 @@ export function isTerminalDynamicOutputProfile(
 ): value is (typeof DYNAMIC_TERMINAL_OUTPUT_PROFILES)[number] {
 	return typeof value === "string" && TERMINAL_OUTPUT_PROFILE_SET.has(value);
 }
+
+export function dynamicOutputProfileValues(): DynamicOutputProfile[] {
+	return [...DYNAMIC_OUTPUT_PROFILES];
+}

package/src/engine-run-graph.ts

@@ -136,10 +136,17 @@ export function reconcileForeachGeneratedRunRecords(
 	const compiledSpecIds = new Set(
 		compiledFlow.tasks.map((task) => compiledTaskSpecId(task)),
 	);
+	const compiledTaskBySpecId = new Map(
+		compiledFlow.tasks.map((task) => [compiledTaskSpecId(task), task]),
+	);
 	const placeholderToGeneratedSpecIds = new Map<string, string[]>();
+	const streamingPlaceholderSpecIds = new Set<string>();
 
 	for (const compiledTask of compiledFlow.tasks) {
 		const specId = compiledTaskSpecId(compiledTask);
+		if (compiledTask.foreach && foreachStreamingEnabled(compiledTask)) {
+			streamingPlaceholderSpecIds.add(specId);
+		}
 		const placeholderSpecId = foreachGeneratedPlaceholderSpecId(
 			compiledTask,
 			compiledFlow,
@@ -170,6 +177,14 @@ export function reconcileForeachGeneratedRunRecords(
 			task.specId,
 		);
 		if (generatedSpecIds && !placeholderSpecId) {
+			const compiledTask = compiledTaskBySpecId.get(task.specId);
+			if (
+				compiledTask?.foreach &&
+				streamingPlaceholderSpecIds.has(task.specId)
+			) {
+				filteredRunTasks.push(task);
+				continue;
+			}
 			if (generatedSpecIds.includes(task.specId)) {
 				placeholderSpecId = task.specId;
 				task.foreachGenerated = { placeholderSpecId };
@@ -246,6 +261,7 @@ export function reconcileForeachGeneratedRunRecords(
 		const replaced = replaceForeachGeneratedDependencies(
 			task.dependsOn,
 			placeholderToGeneratedSpecIds,
+			streamingPlaceholderSpecIds,
 		);
 		if (!sameStringList(task.dependsOn, replaced)) {
 			task.dependsOn = replaced;
@@ -369,12 +385,15 @@ function foreachPlaceholderSpecId(
 function replaceForeachGeneratedDependencies(
 	dependsOn: string[],
 	placeholderToGeneratedSpecIds: Map<string, string[]>,
+	keepPlaceholderSpecIds = new Set<string>(),
 ): string[] {
 	const replaced: string[] = [];
 	for (const dep of dependsOn) {
 		const generatedSpecIds = placeholderToGeneratedSpecIds.get(dep);
-		if (generatedSpecIds) replaced.push(...generatedSpecIds);
-		else replaced.push(dep);
+		if (generatedSpecIds) {
+			if (keepPlaceholderSpecIds.has(dep)) replaced.push(dep);
+			replaced.push(...generatedSpecIds);
+		} else replaced.push(dep);
 	}
 	return [...new Set(replaced)];
 }
@@ -480,6 +499,29 @@ export function dependenciesReady(
 	if (deps.length === 0) return true;
 	const partial =
 		stageSourcePolicy(compiledFlow, compiledTask.stageId ?? "") === "partial";
+	if (foreachStreamingEnabled(compiledTask)) {
+		let completedDependencyReady = false;
+		let runningDependencyMayHavePartialItems = false;
+		let allKnownDependenciesTerminal = true;
+		for (const dep of deps) {
+			const status = bySpecId.get(dep)?.status;
+			if (status === "completed") {
+				completedDependencyReady = true;
+				continue;
+			}
+			if (status && isTerminalTaskStatus(status)) {
+				if (!partial) return false;
+				continue;
+			}
+			if (status === "running") runningDependencyMayHavePartialItems = true;
+			allKnownDependenciesTerminal = false;
+		}
+		return (
+			completedDependencyReady ||
+			runningDependencyMayHavePartialItems ||
+			allKnownDependenciesTerminal
+		);
+	}
 	return deps.every((dep) => {
 		const status = bySpecId.get(dep)?.status;
 		if (status === "completed") return true;
@@ -488,6 +530,22 @@ export function dependenciesReady(
 	});
 }
 
+export function foreachStreamingEnabled(compiledTask: CompiledTask): boolean {
+	const streaming = (compiledTask.foreach?.from as any)?.streaming;
+	return Boolean(
+		streaming &&
+			typeof streaming === "object" &&
+			(streaming as { enabled?: unknown }).enabled === true,
+	);
+}
+
+export function foreachStreamingMinChunk(compiledTask: CompiledTask): number {
+	const value = (compiledTask.foreach?.from as any)?.streaming?.minChunk;
+	return typeof value === "number" && Number.isFinite(value) && value > 0
+		? Math.floor(value)
+		: 1;
+}
+
 export function buildForeachGeneratedTasks(
 	template: CompiledTask,
 	runtimeTask: string | undefined,
@@ -513,9 +571,10 @@ export function buildForeachGeneratedTasks(
 			template.foreach!.injectRuntimeTask && runtimeTask
 				? `# Task\n\n${runtimeTask}`
 				: undefined,
-			`# Workflow Stage\n\nstage=${template.stageId}\ntype=foreach\nitem=${taskId}`,
-			`# Instructions\n\n${instructions}`,
+			`# Workflow Stage\n\nstage=${template.stageId}\ntype=foreach`,
 			template.foreach!.roleText || undefined,
+			`# Workflow Item\n\nitem=${taskId}`,
+			`# Instructions\n\n${instructions}`,
 		]
 			.filter(Boolean)
 			.join("\n\n");