@agwab/pi-workflow 0.2.1 → 0.3.0

package/node_modules/@agwab/pi-subagent/src/runners/headless-model.ts

@@ -79,6 +79,13 @@ export interface PiJsonParseResult {
 	metadata: Partial<ResultMetadata>;
 }
 
+export interface ContextLengthResolution {
+	rawContextLengthExceeded: boolean;
+	contextLengthExceeded: boolean;
+	contextOverflowRecovered: boolean;
+	recoveredStreamErrors: string[];
+}
+
 const CONTEXT_LENGTH_ERROR_PATTERN =
 	/\bcontext[_ -]?length[_ -]?exceeded\b|\bcontext[_ -]?window[_ -]?(?:exceeded|overflow|exhausted)\b|\b(?:maximum|max)[_ -]?context[_ -]?length\b|\btoo many tokens\b|\b(?:prompt|input|request)[^\n]{0,80}\btoo large\b|\bcontext_length_exceeded\b/i;
 
@@ -94,6 +101,32 @@ export function detectContextLengthExceeded(signals: {
 	return CONTEXT_LENGTH_ERROR_PATTERN.test(text);
 }
 
+export function resolveContextLengthState(
+	parsed: PiJsonParseResult,
+	rawContextLengthExceeded: boolean,
+): ContextLengthResolution {
+	const contextOverflowRecovered =
+		rawContextLengthExceeded && finalAssistantSucceeded(parsed);
+	return {
+		rawContextLengthExceeded,
+		contextLengthExceeded:
+			rawContextLengthExceeded && !contextOverflowRecovered,
+		contextOverflowRecovered,
+		recoveredStreamErrors: contextOverflowRecovered
+			? parsed.errors.filter((error) =>
+					detectContextLengthExceeded({ errors: [error] }),
+				)
+			: [],
+	};
+}
+
+function finalAssistantSucceeded(parsed: PiJsonParseResult): boolean {
+	return (
+		parsed.finalAssistantText.length > 0 &&
+		parsed.metadata.stopReason !== "error"
+	);
+}
+
 function normalizeTimeoutMs(timeoutMs: number | undefined): number | undefined {
 	if (timeoutMs === undefined) return undefined;
 	if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
@@ -390,18 +423,29 @@ export function resolvePiJsonOutcome(
 
 export function resultMetadataFromParse(
 	parsed: PiJsonParseResult,
-	contextLengthExceeded: boolean,
+	contextLength: ContextLengthResolution,
 	outcome: ProcessOutcome,
 ): Partial<ResultMetadata> {
 	return {
 		...parsed.metadata,
-		contextLengthExceeded,
+		contextLengthExceeded: contextLength.contextLengthExceeded,
+		...(contextLength.contextOverflowRecovered
+			? { contextOverflowRecovered: true }
+			: {}),
 		...(parsed.errors.length === 0
 			? {}
 			: { streamErrors: parsed.errors.slice(0, MAX_METADATA_ERRORS) }),
 		...(outcome.status === "completed" && parsed.errors.length > 0
 			? { nonFatalStreamErrors: parsed.errors.slice(0, MAX_METADATA_ERRORS) }
 			: {}),
+		...(contextLength.recoveredStreamErrors.length === 0
+			? {}
+			: {
+					recoveredStreamErrors: contextLength.recoveredStreamErrors.slice(
+						0,
+						MAX_METADATA_ERRORS,
+					),
+				}),
 		...(parsed.parseErrors.length === 0
 			? {}
 			: { parseErrors: parsed.parseErrors.slice(0, MAX_METADATA_ERRORS) }),
@@ -774,14 +818,18 @@ export async function runHeadlessModel(
 		stderrText,
 		stderrContextLengthExceeded,
 	} = processResult;
-	const contextLengthExceeded =
+	const rawContextLengthExceeded =
 		stderrContextLengthExceeded ||
 		detectContextLengthExceeded({ stderrText, errors: parsed.errors });
+	const contextLength = resolveContextLengthState(
+		parsed,
+		rawContextLengthExceeded,
+	);
 
 	const outcome = resolvePiJsonOutcome(
 		processOutcome,
 		parsed,
-		contextLengthExceeded,
+		contextLength.contextLengthExceeded,
 	);
 
 	const completedAt = new Date();
@@ -811,7 +859,7 @@ export async function runHeadlessModel(
 		artifacts,
 		correlationId: options.correlationId,
 		metadata: {
-			...resultMetadataFromParse(parsed, contextLengthExceeded, outcome),
+			...resultMetadataFromParse(parsed, contextLength, outcome),
 			...sessionMetadata,
 			...(options.parentSessionId === undefined
 				? {}

package/node_modules/@agwab/pi-subagent/src/runners/tmux.ts

@@ -20,6 +20,7 @@ import {
 	detectContextLengthExceeded,
 	parsePiJsonFile,
 	parsePiJsonLines,
+	resolveContextLengthState,
 	resolvePiJsonOutcome,
 	resultMetadataFromParse,
 	resultSessionMetadata,
@@ -157,9 +158,7 @@ function workerScript(
 	return `import { spawn } from "node:child_process";\nimport { appendFileSync, closeSync, openSync, writeFileSync } from "node:fs";\nconst argv = ${JSON.stringify(argv)};\nconst cwd = ${JSON.stringify(cwd)};\nconst eventPath = ${JSON.stringify(eventPath)};\nconst stderrPath = ${JSON.stringify(stderrPath)};\nconst metaPath = ${JSON.stringify(metaPath)};\nconst messageUpdatePattern = /"type"\\s*:\\s*"message_update"/;\nconst maxStdoutLogLineChars = 64 * 1024 * 1024;\ncloseSync(openSync(eventPath, "w"));\ncloseSync(openSync(stderrPath, "w"));\nlet settled = false;\nlet stdoutBuffer = "";\nlet discardingOversizedLine = false;\nlet omittedMessageUpdates = 0;\nlet omittedMessageUpdateBytes = 0;\nlet omittedOversizedLines = 0;\nlet omittedOversizedBytes = 0;\nfunction writeStdoutLine(line) {\n  if (messageUpdatePattern.test(line)) {\n    omittedMessageUpdates += 1;\n    omittedMessageUpdateBytes += Buffer.byteLength(line, "utf8");\n    return;\n  }\n  appendFileSync(eventPath, line);\n  process.stdout.write(line);\n}\nfunction handleStdoutChunk(chunk) {\n  let text = chunk.toString("utf8");\n  while (text.length > 0) {\n    if (discardingOversizedLine) {\n      const newline = text.indexOf("\\n");\n      omittedOversizedBytes += Buffer.byteLength(newline < 0 ? text : text.slice(0, newline + 1), "utf8");\n      if (newline < 0) return;\n      discardingOversizedLine = false;\n      text = text.slice(newline + 1);\n      continue;\n    }\n    const newline = text.indexOf("\\n");\n    const segment = newline < 0 ? text : text.slice(0, newline + 1);\n    stdoutBuffer += segment;\n    text = newline < 0 ? "" : text.slice(newline + 1);\n    if (stdoutBuffer.length > maxStdoutLogLineChars) {\n      omittedOversizedLines += 1;\n      omittedOversizedBytes += Buffer.byteLength(stdoutBuffer, "utf8");\n      stdoutBuffer = "";\n      discardingOversizedLine = newline < 0;\n      continue;\n    }\n    if (newline >= 0) {\n      writeStdoutLine(stdoutBuffer);\n      stdoutBuffer = "";\n    }\n  }\n}\nfunction finishStdoutFilter() {\n  if (!discardingOversizedLine && stdoutBuffer.length > 0) writeStdoutLine(stdoutBuffer);\n  stdoutBuffer = "";\n  if (omittedMessageUpdates > 0 || omittedOversizedLines > 0) {\n    appendFileSync(eventPath, JSON.stringify({ type: "pi-subagent.stdout_filter", omitted: { messageUpdateEvents: omittedMessageUpdates, messageUpdateBytes: omittedMessageUpdateBytes, oversizedLines: omittedOversizedLines, oversizedBytes: omittedOversizedBytes }, reason: "cumulative message_update snapshots are omitted from durable stdout artifacts; final assistant text is stored in output.log" }) + "\\n");\n  }\n}\nfunction writeMeta(meta) {\n  if (settled) return;\n  settled = true;\n  finishStdoutFilter();\n  writeFileSync(metaPath, JSON.stringify(meta, null, 2) + "\\n");\n}\nconst env = { ...process.env };\ndelete env.TMUX;\nconst child = spawn(argv[0], argv.slice(1), { cwd, shell: false, stdio: ["ignore", "pipe", "pipe"], env });\nchild.stdout?.on("data", handleStdoutChunk);\nchild.stderr?.on("data", (chunk) => { appendFileSync(stderrPath, chunk); process.stderr.write(chunk); });\nchild.on("error", () => { writeMeta({ status: "failed", failureKind: "spawn", exitCode: null, signal: null }); });\nchild.on("close", (exitCode, signal) => {\n  const failureKind = exitCode === 0 ? null : "exit";\n  writeMeta({ status: failureKind === null ? "completed" : "failed", failureKind, exitCode, signal });\n});\n`;
 }
 
-async function runTmuxProcess(
-	options: RunTmuxProcessOptions,
-): Promise<{
+async function runTmuxProcess(options: RunTmuxProcessOptions): Promise<{
 	result: TmuxRunResult | null;
 	store: Awaited<ReturnType<typeof createAttemptArtifactStore>>;
 	cwd: string;
@@ -439,11 +438,19 @@ export async function runTmuxModel(
 		parsePiJsonLines(""),
 	);
 	await unlink(result.eventPath).catch(() => undefined);
-	const contextLengthExceeded = detectContextLengthExceeded({
+	const rawContextLengthExceeded = detectContextLengthExceeded({
 		stderrText,
 		errors: parsed.errors,
 	});
-	const meta = resolvePiJsonOutcome(result.meta, parsed, contextLengthExceeded);
+	const contextLength = resolveContextLengthState(
+		parsed,
+		rawContextLengthExceeded,
+	);
+	const meta = resolvePiJsonOutcome(
+		result.meta,
+		parsed,
+		contextLength.contextLengthExceeded,
+	);
 
 	const outputRef = await store.writeTextArtifact(
 		"output",
@@ -464,7 +471,7 @@ export async function runTmuxModel(
 		tmux: result.tmux,
 		correlationId: options.correlationId,
 		metadata: {
-			...resultMetadataFromParse(parsed, contextLengthExceeded, meta),
+			...resultMetadataFromParse(parsed, contextLength, meta),
 			...sessionMetadata,
 			...(options.parentSessionId === undefined
 				? {}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agwab/pi-workflow",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Workflow orchestration for Pi subagents.",
   "private": false,
   "type": "module",
@@ -76,7 +76,7 @@
     "node": ">=22.19.0"
   },
   "dependencies": {
-    "@agwab/pi-subagent": "^0.3.6",
+    "@agwab/pi-subagent": "^0.4.0",
     "pi-web-access": "^0.10.7",
     "typebox": "^1.1.39"
   },

package/src/compiler.ts

@@ -3,6 +3,7 @@ import { dirname, resolve } from "node:path";
 
 import { loadAgentByName } from "./agents.js";
 import { DYNAMIC_OUTPUT_PROFILES } from "./dynamic-profiles.js";
+import { compileRole } from "./roles.js";
 import {
 	classifyToolCapability,
 	effectiveToolClassification,
@@ -663,15 +664,19 @@ async function compileArtifactGraphPlan(
 		return defaultAgent;
 	};
 	const roleEntries = Object.entries(spec.roles ?? {});
-	const roles = roleEntries.map(([name, role]: [string, any]) => ({
-		name,
-		fromAgent: role.fromAgent,
-		content: role.prompt ?? "",
-		maxChars: role.maxChars ?? 8000,
-		truncated: false,
-		includedSections: [],
-		excludedSections: [],
-	}));
+	const roles = await Promise.all(
+		roleEntries.map(async ([name, role]: [string, any]) => {
+			const sourceAgent = role.fromAgent
+				? await loadWorkflowAgent(
+						role.fromAgent,
+						options.cwd,
+						agentCache,
+						`$.roles.${name}.fromAgent`,
+					)
+				: undefined;
+			return compileRole(name, role, sourceAgent);
+		}),
+	);
 	const roleText = roles.length
 		? `# Role Context\n\n${roles.map((r) => `## Role: ${r.name}\n${r.content}`).join("\n\n")}`
 		: "";

package/src/dynamic-decision.ts

@@ -283,17 +283,6 @@ export function validateDynamicDecision(
 	};
 }
 
-export function assertValidDynamicDecision(
-	value: unknown,
-	context: DynamicDecisionValidationContext = {},
-): NormalizedDynamicDecision {
-	const result = validateDynamicDecision(value, context);
-	if (!result.ok || !result.decision) {
-		throw new Error(`invalid dynamic decision: ${result.errors.join("; ")}`);
-	}
-	return result.decision;
-}
-
 export function hashDynamicDecision(value: unknown): string {
 	return createHash("sha256")
 		.update(stableStringify(toJsonNormalizedValue(value)))

package/src/dynamic-profiles.ts

@@ -44,7 +44,3 @@ export function isTerminalDynamicOutputProfile(
 ): value is (typeof DYNAMIC_TERMINAL_OUTPUT_PROFILES)[number] {
 	return typeof value === "string" && TERMINAL_OUTPUT_PROFILE_SET.has(value);
 }
-
-export function dynamicOutputProfileValues(): string[] {
-	return [...DYNAMIC_OUTPUT_PROFILES];
-}

package/src/engine-run-graph.ts

@@ -127,6 +127,135 @@ export function reconcileDynamicGeneratedRunRecords(
 	return changed;
 }
 
+export function reconcileForeachGeneratedRunRecords(
+	cwd: string,
+	run: WorkflowRunRecord,
+	compiledFlow: CompiledWorkflow,
+): boolean {
+	let changed = false;
+	const compiledSpecIds = new Set(
+		compiledFlow.tasks.map((task) => compiledTaskSpecId(task)),
+	);
+	const placeholderToGeneratedSpecIds = new Map<string, string[]>();
+
+	for (const compiledTask of compiledFlow.tasks) {
+		const specId = compiledTaskSpecId(compiledTask);
+		const placeholderSpecId = foreachGeneratedPlaceholderSpecId(
+			compiledTask,
+			compiledFlow,
+			specId,
+		);
+		if (!placeholderSpecId) continue;
+		if (
+			compiledTask.foreachGenerated?.placeholderSpecId !== placeholderSpecId
+		) {
+			compiledTask.foreachGenerated = { placeholderSpecId };
+			changed = true;
+		}
+		const generated =
+			placeholderToGeneratedSpecIds.get(placeholderSpecId) ?? [];
+		generated.push(specId);
+		placeholderToGeneratedSpecIds.set(placeholderSpecId, generated);
+	}
+
+	if (placeholderToGeneratedSpecIds.size === 0) return changed;
+
+	const filteredRunTasks: WorkflowTaskRunRecord[] = [];
+	const seenGeneratedSpecIds = new Set<string>();
+	for (const task of run.tasks) {
+		const generatedSpecIds = placeholderToGeneratedSpecIds.get(task.specId);
+		let placeholderSpecId = foreachGeneratedPlaceholderSpecId(
+			task,
+			compiledFlow,
+			task.specId,
+		);
+		if (generatedSpecIds && !placeholderSpecId) {
+			if (generatedSpecIds.includes(task.specId)) {
+				placeholderSpecId = task.specId;
+				task.foreachGenerated = { placeholderSpecId };
+				changed = true;
+			} else {
+				changed = true;
+				continue;
+			}
+		}
+		if (placeholderSpecId && !compiledSpecIds.has(task.specId)) {
+			changed = true;
+			continue;
+		}
+		if (placeholderSpecId && seenGeneratedSpecIds.has(task.specId)) {
+			changed = true;
+			continue;
+		}
+		if (placeholderSpecId) {
+			seenGeneratedSpecIds.add(task.specId);
+			if (task.foreachGenerated?.placeholderSpecId !== placeholderSpecId) {
+				task.foreachGenerated = { placeholderSpecId };
+				changed = true;
+			}
+		}
+		filteredRunTasks.push(task);
+	}
+
+	const runTaskBySpecId = new Map<string, WorkflowTaskRunRecord>();
+	for (const task of filteredRunTasks) {
+		if (!runTaskBySpecId.has(task.specId))
+			runTaskBySpecId.set(task.specId, task);
+	}
+
+	const reordered: WorkflowTaskRunRecord[] = [];
+	const usedSpecIds = new Set<string>();
+	let nextIndex = nextTaskRecordIndex({ ...run, tasks: filteredRunTasks });
+	for (const compiledTask of compiledFlow.tasks) {
+		const specId = compiledTaskSpecId(compiledTask);
+		const existing = runTaskBySpecId.get(specId);
+		if (existing) {
+			const placeholderSpecId =
+				compiledTask.foreachGenerated?.placeholderSpecId;
+			if (
+				placeholderSpecId &&
+				existing.foreachGenerated?.placeholderSpecId !== placeholderSpecId
+			) {
+				existing.foreachGenerated = { placeholderSpecId };
+				changed = true;
+			}
+			reordered.push(existing);
+			usedSpecIds.add(specId);
+			continue;
+		}
+		if (!compiledTask.foreachGenerated) continue;
+		const created = createTaskRunRecord(
+			cwd,
+			run.runId,
+			compiledTask,
+			nextIndex,
+		);
+		nextIndex += 1;
+		reordered.push(created);
+		usedSpecIds.add(specId);
+		changed = true;
+	}
+
+	for (const task of filteredRunTasks) {
+		if (!usedSpecIds.has(task.specId)) reordered.push(task);
+	}
+
+	if (!sameTaskRecordOrder(run.tasks, reordered)) changed = true;
+	for (const task of reordered) {
+		if (!task.dependsOn) continue;
+		const replaced = replaceForeachGeneratedDependencies(
+			task.dependsOn,
+			placeholderToGeneratedSpecIds,
+		);
+		if (!sameStringList(task.dependsOn, replaced)) {
+			task.dependsOn = replaced;
+			changed = true;
+		}
+	}
+	if (changed) run.tasks = reordered;
+	return changed;
+}
+
 export function assertRunTaskPositionalAlignment(
 	run: WorkflowRunRecord,
 	compiledFlow: CompiledWorkflow,
@@ -209,6 +338,54 @@ export function compiledTaskSpecId(task: CompiledTask): string {
 	return typeof specId === "string" && specId.trim() !== "" ? specId : task.id;
 }
 
+function foreachGeneratedPlaceholderSpecId(
+	task: CompiledTask | WorkflowTaskRunRecord,
+	compiledFlow: CompiledWorkflow,
+	specId: string,
+): string | undefined {
+	const explicit = task.foreachGenerated?.placeholderSpecId;
+	if (typeof explicit === "string" && explicit.trim() !== "") return explicit;
+	if ((task as CompiledTask).foreach) return undefined;
+	if (task.kind !== "foreach" || !task.stageId) return undefined;
+	const placeholderSpecId = foreachPlaceholderSpecId(
+		compiledFlow,
+		task.stageId,
+	);
+	if (!placeholderSpecId || specId === placeholderSpecId) return undefined;
+	return placeholderSpecId;
+}
+
+function foreachPlaceholderSpecId(
+	compiledFlow: CompiledWorkflow,
+	stageId: string,
+): string | undefined {
+	const stage = ((compiledFlow as any).stages ?? []).find(
+		(candidate: any) => candidate?.id === stageId,
+	);
+	if (stage?.type !== "foreach") return undefined;
+	return `${stageId}.item`;
+}
+
+function replaceForeachGeneratedDependencies(
+	dependsOn: string[],
+	placeholderToGeneratedSpecIds: Map<string, string[]>,
+): string[] {
+	const replaced: string[] = [];
+	for (const dep of dependsOn) {
+		const generatedSpecIds = placeholderToGeneratedSpecIds.get(dep);
+		if (generatedSpecIds) replaced.push(...generatedSpecIds);
+		else replaced.push(dep);
+	}
+	return [...new Set(replaced)];
+}
+
+function sameStringList(left: string[], right: string[]): boolean {
+	return (
+		left.length === right.length &&
+		left.every((value, index) => value === right[index])
+	);
+}
+
 function isLoopGeneratedCompiledTask(
 	task: CompiledTask,
 	loopIds: Set<string>,
@@ -330,7 +507,7 @@ export function buildForeachGeneratedTasks(
 		const itemText = formatForeachItem(item);
 		const instructions = template.foreach!.prompt.replace(
 			/\$\{item\}/g,
-			itemText,
+			escapeReplacementText(itemText),
 		);
 		const compiledPrompt = [
 			template.foreach!.injectRuntimeTask && runtimeTask
@@ -352,6 +529,7 @@ export function buildForeachGeneratedTasks(
 			compiledPrompt,
 			dependsOn: [...(template.dependsOn ?? [])],
 			foreach: undefined,
+			foreachGenerated: { placeholderSpecId: template.id },
 		} as CompiledTask);
 	}
 	return { tasks };
@@ -382,6 +560,10 @@ function formatForeachItem(item: unknown): string {
 	return typeof item === "string" ? item : JSON.stringify(item);
 }
 
+function escapeReplacementText(value: string): string {
+	return value.replace(/\$/g, "$$$$");
+}
+
 export function sourceStageIdsForFrom(from: unknown): string[] {
 	if (Array.isArray(from))
 		return from.filter((item): item is string => typeof item === "string");
@@ -453,7 +635,8 @@ export function markDagDependentsSkipped(
 				return (
 					status === "failed" ||
 					status === "interrupted" ||
-					status === "skipped"
+					status === "skipped" ||
+					status === "blocked"
 				);
 			});
 			if (!failedDep) continue;