@bastani/atomic 0.8.31-alpha.5 → 0.9.0-alpha.1

package/dist/builtin/subagents/src/runs/background/subagent-runner-step.ts

@@ -0,0 +1,273 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { getArtifactPaths } from "../../shared/artifacts.ts";
+import { detectSubagentError } from "../../shared/utils.ts";
+import { resolveEffectiveThinking } from "../../shared/model-info.ts";
+import { captureSingleOutputSnapshot, finalizeSingleOutput, formatSavedOutputReference, resolveSingleOutput, type SingleOutputSnapshot } from "../shared/single-output.ts";
+import { buildPiArgs, cleanupTempDir } from "../shared/pi-args.ts";
+import { outputEntryFromAsyncResult, resolveOutputReferences } from "../shared/chain-outputs.ts";
+import {
+	STRUCTURED_OUTPUT_MAX_CORRECTIVE_PROMPTS,
+	createStructuredOutputRuntime,
+	formatStructuredOutputCorrectionPrompt,
+	isStructuredOutputContractError,
+	latestStructuredOutputToolErrorFromMessages,
+	readStructuredOutput,
+} from "../shared/structured-output.ts";
+import { formatModelAttemptNote, isRetryableModelFailure } from "../shared/model-fallback.ts";
+import type { ArtifactPaths, ModelAttempt } from "../../shared/types.ts";
+import type { RunPiStreamingResult, SingleStepContext, SubagentStep } from "./subagent-runner-types.ts";
+import { fastModeForStepAttempt } from "./subagent-runner-utils.ts";
+import { runPiStreaming } from "./subagent-runner-streaming.ts";
+
+export { outputEntryFromAsyncResult };
+
+export async function runSingleStep(
+	step: SubagentStep,
+	ctx: SingleStepContext,
+): Promise<{
+	agent: string;
+	output: string;
+	exitCode: number | null;
+	error?: string;
+	model?: string;
+	fastMode?: boolean;
+	attemptedModels?: string[];
+	modelAttempts?: ModelAttempt[];
+	artifactPaths?: ArtifactPaths;
+	interrupted?: boolean;
+	sessionFile?: string;
+	intercomTarget?: string;
+	structuredOutput?: unknown;
+	structuredOutputPath?: string;
+	structuredOutputSchemaPath?: string;
+}> {
+	const effectiveStructuredOutput = step.structuredOutput ?? (step.structuredOutputSchema
+		? createStructuredOutputRuntime(step.structuredOutputSchema, path.join(path.dirname(ctx.outputFile), "structured-output"))
+		: undefined);
+	const placeholderRegex = new RegExp(ctx.placeholder.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g");
+	let task = step.task.replace(placeholderRegex, () => ctx.previousOutput);
+	task = resolveOutputReferences(task, ctx.outputs ?? {});
+	const sessionEnabled = Boolean(step.sessionFile) || ctx.sessionEnabled;
+	const sessionDir = step.sessionFile ? undefined : ctx.sessionDir;
+
+	let artifactPaths: ArtifactPaths | undefined;
+	if (ctx.artifactsDir && ctx.artifactConfig?.enabled !== false) {
+		const index = ctx.flatStepCount > 1 ? ctx.flatIndex : undefined;
+		artifactPaths = getArtifactPaths(ctx.artifactsDir, ctx.id, step.agent, index);
+		fs.mkdirSync(ctx.artifactsDir, { recursive: true });
+		if (ctx.artifactConfig?.includeInput !== false) {
+			fs.writeFileSync(artifactPaths.inputPath, `# Task for ${step.agent}\n\n${task}`, "utf-8");
+		}
+	}
+
+	const candidates = step.modelCandidates && step.modelCandidates.length > 0
+		? step.modelCandidates
+		: step.model
+			? [step.model]
+			: [undefined];
+	const attemptedModels: string[] = [];
+	const modelAttempts: ModelAttempt[] = [];
+	const attemptNotes: string[] = [];
+	const pendingAttemptNotes: string[] = [];
+	const eventsPath = path.join(path.dirname(ctx.outputFile), "events.jsonl");
+	let finalResult: RunPiStreamingResult | undefined;
+	let finalFastMode: boolean | undefined;
+	let finalOutputSnapshot: SingleOutputSnapshot | undefined;
+
+	for (let index = 0; index < candidates.length; index++) {
+		const candidate = candidates[index];
+		const attemptFastMode = fastModeForStepAttempt(step, candidate);
+		ctx.onAttemptStart?.({ model: candidate, thinking: resolveEffectiveThinking(candidate, step.thinking), fastMode: attemptFastMode ? true : undefined });
+		if (candidate) attemptedModels.push(candidate);
+		let nextTask = task;
+		let correctiveAttempts = 0;
+		let tryNextModel = false;
+
+		while (true) {
+			const outputSnapshot = captureSingleOutputSnapshot(step.outputPath);
+			if (effectiveStructuredOutput) {
+				try {
+					if (fs.existsSync(effectiveStructuredOutput.outputPath)) fs.unlinkSync(effectiveStructuredOutput.outputPath);
+				} catch {
+					// Missing/stale structured-output files are handled after the child exits.
+				}
+			}
+			const { args, env, tempDir } = buildPiArgs({
+				baseArgs: ["--mode", "json", "-p"],
+				task: nextTask,
+				sessionEnabled,
+				sessionDir,
+				sessionFile: step.sessionFile,
+				model: candidate,
+				inheritProjectContext: step.inheritProjectContext,
+				inheritSkills: step.inheritSkills,
+				tools: step.tools,
+				extensions: step.extensions,
+				systemPrompt: step.systemPrompt,
+				systemPromptMode: step.systemPromptMode,
+				mcpDirectTools: step.mcpDirectTools,
+				cwd: step.cwd ?? ctx.cwd,
+				promptFileStem: step.agent,
+				intercomSessionName: ctx.childIntercomTarget,
+				orchestratorIntercomTarget: ctx.orchestratorIntercomTarget,
+				runId: ctx.id,
+				childAgentName: step.agent,
+				childIndex: ctx.flatIndex,
+				parentEventSink: ctx.nestedRoute?.eventSink,
+				parentControlInbox: ctx.nestedRoute?.controlInbox,
+				parentRootRunId: ctx.nestedRoute?.rootRunId,
+				parentCapabilityToken: ctx.nestedRoute?.capabilityToken,
+				codexFastModeSettings: step.codexFastModeSettings,
+				codexFastModeScope: step.codexFastModeScope,
+				structuredOutput: effectiveStructuredOutput,
+			});
+			const run = await runPiStreaming(
+				args,
+				step.cwd ?? ctx.cwd,
+				ctx.outputFile,
+				env,
+				ctx.piPackageRoot,
+				ctx.piArgv1,
+				step.maxSubagentDepth,
+				step.workflowStageSubagentGuard ?? ctx.workflowStageSubagentGuard,
+				{ eventsPath, runId: ctx.id, stepIndex: ctx.flatIndex, agent: step.agent },
+				ctx.registerInterrupt,
+				ctx.onChildEvent,
+			);
+			cleanupTempDir(tempDir);
+
+			const hiddenError = run.exitCode === 0 && !run.error ? detectSubagentError(run.messages) : null;
+			let structuredOutput: unknown;
+			let structuredError: string | undefined;
+			if (effectiveStructuredOutput && run.exitCode === 0 && !run.error && !hiddenError?.hasError) {
+				const structured = readStructuredOutput(effectiveStructuredOutput);
+				if (structured.error) structuredError = structured.error;
+				else structuredOutput = structured.value;
+			}
+			const structuredContractError = structuredError
+				? latestStructuredOutputToolErrorFromMessages(run.messages) ?? structuredError
+				: undefined;
+			const effectiveExitCode = structuredContractError
+				? 1
+				: hiddenError?.hasError
+				? (hiddenError.exitCode ?? 1)
+				: run.error && run.exitCode === 0
+					? 1
+					: run.exitCode;
+			const error = structuredContractError
+				?? (hiddenError?.hasError
+					? hiddenError.details
+						? `${hiddenError.errorType} failed (exit ${effectiveExitCode}): ${hiddenError.details}`
+						: `${hiddenError.errorType} failed with exit code ${effectiveExitCode}`
+					: run.error || (run.exitCode !== 0 && run.stderr.trim() ? run.stderr.trim() : undefined));
+			const attemptModel = candidate ?? run.model ?? step.model ?? "default";
+			const attempt: ModelAttempt = {
+				model: attemptModel,
+				reasoningLevel: resolveEffectiveThinking(attemptModel, step.thinking),
+				success: effectiveExitCode === 0 && !error,
+				exitCode: effectiveExitCode,
+				error,
+				usage: run.usage,
+			};
+			modelAttempts.push(attempt);
+			finalFastMode = attemptFastMode;
+			finalOutputSnapshot = outputSnapshot;
+			finalResult = { ...run, exitCode: effectiveExitCode, model: candidate ?? run.model, error, structuredOutput } as RunPiStreamingResult & { structuredOutput?: unknown };
+			if (attempt.success) break;
+			if (
+				effectiveStructuredOutput
+				&& isStructuredOutputContractError(structuredContractError)
+				&& correctiveAttempts < STRUCTURED_OUTPUT_MAX_CORRECTIVE_PROMPTS
+			) {
+				correctiveAttempts += 1;
+				nextTask = formatStructuredOutputCorrectionPrompt({
+					originalTask: task,
+					error: structuredContractError!,
+					attempt: correctiveAttempts,
+				});
+				continue;
+			}
+			const retrySignal = run.modelFailureSignal ?? error;
+			if (
+				structuredContractError === undefined
+				&& hiddenError?.hasError !== true
+				&& isRetryableModelFailure(retrySignal)
+				&& index < candidates.length - 1
+			) {
+				pendingAttemptNotes.push(formatModelAttemptNote(attempt, candidates[index + 1]));
+				tryNextModel = true;
+				break;
+			}
+			attemptNotes.push(...pendingAttemptNotes);
+			break;
+		}
+		if (!tryNextModel) break;
+	}
+
+	const rawOutput = finalResult?.finalOutput ?? "";
+	const outputForPersistence = rawOutput;
+	const resolvedOutput = step.outputPath && finalResult?.exitCode === 0
+		? resolveSingleOutput(step.outputPath, outputForPersistence, finalOutputSnapshot)
+		: { fullOutput: outputForPersistence };
+	const output = resolvedOutput.fullOutput;
+	const outputReference = resolvedOutput.savedPath ? formatSavedOutputReference(resolvedOutput.savedPath, output) : undefined;
+	let outputForSummary = output;
+	if (attemptNotes.length > 0) {
+		outputForSummary = `${attemptNotes.join("\n")}\n\n${outputForSummary}`.trim();
+	}
+	const finalizedOutput = finalizeSingleOutput({
+		fullOutput: outputForSummary,
+		outputPath: step.outputPath,
+		outputMode: step.outputMode,
+		exitCode: finalResult?.exitCode ?? 1,
+		savedPath: resolvedOutput.savedPath,
+		outputReference,
+		saveError: resolvedOutput.saveError,
+	});
+	outputForSummary = finalizedOutput.displayOutput;
+	const effectiveFinalExitCode = finalResult?.exitCode ?? 1;
+	const effectiveFinalError = finalResult?.error;
+
+	if (artifactPaths && ctx.artifactConfig?.enabled !== false) {
+		if (ctx.artifactConfig?.includeOutput !== false) {
+			fs.writeFileSync(artifactPaths.outputPath, output, "utf-8");
+		}
+		if (ctx.artifactConfig?.includeMetadata !== false) {
+			fs.writeFileSync(
+				artifactPaths.metadataPath,
+				JSON.stringify({
+					runId: ctx.id,
+					agent: step.agent,
+					task,
+					exitCode: effectiveFinalExitCode,
+					model: finalResult?.model,
+					...(finalFastMode ? { fastMode: true } : {}),
+					attemptedModels: attemptedModels.length > 0 ? attemptedModels : undefined,
+					modelAttempts,
+					skills: step.skills,
+					timestamp: Date.now(),
+				}, null, 2),
+				"utf-8",
+			);
+		}
+	}
+
+	return {
+		agent: step.agent,
+		output: outputForSummary,
+		exitCode: effectiveFinalExitCode,
+		error: effectiveFinalError,
+		sessionFile: step.sessionFile,
+		intercomTarget: ctx.childIntercomTarget,
+		model: finalResult?.model,
+		...(finalFastMode ? { fastMode: true } : {}),
+		attemptedModels: attemptedModels.length > 0 ? attemptedModels : undefined,
+		modelAttempts,
+		artifactPaths,
+		interrupted: finalResult?.interrupted,
+		structuredOutput: (finalResult as (RunPiStreamingResult & { structuredOutput?: unknown }) | undefined)?.structuredOutput,
+		structuredOutputPath: effectiveStructuredOutput?.outputPath,
+		structuredOutputSchemaPath: effectiveStructuredOutput?.schemaPath,
+	};
+}

package/dist/builtin/subagents/src/runs/background/subagent-runner-streaming.ts

@@ -0,0 +1,269 @@
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import type { Message } from "@earendil-works/pi-ai";
+import { appendJsonl } from "../../shared/artifacts.ts";
+import { attachPostExitStdioGuard, trySignalChild } from "../../shared/post-exit-stdio-guard.ts";
+import { detectSubagentError, extractTextFromContent, extractToolArgsPreview, getFinalOutput } from "../../shared/utils.ts";
+import { getSubagentDepthEnv } from "../../shared/types.ts";
+import { getPiSpawnCommand } from "../shared/pi-spawn.ts";
+import {
+	assistantStopReason,
+	isAssistantFailureStopReason,
+	shouldStartSubagentFinalDrain,
+} from "../shared/final-drain.ts";
+import { modelFailureMessage } from "../shared/model-fallback.ts";
+import type { ChildEvent, ChildEventContext, RunPiStreamingResult } from "./subagent-runner-types.ts";
+import { emptyUsage } from "./subagent-runner-utils.ts";
+
+export function runPiStreaming(
+	args: string[],
+	cwd: string,
+	outputFile: string,
+	env?: Record<string, string | undefined>,
+	piPackageRoot?: string,
+	piArgv1?: string,
+	maxSubagentDepth?: number,
+	workflowStageSubagentGuard?: boolean,
+	childEventContext?: ChildEventContext,
+	registerInterrupt?: (interrupt: (() => void) | undefined) => void,
+	onChildEvent?: (event: ChildEvent) => void,
+): Promise<RunPiStreamingResult> {
+	return new Promise((resolve) => {
+		const outputStream = fs.createWriteStream(outputFile, { flags: "w" });
+		const spawnEnv = {
+			...process.env,
+			...(env ?? {}),
+			...getSubagentDepthEnv(maxSubagentDepth, { workflowStageSubagentGuard }),
+		};
+		const spawnSpec = getPiSpawnCommand(args, {
+			...(piPackageRoot ? { piPackageRoot } : {}),
+			...(piArgv1 ? { argv1: piArgv1 } : {}),
+		});
+		const child = spawn(spawnSpec.command, spawnSpec.args, {
+			cwd,
+			stdio: ["ignore", "pipe", "pipe"],
+			env: spawnEnv,
+			windowsHide: true,
+		});
+		let stderr = "";
+		let stdoutBuf = "";
+		let stderrBuf = "";
+		const messages: Message[] = [];
+		const usage = emptyUsage();
+		let model: string | undefined;
+		let error: string | undefined;
+		let assistantError: string | undefined;
+		let assistantFailureSignal: unknown;
+		let interrupted = false;
+		const rawStdoutLines: string[] = [];
+
+		const writeOutputLine = (line: string) => {
+			if (!line.trim()) return;
+			outputStream.write(`${line}\n`);
+		};
+
+		const writeOutputText = (text: string) => {
+			for (const line of text.split("\n")) {
+				writeOutputLine(line);
+			}
+		};
+
+		const appendChildEvent = (event: Record<string, unknown> | ChildEvent) => {
+			if (!childEventContext) return;
+			appendJsonl(childEventContext.eventsPath, JSON.stringify({
+				...event,
+				subagentSource: "child",
+				subagentRunId: childEventContext.runId,
+				subagentStepIndex: childEventContext.stepIndex,
+				subagentAgent: childEventContext.agent,
+				observedAt: Date.now(),
+			}));
+		};
+
+		const appendChildLine = (type: "subagent.child.stdout" | "subagent.child.stderr", line: string) => {
+			appendChildEvent({ type, line });
+		};
+
+		const processStdoutLine = (line: string) => {
+			if (!line.trim()) return;
+			let event: ChildEvent;
+			try {
+				event = JSON.parse(line) as ChildEvent;
+			} catch {
+				rawStdoutLines.push(line);
+				writeOutputLine(line);
+				appendChildLine("subagent.child.stdout", line);
+				return;
+			}
+
+			appendChildEvent(event);
+			onChildEvent?.(event);
+
+			if (event.type === "tool_execution_start" && event.toolName) {
+				const toolArgs = extractToolArgsPreview(event.args ?? {});
+				writeOutputLine(toolArgs ? `${event.toolName}: ${toolArgs}` : event.toolName);
+				return;
+			}
+
+			if ((event.type === "message_end" || event.type === "tool_result_end") && event.message) {
+				messages.push(event.message);
+				const text = extractTextFromContent(event.message.content);
+				if (text) writeOutputText(text);
+
+				if (event.type !== "message_end" || event.message.role !== "assistant") return;
+				if (event.message.model) model = event.message.model;
+				const eventUsage = event.message.usage;
+				if (eventUsage) {
+					usage.turns++;
+					usage.input += eventUsage.input ?? eventUsage.inputTokens ?? 0;
+					usage.output += eventUsage.output ?? eventUsage.outputTokens ?? 0;
+					usage.cacheRead += eventUsage.cacheRead ?? 0;
+					usage.cacheWrite += eventUsage.cacheWrite ?? 0;
+					usage.cost += eventUsage.cost?.total ?? 0;
+				}
+				const stopReason = assistantStopReason(event.message);
+				if (event.message.errorMessage) {
+					assistantError = event.message.errorMessage;
+					assistantFailureSignal = event.message;
+				}
+				if (isAssistantFailureStopReason(stopReason)) {
+					assistantError = modelFailureMessage(event.message);
+					assistantFailureSignal = event.message;
+				}
+				if (shouldStartSubagentFinalDrain(event.message)) {
+					if (extractTextFromContent(event.message.content).trim()) {
+						assistantError = undefined;
+						assistantFailureSignal = undefined;
+					}
+					cleanTerminalAssistantStopReceived = true;
+					startFinalDrain();
+				}
+			}
+		};
+
+		const processStderrText = (text: string) => {
+			stderr += text;
+			stderrBuf += text;
+			outputStream.write(text);
+			if (!childEventContext) return;
+			const lines = stderrBuf.split("\n");
+			stderrBuf = lines.pop() || "";
+			for (const line of lines) {
+				if (!line.trim()) continue;
+				appendChildLine("subagent.child.stderr", line);
+			}
+		};
+
+		const FINAL_STOP_GRACE_MS = 1000;
+		const HARD_KILL_MS = 3000;
+		let childExited = false;
+		let forcedTerminationSignal = false;
+		let cleanTerminalAssistantStopReceived = false;
+		let finalDrainTimer: NodeJS.Timeout | undefined;
+		let finalHardKillTimer: NodeJS.Timeout | undefined;
+		let settled = false;
+		const clearStdioGuard = attachPostExitStdioGuard(child, { idleMs: 2000, hardMs: 8000 });
+		child.stdout.on("data", (chunk: Buffer) => {
+			const text = chunk.toString();
+			stdoutBuf += text;
+			const lines = stdoutBuf.split("\n");
+			stdoutBuf = lines.pop() || "";
+			for (const line of lines) processStdoutLine(line);
+		});
+
+		child.stderr.on("data", (chunk: Buffer) => {
+			processStderrText(chunk.toString());
+		});
+		registerInterrupt?.(() => {
+			if (settled) return;
+			interrupted = true;
+			if (!error) error = "Interrupted. Waiting for explicit next action.";
+			trySignalChild(child, "SIGINT");
+			setTimeout(() => {
+				if (!settled) trySignalChild(child, "SIGTERM");
+			}, 1000).unref?.();
+		});
+		const clearDrainTimers = () => {
+			if (finalDrainTimer) {
+				clearTimeout(finalDrainTimer);
+				finalDrainTimer = undefined;
+			}
+			if (finalHardKillTimer) {
+				clearTimeout(finalHardKillTimer);
+				finalHardKillTimer = undefined;
+			}
+		};
+		function startFinalDrain(): void {
+			if (childExited || finalDrainTimer || settled) return;
+			finalDrainTimer = setTimeout(() => {
+				if (settled) return;
+				const termSent = trySignalChild(child, "SIGTERM");
+				if (!termSent) return;
+				forcedTerminationSignal = true;
+				if (!cleanTerminalAssistantStopReceived && !error && !assistantError) {
+					error = `Subagent process did not exit within ${FINAL_STOP_GRACE_MS}ms after its final message. Forcing termination.`;
+				}
+				finalHardKillTimer = setTimeout(() => {
+					if (settled) return;
+					forcedTerminationSignal = trySignalChild(child, "SIGKILL") || forcedTerminationSignal;
+				}, HARD_KILL_MS);
+				finalHardKillTimer.unref?.();
+			}, FINAL_STOP_GRACE_MS);
+			finalDrainTimer.unref?.();
+		}
+		child.on("exit", () => {
+			childExited = true;
+			clearDrainTimers();
+		});
+		child.on("close", (exitCode, signal) => {
+			settled = true;
+			registerInterrupt?.(undefined);
+			clearDrainTimers();
+			clearStdioGuard();
+			if (stdoutBuf.trim()) processStdoutLine(stdoutBuf);
+			if (stderrBuf.trim()) appendChildLine("subagent.child.stderr", stderrBuf);
+			outputStream.end();
+			const finalOutput = getFinalOutput(messages) || rawStdoutLines.join("\n").trim();
+			const finalError = error ?? assistantError;
+			const forcedDrainAfterFinalSuccess = forcedTerminationSignal && cleanTerminalAssistantStopReceived && !finalError;
+			resolve({
+				stderr,
+				exitCode: interrupted || forcedDrainAfterFinalSuccess ? 0 : forcedTerminationSignal || signal ? (exitCode ?? 1) : exitCode,
+				messages,
+				usage,
+				model,
+				error: interrupted || forcedDrainAfterFinalSuccess ? undefined : finalError,
+				finalOutput,
+				interrupted,
+				...(assistantFailureSignal !== undefined && finalError === assistantError
+					? { modelFailureSignal: assistantFailureSignal }
+					: {}),
+			});
+		});
+
+		child.on("error", (spawnError) => {
+			settled = true;
+			registerInterrupt?.(undefined);
+			clearDrainTimers();
+			clearStdioGuard();
+			outputStream.end();
+			const finalOutput = getFinalOutput(messages) || rawStdoutLines.join("\n").trim();
+			const spawnErrorMessage = spawnError instanceof Error ? spawnError.message : String(spawnError);
+			const finalError = error ?? assistantError ?? spawnErrorMessage;
+			resolve({
+				stderr,
+				exitCode: 1,
+				messages,
+				usage,
+				model,
+				error: finalError,
+				finalOutput,
+				...(assistantFailureSignal !== undefined && finalError === assistantError
+					? { modelFailureSignal: assistantFailureSignal }
+					: {}),
+			});
+		});
+	});
+}
+
+export { detectSubagentError };