@abdwhb-png/pi-test-harness 0.7.0

package/src/sandbox.ts

@@ -0,0 +1,334 @@
+/**
+ * Sandbox install verification — verifies npm packages work when installed clean.
+ *
+ * 1. npm pack → tarball
+ * 2. Install in temp dir
+ * 3. DefaultResourceLoader discovers extensions/skills
+ * 4. Verify resources load without errors
+ * 5. Optional smoke test
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import { execFileSync } from "node:child_process";
+import {
+	DefaultResourceLoader,
+	SettingsManager,
+} from "@earendil-works/pi-coding-agent";
+import { withoutJitiNativeImport } from "./pi-loader-parity.js";
+import type { SandboxOptions, SandboxResult } from "./types.js";
+import { createTestSession } from "./session.js";
+
+/** Resolve the npm command array. Defaults to platform-aware npm. */
+function resolveNpmCommand(npmCommand?: string[]): string[] {
+	if (npmCommand && npmCommand.length > 0) return npmCommand;
+	return [process.platform === "win32" ? "npm.cmd" : "npm"];
+}
+
+/**
+ * Resolve `command` to something execFileSync can actually start.
+ *
+ * **Why this exists**: on Windows a bare `sfw` (or `npm`) is the `.cmd` shim, and
+ * child_process applies no PATHEXT when creating the process, so spawning it
+ * fails with ENOENT — which broke the documented `npmCommand: ["sfw", "npm"]`
+ * preset on the Windows integration matrix. Candidate extensions come from
+ * PATHEXT (or the usual Windows default) and are matched against PATH in the
+ * same order the platform itself would. An unmatched command is returned
+ * unchanged, so a genuinely missing executable still raises a clear ENOENT.
+ *
+ * Exported for unit testing — not part of the public API contract.
+ * @internal
+ */
+export function _resolveExecutable(
+	command: string,
+	platform: NodeJS.Platform = process.platform,
+	env: NodeJS.ProcessEnv = process.env,
+): string {
+	if (platform !== "win32") return command;
+	if (path.extname(command) !== "") return command;
+
+	const extensions = (env.PATHEXT ?? ".COM;.EXE;.BAT;.CMD")
+		.split(";")
+		.filter(Boolean);
+
+	// An explicit path never goes through PATH resolution — only PATHEXT applies.
+	if (command.includes("/") || command.includes("\\")) {
+		for (const extension of extensions) {
+			const candidate = `${command}${extension}`;
+			if (fs.existsSync(candidate)) return candidate;
+		}
+		return command;
+	}
+
+	const directories = (env.PATH ?? "").split(path.delimiter).filter(Boolean);
+	for (const directory of directories) {
+		for (const extension of extensions) {
+			const candidate = path.join(directory, `${command}${extension}`);
+			if (fs.existsSync(candidate)) return candidate;
+		}
+	}
+
+	return command;
+}
+
+/**
+ * Run a command via execFileSync with safe string conversion for the full
+ * command line in the error message.
+ */
+function run(args: string[], cwd: string, label: string): string {
+	const [cmd, ...cmdArgs] = args;
+	try {
+		return execFileSync(_resolveExecutable(cmd), cmdArgs, {
+			cwd,
+			encoding: "utf-8",
+			stdio: ["pipe", "pipe", "pipe"],
+		}).trim();
+	} catch (err: any) {
+		// Enhance the error message with context
+		const stderr = err.stderr?.toString().trim() ?? "";
+		const enhanced = new Error(
+			`${label} failed: ${err.message}${stderr ? `\nstderr: ${stderr}` : ""}`,
+		);
+		// Preserve the original error code (ENOENT, etc.)
+		(enhanced as any).code = err.code;
+		throw enhanced;
+	}
+}
+
+/** Manifest fields the sandbox install path needs. */
+interface PackageManifest {
+	name?: string;
+	pi?: {
+		extensions?: string[];
+	};
+}
+
+/**
+ * Read a package.json, reporting the offending path instead of surfacing a bare
+ * SyntaxError. A malformed manifest is fatal for both call sites below, so the
+ * failure is rethrown with the path that caused it.
+ */
+function readPackageJson(filePath: string): PackageManifest {
+	let raw: string;
+	try {
+		raw = fs.readFileSync(filePath, "utf-8");
+	} catch (err) {
+		throw new Error(
+			`Could not read ${filePath}: ${err instanceof Error ? err.message : String(err)}`,
+			{ cause: err },
+		);
+	}
+	try {
+		return JSON.parse(raw) as PackageManifest;
+	} catch (err) {
+		throw new Error(
+			`Invalid JSON in ${filePath}: ${err instanceof Error ? err.message : String(err)}`,
+			{ cause: err },
+		);
+	}
+}
+
+/**
+ * Read the package name from a package.json, decoded here rather than at the
+ * call site: every consumer of this value needs it to be a non-empty string.
+ */
+function readPackageName(filePath: string): string {
+	const manifest = readPackageJson(filePath);
+	const name = manifest.name;
+	if (typeof name !== "string" || name.length === 0) {
+		throw new Error(`package.json at ${filePath} has no "name" field`);
+	}
+	return name;
+}
+
+export async function verifySandboxInstall(
+	options: SandboxOptions,
+): Promise<SandboxResult> {
+	const packageDir = path.resolve(options.packageDir);
+	const npmCmd = resolveNpmCommand(options.npmCommand);
+
+	// Validate package directory
+	const pkgJsonPath = path.join(packageDir, "package.json");
+	if (!fs.existsSync(pkgJsonPath)) {
+		throw new Error(`No package.json found at ${pkgJsonPath}`);
+	}
+	const pkgName = readPackageName(pkgJsonPath);
+
+	// Create sandbox temp dir
+	const sandboxDir = fs.mkdtempSync(path.join(os.tmpdir(), "pi-sandbox-"));
+
+	try {
+		// 1. npm pack → tarball
+		const packArgs = [...npmCmd.slice(1), "pack", "--pack-destination", "."];
+		const packOutput = run(
+			[...npmCmd.slice(0, 1), ...packArgs],
+			packageDir,
+			`npm pack in ${packageDir}`,
+		);
+
+		// The output is the tarball filename
+		const tarballName = packOutput.split("\n").pop()!.trim();
+		const tarballSrc = path.join(packageDir, tarballName);
+		const tarballDest = path.join(sandboxDir, tarballName);
+		try {
+			fs.copyFileSync(tarballSrc, tarballDest);
+		} finally {
+			// Always clean up tarball from source (even if copy fails)
+			try {
+				if (fs.existsSync(tarballSrc)) fs.unlinkSync(tarballSrc);
+			} catch {
+				/* best-effort */
+			}
+		}
+
+		// 2. Create minimal package.json in sandbox
+		const sandboxPkg = {
+			name: "pi-test-sandbox",
+			private: true,
+			type: "module",
+			dependencies: {
+				[pkgName]: `file:./${tarballName}`,
+			},
+		};
+		fs.writeFileSync(
+			path.join(sandboxDir, "package.json"),
+			JSON.stringify(sandboxPkg, null, 2),
+		);
+
+		// 3. npm install
+		const installArgs = [...npmCmd.slice(1), "install", "--ignore-scripts=false"];
+		run(
+			[...npmCmd.slice(0, 1), ...installArgs],
+			sandboxDir,
+			`npm install in ${sandboxDir}`,
+		);
+
+		// 4. Find the installed package and use DefaultResourceLoader
+		const installedPkgDir = path.join(
+			sandboxDir,
+			"node_modules",
+			...pkgName.split("/"),
+		);
+
+		if (!fs.existsSync(installedPkgDir)) {
+			throw new Error(`Package not found after install: ${installedPkgDir}`);
+		}
+
+		// Read installed package.json for pi manifest
+		const installedPkgJson = readPackageJson(
+			path.join(installedPkgDir, "package.json"),
+		);
+		const piManifest = installedPkgJson.pi;
+
+		// Resolve extension paths from the installed package
+		const extensionPaths: string[] = [];
+		if (piManifest?.extensions) {
+			for (const ext of piManifest.extensions) {
+				const resolved = path.resolve(installedPkgDir, ext);
+				if (fs.existsSync(resolved)) {
+					extensionPaths.push(resolved);
+				} else {
+					// Try as glob/directory
+					const dir = path.resolve(installedPkgDir, ext);
+					if (fs.existsSync(dir) && fs.statSync(dir).isDirectory()) {
+						const files = fs
+							.readdirSync(dir)
+							.filter((f) => f.endsWith(".ts") || f.endsWith(".js"));
+						extensionPaths.push(...files.map((f) => path.join(dir, f)));
+					}
+				}
+			}
+		}
+
+		// Load extensions via DefaultResourceLoader
+		const settingsManager = SettingsManager.inMemory();
+		const loader = new DefaultResourceLoader({
+			cwd: sandboxDir,
+			agentDir: sandboxDir,
+			settingsManager,
+			additionalExtensionPaths: extensionPaths,
+		});
+		// Extension modules evaluate here; keep the loader configuration in parity
+		// with Pi's shipped runtimes (see withoutJitiNativeImport).
+		await withoutJitiNativeImport(() => loader.reload());
+
+		const extensionsResult = loader.getExtensions();
+		const skillsResult = loader.getSkills();
+
+		// Collect tool names from loaded extensions (no cast needed)
+		const toolNames: string[] = [];
+		for (const ext of extensionsResult.extensions) {
+			for (const [name] of ext.tools ?? new Map()) {
+				toolNames.push(name);
+			}
+		}
+
+		const result: SandboxResult = {
+			loaded: {
+				extensions: extensionsResult.extensions.length,
+				extensionErrors: extensionsResult.errors.map(
+					(e) => `${e.path}: ${e.error}`,
+				),
+				tools: toolNames,
+				skills: skillsResult.skills.length,
+			},
+		};
+
+		// 5. Verify expectations
+		if (options.expect) {
+			if (options.expect.extensions !== undefined) {
+				if (extensionsResult.extensions.length !== options.expect.extensions) {
+					throw new Error(
+						`Expected ${options.expect.extensions} extension(s), got ${extensionsResult.extensions.length}`,
+					);
+				}
+			}
+			if (options.expect.tools) {
+				for (const expectedTool of options.expect.tools) {
+					if (!toolNames.includes(expectedTool)) {
+						throw new Error(
+							`Expected tool "${expectedTool}" not found. Available: ${toolNames.join(", ")}`,
+						);
+					}
+				}
+			}
+			if (options.expect.skills !== undefined) {
+				if (skillsResult.skills.length !== options.expect.skills) {
+					throw new Error(
+						`Expected ${options.expect.skills} skill(s), got ${skillsResult.skills.length}`,
+					);
+				}
+			}
+		}
+
+		// 6. Optional smoke test
+		if (options.smoke) {
+			const t = await createTestSession({
+				extensions: extensionPaths,
+				cwd: sandboxDir,
+				mockTools: options.smoke.mockTools,
+			});
+
+			await t.run(...options.smoke.script);
+			result.smoke = { events: t.events };
+			t.dispose();
+		}
+
+		return result;
+	} finally {
+		// Clean up sandbox (retry for Windows EBUSY on open handles)
+		if (fs.existsSync(sandboxDir)) {
+			try {
+				fs.rmSync(sandboxDir, {
+					recursive: true,
+					force: true,
+					maxRetries: 3,
+					retryDelay: 200,
+				});
+			} catch {
+				// Best-effort cleanup — temp dir will be cleaned by OS
+			}
+		}
+	}
+}

package/src/session.ts

@@ -0,0 +1,249 @@
+/**
+ * TestSession — orchestrates a test run with playbook, mock tools, and mock UI.
+ *
+ * 1. Creates a real pi environment (extensions, tools, hooks, session)
+ * 2. Replaces streamFn with playbook
+ * 3. Intercepts tool.execute() for mockTools
+ * 4. Injects mock UI context
+ * 5. Collects events
+ * 6. Runs conversation script
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import {
+	createAgentSession,
+	DefaultResourceLoader,
+	SessionManager,
+	SettingsManager,
+	type AgentSessionEvent,
+	ModelRuntime,
+} from "@earendil-works/pi-coding-agent";
+import type { AgentTool } from "@earendil-works/pi-agent-core";
+import { createPlaybookStreamFn, type PlaybookState } from "./playbook.js";
+import { interceptToolExecution } from "./mock-tools.js";
+import { createMockUIContext } from "./mock-ui.js";
+import { createEventCollector } from "./events.js";
+import { formatPlaybookDiagnostic } from "./diagnostics.js";
+import { withoutJitiNativeImport } from "./pi-loader-parity.js";
+import type {
+	TestSessionOptions,
+	TestSession,
+	Turn,
+	ToolCallRecord,
+} from "./types.js";
+
+export async function createTestSession(
+	options: TestSessionOptions = {},
+): Promise<TestSession> {
+	const propagateErrors = options.propagateErrors ?? true;
+	const ownsTmpDir = !options.cwd;
+	const cwd =
+		options.cwd ?? fs.mkdtempSync(path.join(os.tmpdir(), "pi-test-harness-"));
+
+	if (!fs.existsSync(cwd)) {
+		fs.mkdirSync(cwd, { recursive: true });
+	}
+
+	const settingsManager = SettingsManager.inMemory();
+	const loader = new DefaultResourceLoader({
+		cwd,
+		agentDir: cwd, // Use cwd as agent dir to avoid touching real ~/.pi
+		settingsManager,
+		additionalExtensionPaths:
+			options.extensions?.map((p) => path.resolve(cwd, p)) ?? [],
+		extensionFactories: options.extensionFactories,
+		systemPromptOverride: options.systemPrompt
+			? () => options.systemPrompt!
+			: undefined,
+	});
+	// Extensions load here. Pinned to the loader configuration Pi's shipped
+	// runtimes use — see withoutJitiNativeImport for why that matters.
+	await withoutJitiNativeImport(() => loader.reload());
+
+	// Create isolated ModelRuntime with auth under cwd and no persisted model catalog
+	const modelRuntime = await ModelRuntime.create({
+		authPath: path.join(cwd, "auth.json"),
+		modelsPath: null,
+	});
+
+	// Use a builtin model as placeholder (never actually called — playbook replaces streamFn)
+	const playbookModel = modelRuntime.getModel("openai", "gpt-4o");
+	if (!playbookModel) {
+		throw new Error(
+			"Model openai/gpt-4o not found in isolated ModelRuntime. " +
+				"This should not happen — builtin providers are always registered. " +
+				"Check that @earendil-works/pi-ai is installed.",
+		);
+	}
+
+	// Provide a dummy API key so AgentSession.prompt does not reject before
+	// the playbook replaces streamFunction. The key is never sent to any LLM.
+	// Pi 0.84 synchronizes runtime credentials with an offline model refresh.
+	await modelRuntime.setRuntimeApiKey("openai", "sk-test-harness-dummy");
+
+	const { session, extensionsResult } = await withoutJitiNativeImport(() =>
+		createAgentSession({
+			cwd,
+			agentDir: cwd,
+			model: playbookModel,
+			modelRuntime,
+			sessionManager: SessionManager.inMemory(),
+			settingsManager,
+			resourceLoader: loader,
+		}),
+	);
+
+	if (extensionsResult.errors.length > 0) {
+		session.dispose();
+		if (ownsTmpDir && fs.existsSync(cwd)) {
+			fs.rmSync(cwd, { recursive: true, force: true });
+		}
+		const errors = extensionsResult.errors
+			.map((e) => `  ${e.path}: ${e.error}`)
+			.join("\n");
+		throw new Error(`Extension load errors:\n${errors}`);
+	}
+
+	const events = createEventCollector();
+	let currentStep = 0;
+	let mockedToolNames: ReadonlySet<string> = new Set();
+	// toolCallIds whose mock returned a ToolResult with isError:true — Pi 0.84
+	// hardcodes successful execute() as non-error, so records must consult this.
+	let mockedErrorToolCallIds: ReadonlySet<string> = new Set();
+
+	session.subscribe((event: AgentSessionEvent) => {
+		events.all.push(event);
+
+		if (event.type === "tool_execution_start") {
+			const record: ToolCallRecord = {
+				step: currentStep,
+				toolName: event.toolName,
+				input: (event as any).args ?? {},
+				blocked: false,
+			};
+			events.toolCalls.push(record);
+		}
+
+		if (event.type === "tool_execution_end") {
+			const resultText =
+				event.result?.content
+					?.filter((c: any) => c.type === "text")
+					?.map((c: any) => c.text)
+					?.join("\n") ?? "";
+
+			if (event.isError) {
+				const lastCall = events.toolCalls[events.toolCalls.length - 1];
+				if (lastCall && lastCall.toolName === event.toolName) {
+					if (resultText.includes("blocked") || resultText.includes("Plan mode")) {
+						lastCall.blocked = true;
+						lastCall.blockReason = resultText;
+					}
+				}
+			}
+
+			// Record the final result (after afterToolCall modifications).
+			// Always push — each tool_execution_end has a unique toolCallId
+			// within a run, and the subscriber is the only source of results.
+			const isMocked = mockedToolNames.has(event.toolName);
+			events.toolResults.push({
+				step: currentStep,
+				toolName: event.toolName,
+				toolCallId: event.toolCallId,
+				text: resultText,
+				content: event.result?.content ?? [],
+				isError: event.isError || mockedErrorToolCallIds.has(event.toolCallId),
+				details: event.result?.details,
+				mocked: isMocked,
+			});
+		}
+
+		if (event.type === "message_end") {
+			events.messages.push(event.message);
+		}
+	});
+
+	let playbookState: PlaybookState | null = null;
+
+	const mockUI = createMockUIContext(options.mockUI, events.ui);
+
+	await session.bindExtensions({
+		uiContext: mockUI,
+		onError: (err) => {
+			console.error(
+				`[pi-test-harness] Extension error: ${err.event} — ${err.error}`,
+			);
+		},
+	});
+
+	const originalTools: AgentTool[] = [...session.agent.state.tools];
+
+	const testSession: TestSession = {
+		session,
+		cwd,
+		events,
+
+		get playbook() {
+			return {
+				consumed: playbookState?.consumed ?? 0,
+				remaining: playbookState?.remaining ?? 0,
+			};
+		},
+
+		async run(...turns: Turn[]): Promise<void> {
+			const { streamFn, state } = createPlaybookStreamFn(turns);
+			playbookState = state;
+
+			// Assign playbook streamFn to the public Agent.streamFunction
+			session.agent.streamFunction = streamFn;
+
+			const effectiveMockTools = options.mockTools ?? {};
+			const currentTools = originalTools;
+			const {
+				tools: interceptedTools,
+				mockedNames,
+				mockedErrorToolCallIds: errorIds,
+			} = interceptToolExecution(
+				currentTools,
+				effectiveMockTools,
+				state,
+				propagateErrors,
+			);
+			mockedToolNames = mockedNames;
+			mockedErrorToolCallIds = errorIds;
+			session.agent.state.tools = interceptedTools;
+
+			for (const turn of turns) {
+				currentStep = state.consumed;
+				await session.prompt(turn.prompt);
+				await session.agent.waitForIdle();
+			}
+
+			if (state.remaining > 0) {
+				const allActions = turns.flatMap((t) => t.actions);
+				const remaining = allActions.slice(state.consumed);
+				const diagnostic = formatPlaybookDiagnostic("remaining", state, remaining);
+				throw new Error(diagnostic);
+			}
+		},
+
+		/**
+		 * Dispose the test session and clean up the temp directory (if owned).
+		 *
+		 * Note: `session.dispose()` does NOT fire `session_shutdown`. That event is
+		 * dispatched by pi at Node.js process exit. Extensions that open resources in
+		 * `session_start` (e.g., SQLite databases) keep those resources open until the
+		 * process exit. Use `safeRmSync` when cleaning up extension-owned files in
+		 * afterEach hooks on Windows to avoid EPERM errors.
+		 */
+		dispose(): void {
+			session.dispose();
+			if (ownsTmpDir && fs.existsSync(cwd)) {
+				fs.rmSync(cwd, { recursive: true, force: true });
+			}
+		},
+	};
+
+	return testSession;
+}