@abdwhb-png/pi-test-harness 0.7.0

package/src/diagnostics.ts

@@ -0,0 +1,90 @@
+/**
+ * Playbook diagnostic messages — clear errors when things diverge.
+ */
+
+import type { PlaybookAction } from "./types.js";
+
+function formatAction(action: PlaybookAction): string {
+	if (action.type === "say") {
+		return `says("${action.text?.slice(0, 60)}${(action.text?.length ?? 0) > 60 ? "..." : ""}")`;
+	}
+	if (action.type === "call") {
+		const params = typeof action.params === "function" ? "<late-bound>" : JSON.stringify(action.params);
+		const truncated = params.length > 80 ? params.slice(0, 80) + "..." : params;
+		return `calls("${action.toolName}", ${truncated})`;
+	}
+	return `unknown(${action.type})`;
+}
+
+export function formatPlaybookDiagnostic(
+	type: "exhausted" | "remaining",
+	state: { consumed: number; remaining: number; consumedActions: PlaybookAction[] },
+	remainingActions?: PlaybookAction[],
+): string {
+	if (type === "exhausted") {
+		const last = state.consumedActions[state.consumedActions.length - 1];
+		const lines = [
+			`Playbook exhausted unexpectedly.`,
+			`  Consumed ${state.consumed} action(s).`,
+		];
+		if (last) {
+			lines.push(`  Last consumed: ${formatAction(last)} at step ${state.consumed}`);
+		}
+		lines.push(
+			"",
+			"  The agent loop called streamFn but no more playbook actions were available.",
+			"  This usually means a tool call produced an unexpected result that caused",
+			"  additional streamFn calls (retries, error handling).",
+		);
+		return lines.join("\n");
+	}
+
+	if (type === "remaining" && remainingActions) {
+		const lines = [
+			`Playbook not fully consumed after run() completed.`,
+			`  Consumed ${state.consumed} of ${state.consumed + remainingActions.length} action(s).`,
+			`  Remaining:`,
+		];
+		for (const action of remainingActions.slice(0, 5)) {
+			lines.push(`    - ${formatAction(action)}`);
+		}
+		if (remainingActions.length > 5) {
+			lines.push(`    ... +${remainingActions.length - 5} more`);
+		}
+		lines.push(
+			"",
+			"  The agent loop ended before all playbook actions were used.",
+			"  This usually means a tool was blocked by a hook or returned early,",
+			"  causing fewer streamFn calls than expected.",
+		);
+		return lines.join("\n");
+	}
+
+	return "Unknown playbook diagnostic.";
+}
+
+export function formatToolError(
+	step: number,
+	toolName: string,
+	error: unknown,
+): string {
+	const message = error instanceof Error ? error.message : String(error);
+	const stack = error instanceof Error ? error.stack : undefined;
+	const lines = [
+		`Error during tool execution at playbook step ${step} (call "${toolName}"):`,
+		`  ${message}`,
+	];
+	if (stack) {
+		const stackLines = stack.split("\n").slice(1, 4);
+		for (const line of stackLines) {
+			lines.push(`  ${line.trim()}`);
+		}
+	}
+	lines.push(
+		"",
+		"This error was thrown by the real tool execution, not by the playbook.",
+		"To capture errors as tool results instead of aborting, set:",
+		"  createTestSession({ propagateErrors: false })",
+	);
+	return lines.join("\n");
+}

package/src/events.ts

@@ -0,0 +1,43 @@
+/**
+ * Event collection — passively collects all events during a test run.
+ */
+
+import type { AgentSessionEvent } from "@earendil-works/pi-coding-agent";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import type { TestEvents, ToolCallRecord, ToolResultRecord, UICallRecord } from "./types.js";
+
+export function createEventCollector(): TestEvents {
+	const all: AgentSessionEvent[] = [];
+	const toolCalls: ToolCallRecord[] = [];
+	const toolResults: ToolResultRecord[] = [];
+	const messages: AgentMessage[] = [];
+	const ui: UICallRecord[] = [];
+
+	return {
+		all,
+		toolCalls,
+		toolResults,
+		messages,
+		ui,
+
+		toolCallsFor(name: string): ToolCallRecord[] {
+			return toolCalls.filter((tc) => tc.toolName === name);
+		},
+
+		toolResultsFor(name: string): ToolResultRecord[] {
+			return toolResults.filter((tr) => tr.toolName === name);
+		},
+
+		blockedCalls(): ToolCallRecord[] {
+			return toolCalls.filter((tc) => tc.blocked);
+		},
+
+		uiCallsFor(method: string): UICallRecord[] {
+			return ui.filter((u) => u.method === method);
+		},
+
+		toolSequence(): string[] {
+			return toolCalls.map((tc) => tc.toolName);
+		},
+	};
+}

package/src/index.ts

@@ -0,0 +1,42 @@
+/**
+ * @abdwhb-png/pi-test-harness
+ *
+ * Test harness for pi extensions — playbook-based model mocking,
+ * session testing, sandbox install verification.
+ */
+
+// DSL builders
+export { when, calls, says } from "./playbook.js";
+
+// Session
+export { createTestSession } from "./session.js";
+
+// Sandbox
+export { verifySandboxInstall } from "./sandbox.js";
+
+// Mock Pi
+export { createMockPi } from "./mock-pi.js";
+
+// Types
+export type {
+	TestSession,
+	TestSessionOptions,
+	TestEvents,
+	ToolCallRecord,
+	ToolResultRecord,
+	UICallRecord,
+	MockToolHandler,
+	MockUIConfig,
+	SandboxOptions,
+	SandboxResult,
+	MockPi,
+	MockPiCall,
+	Turn,
+	PlaybookAction,
+} from "./types.js";
+
+// Errors
+export { ToolBlockedError } from "./mock-tools.js";
+
+// Utilities
+export { safeRmSync } from "./utils.js";

package/src/mock-pi-script.mjs

@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+/**
+ * Mock pi CLI for integration tests.
+ *
+ * Reads response queue from MOCK_PI_QUEUE_DIR (set by the shim created by createMockPi).
+ * Each invocation consumes the next entry from the queue. When the queue is exhausted,
+ * the last entry repeats. If no entries are queued, outputs a default echo response.
+ *
+ * Queue protocol:
+ *   {queueDir}/queue.json  — JSON array of MockPiCall objects
+ *   {queueDir}/counter     — integer: current call index (auto-incremented)
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+// Safety timeout — prevent hanging tests if something goes wrong
+const TIMEOUT_MS = 30_000;
+setTimeout(() => {
+	process.stderr.write("mock-pi-script: timeout after 30s\n");
+	process.exit(124);
+}, TIMEOUT_MS).unref();
+
+// ---------------------------------------------------------------------------
+// Queue directory (set by the shim script)
+// ---------------------------------------------------------------------------
+const queueDir = process.env.MOCK_PI_QUEUE_DIR;
+if (!queueDir) {
+	process.stderr.write("mock-pi-script: MOCK_PI_QUEUE_DIR not set\n");
+	process.exit(99);
+}
+
+// ---------------------------------------------------------------------------
+// Parse CLI arguments (matches what pi-subagents passes to pi)
+// ---------------------------------------------------------------------------
+const args = process.argv.slice(2);
+let task = "";
+let sessionDir = null;
+
+let i = 0;
+while (i < args.length) {
+	const arg = args[i];
+
+	// Flags with a value — skip the value
+	if (
+		arg === "--session-dir" ||
+		arg === "--mode" ||
+		arg === "--models" ||
+		arg === "--tools" ||
+		arg === "--extension" ||
+		arg === "--append-system-prompt"
+	) {
+		if (arg === "--session-dir") sessionDir = args[i + 1] ?? null;
+		i += 2;
+		continue;
+	}
+
+	// Flags without a value
+	if (arg === "-p" || arg === "--no-session" || arg === "--no-extensions") {
+		i++;
+		continue;
+	}
+
+	// @file — read task from file
+	if (arg?.startsWith("@")) {
+		try {
+			task = fs.readFileSync(arg.slice(1), "utf-8");
+		} catch {
+			task = "(could not read " + arg.slice(1) + ")";
+		}
+		i++;
+		continue;
+	}
+
+	// Positional — treat as task text
+	if (arg && !arg.startsWith("-")) {
+		task = arg;
+	}
+	i++;
+}
+
+// ---------------------------------------------------------------------------
+// Read queue and counter
+// ---------------------------------------------------------------------------
+let queue = [];
+const queueFile = path.join(queueDir, "queue.json");
+if (fs.existsSync(queueFile)) {
+	try {
+		queue = JSON.parse(fs.readFileSync(queueFile, "utf-8"));
+	} catch {
+		// Malformed queue — treat as empty
+	}
+}
+
+const counterFile = path.join(queueDir, "counter");
+let counter = 0;
+if (fs.existsSync(counterFile)) {
+	try {
+		counter = parseInt(fs.readFileSync(counterFile, "utf-8").trim(), 10) || 0;
+	} catch {
+		// Missing or unreadable — start at 0
+	}
+}
+
+// Increment counter for next invocation
+fs.writeFileSync(counterFile, String(counter + 1));
+
+// Get the current entry (repeat last when exhausted, null if no queue)
+const entry =
+	queue.length > 0 ? queue[Math.min(counter, queue.length - 1)] : null;
+
+// ---------------------------------------------------------------------------
+// Delay
+// ---------------------------------------------------------------------------
+if (entry?.delay > 0) {
+	await new Promise((r) => setTimeout(r, entry.delay));
+}
+
+// ---------------------------------------------------------------------------
+// Stderr
+// ---------------------------------------------------------------------------
+if (entry?.stderr) {
+	process.stderr.write(entry.stderr + "\n");
+}
+
+// ---------------------------------------------------------------------------
+// Write files (for chain_dir output simulation)
+// ---------------------------------------------------------------------------
+if (entry?.writeFiles) {
+	for (const [filePath, content] of Object.entries(entry.writeFiles)) {
+		fs.mkdirSync(path.dirname(filePath), { recursive: true });
+		fs.writeFileSync(filePath, content);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// JSONL output
+// ---------------------------------------------------------------------------
+if (entry?.jsonl) {
+	for (const event of entry.jsonl) {
+		console.log(typeof event === "string" ? event : JSON.stringify(event));
+	}
+} else {
+	const taskClean = task.replace(/^Task:\s*/i, "").slice(0, 500);
+	const output = entry?.output ?? "Mock output for: " + taskClean;
+	console.log(
+		JSON.stringify({
+			type: "message_end",
+			message: {
+				role: "assistant",
+				content: [{ type: "text", text: output }],
+				model: "mock/test-model",
+				usage: {
+					input: 100,
+					output: 50,
+					cacheRead: 0,
+					cacheWrite: 0,
+					cost: { total: 0.001 },
+				},
+			},
+		}),
+	);
+}
+
+// ---------------------------------------------------------------------------
+// Session file (if requested)
+// ---------------------------------------------------------------------------
+if (sessionDir) {
+	fs.mkdirSync(sessionDir, { recursive: true });
+	fs.writeFileSync(
+		path.join(sessionDir, "session-" + Date.now() + ".jsonl"),
+		JSON.stringify({ type: "session_start" }) + "\n",
+	);
+}
+
+process.exit(entry?.exitCode ?? 0);

package/src/mock-pi.ts

@@ -0,0 +1,169 @@
+/**
+ * createMockPi — mock pi CLI for testing extensions that spawn pi as a subprocess.
+ *
+ * Creates a temp directory with a `pi` shim (`.cmd` on Windows, shell script on Linux)
+ * that prepends to PATH. The shim invokes a mock-pi-script.mjs that reads queued
+ * responses from a file-based queue. Responses are consumed in order; the last one
+ * repeats when the queue is exhausted.
+ *
+ * ```ts
+ * const mockPi = createMockPi();
+ * mockPi.install();
+ *
+ * mockPi.onCall({ output: "Hello from agent" });
+ * mockPi.onCall({ stderr: "crashed", exitCode: 1 });
+ *
+ * // ... test code that spawns pi ...
+ *
+ * mockPi.reset();
+ * mockPi.uninstall();
+ * ```
+ */
+
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+import type { MockPi, MockPiCall } from "./types.js";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+/** Valid keys for MockPiCall — used for runtime validation. */
+const VALID_MOCK_PI_CALL_KEYS = new Set([
+	"output",
+	"exitCode",
+	"stderr",
+	"delay",
+	"jsonl",
+	"writeFiles",
+]);
+
+/**
+ * Resolve the mock-pi-script.mjs path.
+ *
+ * During dev (vitest): src/mock-pi.ts → sibling src/mock-pi-script.mjs
+ * After build (dist):  dist/mock-pi.js → ../src/mock-pi-script.mjs
+ */
+function findMockPiScript(): string {
+	// Direct sibling (running from src/ via vitest)
+	const sibling = path.join(__dirname, "mock-pi-script.mjs");
+	if (fs.existsSync(sibling)) return sibling;
+
+	// One level up to package root, then into src/ (running from dist/)
+	const fromDist = path.join(__dirname, "..", "src", "mock-pi-script.mjs");
+	if (fs.existsSync(fromDist)) return fromDist;
+
+	throw new Error(
+		"Could not find mock-pi-script.mjs. Searched:\n" +
+			`  ${sibling}\n` +
+			`  ${fromDist}`,
+	);
+}
+
+/**
+ * Create a mock pi CLI for testing extensions that spawn pi as a subprocess.
+ *
+ * **Concurrency constraint**: Designed for serial subprocess spawns within a single test.
+ * If your test spawns multiple pi processes concurrently, responses may be consumed
+ * out of order. Use separate `createMockPi()` instances for concurrent scenarios,
+ * or ensure your test logic doesn't depend on response ordering.
+ */
+export function createMockPi(): MockPi {
+	const dir = fs.mkdtempSync(path.join(os.tmpdir(), "pi-mock-"));
+	let originalPath: string | undefined;
+	let installed = false;
+
+	const queueFile = path.join(dir, "queue.json");
+	const counterFile = path.join(dir, "counter");
+
+	// Initialize empty queue
+	fs.writeFileSync(queueFile, "[]");
+	fs.writeFileSync(counterFile, "0");
+
+	const scriptPath = findMockPiScript();
+	const nodeExe = process.execPath;
+
+	// Safety net: restore PATH if process exits without uninstall()
+	const exitHandler = () => {
+		if (originalPath !== undefined) {
+			process.env.PATH = originalPath;
+		}
+	};
+
+	return {
+		dir,
+
+		install() {
+			if (installed) return;
+			originalPath = process.env.PATH;
+
+			if (process.platform === "win32") {
+				// Windows: .cmd batch file shim
+				const cmd = [
+					"@echo off",
+					`set "MOCK_PI_QUEUE_DIR=${dir}"`,
+					`"${nodeExe}" "${scriptPath}" %*`,
+				].join("\r\n") + "\r\n";
+				fs.writeFileSync(path.join(dir, "pi.cmd"), cmd);
+			} else {
+				// Linux/macOS: shell script shim
+				const sh = [
+					"#!/bin/sh",
+					`MOCK_PI_QUEUE_DIR="${dir}" exec "${nodeExe}" "${scriptPath}" "$@"`,
+				].join("\n") + "\n";
+				const piPath = path.join(dir, "pi");
+				fs.writeFileSync(piPath, sh);
+				fs.chmodSync(piPath, 0o755);
+			}
+
+			process.env.PATH = `${dir}${path.delimiter}${originalPath}`;
+			process.on("exit", exitHandler);
+			installed = true;
+		},
+
+		uninstall() {
+			if (!installed) return;
+			process.removeListener("exit", exitHandler);
+			if (originalPath !== undefined) {
+				process.env.PATH = originalPath;
+				originalPath = undefined;
+			}
+			try {
+				fs.rmSync(dir, { recursive: true, force: true });
+			} catch {
+				// Best-effort cleanup
+			}
+			installed = false;
+		},
+
+		onCall(response: MockPiCall) {
+			// Validate keys to catch typos early
+			const unknown = Object.keys(response).filter(
+				(k) => !VALID_MOCK_PI_CALL_KEYS.has(k),
+			);
+			if (unknown.length > 0) {
+				throw new Error(
+					`Unknown MockPiCall key(s): ${unknown.join(", ")}. ` +
+						`Valid keys: ${[...VALID_MOCK_PI_CALL_KEYS].join(", ")}`,
+				);
+			}
+
+			const queue: MockPiCall[] = JSON.parse(fs.readFileSync(queueFile, "utf-8"));
+			queue.push(response);
+			fs.writeFileSync(queueFile, JSON.stringify(queue));
+		},
+
+		reset() {
+			fs.writeFileSync(queueFile, "[]");
+			fs.writeFileSync(counterFile, "0");
+		},
+
+		callCount(): number {
+			try {
+				return parseInt(fs.readFileSync(counterFile, "utf-8").trim(), 10) || 0;
+			} catch {
+				return 0;
+			}
+		},
+	};
+}