@abdwhb-png/pi-test-harness 0.7.0

package/src/mock-tools.ts

@@ -0,0 +1,252 @@
+/**
+ * Tool execution interceptor — wraps tool.execute() for tools in mockTools.
+ *
+ * For mocked tools, the mock replaces tool.execute() and returns controlled
+ * values. Extension hooks (tool_call / tool_result) are handled by
+ * AgentSession 0.83's internal beforeToolCall/afterToolCall — the mock
+ * must NOT re-emit them.
+ *
+ * For non-mocked tools, the real execute() is called and results are
+ * collected for event queries.
+ */
+
+import type { AgentTool } from "@earendil-works/pi-agent-core";
+import type { MockToolHandler, ToolResult, ToolResultRecord } from "./types.js";
+import type { PlaybookState } from "./playbook.js";
+import { formatToolError } from "./diagnostics.js";
+
+/**
+ * Thrown when an extension hook blocks a tool call.
+ * Exported for test assertions — no longer thrown by the mock itself since
+ * AgentSession 0.83's beforeToolCall handles blocking before execute().
+ */
+export class ToolBlockedError extends Error {
+	readonly toolBlocked = true as const;
+
+	constructor(reason: string) {
+		super(reason);
+		this.name = "ToolBlockedError";
+	}
+}
+
+/**
+ * Returns true if `err` represents a hook-based tool block.
+ *
+ * Kept for consumers that catch errors from tool execution flows, though
+ * AgentSession 0.83's beforeToolCall blocks before execute() is reached.
+ */
+export function isBlockedError(err: unknown): boolean {
+	if (err instanceof ToolBlockedError) return true;
+	if (err instanceof Error) {
+		const msg = err.message;
+		return (
+			msg.includes("blocked") ||
+			msg.includes("Plan mode") ||
+			msg.includes("WRITE operation")
+		);
+	}
+	return false;
+}
+
+function normalizeMockResult(
+	handler: MockToolHandler,
+	params: Record<string, unknown>,
+): ToolResult {
+	let raw: string | ToolResult;
+
+	if (typeof handler === "string") {
+		raw = handler;
+	} else if (typeof handler === "function") {
+		raw = handler(params);
+	} else {
+		raw = handler;
+	}
+
+	if (typeof raw === "string") {
+		return {
+			content: [{ type: "text", text: raw }],
+			details: {},
+		};
+	}
+
+	return raw;
+}
+
+/**
+ * Intercept tool execution for mocked tools.
+ *
+ * Unlike the old approach, this does NOT emit tool_call/tool_result hooks
+ * manually — AgentSession 0.83's beforeToolCall/afterToolCall handles that.
+ * The mock only replaces execute() to return controlled values. Result
+ * recording is handled by the session subscriber from tool_execution_end
+ * events (which carry the final afterToolCall-modified result).
+ *
+ * Returns a Set of mocked tool names for the session subscriber to set the
+ * mocked flag on recorded results, plus a Set of toolCallIds whose mock
+ * returned a ToolResult with isError:true. Pi 0.84's agent loop hardcodes
+ * successful execute() as non-error (isError:false), so the subscriber must
+ * consult this set to preserve the mock's error intent in collected records.
+ */
+export function interceptToolExecution(
+	tools: AgentTool[],
+	mockTools: Record<string, MockToolHandler>,
+	playbookState: PlaybookState,
+	propagateErrors: boolean,
+): {
+	tools: AgentTool[];
+	mockedNames: ReadonlySet<string>;
+	mockedErrorToolCallIds: ReadonlySet<string>;
+} {
+	const mockedNames = new Set(Object.keys(mockTools));
+	const mockedErrorToolCallIds = new Set<string>();
+
+	const wrapped = tools.map((tool) => {
+		const mockHandler = mockTools[tool.name];
+		if (!mockHandler) {
+			return wrapForCollection(tool, playbookState, propagateErrors);
+		}
+
+		return {
+			...tool,
+			execute: async (
+				toolCallId: string,
+				params: Record<string, unknown>,
+				_signal?: AbortSignal,
+				_onUpdate?: any,
+			) => {
+				const result = normalizeMockResult(mockHandler, params);
+				const text = result.content
+					.filter((c) => c.type === "text")
+					.map((c) => c.text)
+					.join("\n");
+				if (result.isError) {
+					// Pi 0.84 hardcodes successful execute() as non-error; remember the
+					// toolCallId so the session subscriber can flag the record.
+					mockedErrorToolCallIds.add(toolCallId);
+				}
+				// fireThenCallback fires synchronously so .then() sees real data
+				fireThenCallback(playbookState, toolCallId, {
+					step: playbookState.consumed,
+					toolName: tool.name,
+					toolCallId,
+					text,
+					content: result.content,
+					isError: result.isError ?? false,
+					details: result.details,
+					mocked: true,
+				});
+
+				return {
+					content: result.content,
+					details: result.details ?? {},
+				};
+			},
+		} as AgentTool;
+	});
+
+	return { tools: wrapped, mockedNames, mockedErrorToolCallIds };
+}
+
+/**
+ * Wrap a real tool for event collection (non-mocked tools).
+ * Does not push to toolResults — the session subscriber handles recording
+ * from tool_execution_end events.
+ */
+function wrapForCollection(
+	tool: AgentTool,
+	playbookState: PlaybookState,
+	propagateErrors: boolean,
+): AgentTool {
+	const originalExecute = tool.execute;
+
+	return {
+		...tool,
+		execute: async (
+			toolCallId: string,
+			params: Record<string, unknown>,
+			signal?: AbortSignal,
+			onUpdate?: any,
+		) => {
+			const step = playbookState.consumed;
+
+			try {
+				const result = await originalExecute.call(
+					tool,
+					toolCallId,
+					params,
+					signal,
+					onUpdate,
+				);
+
+				const text = (result.content ?? [])
+					.filter((c: any) => c.type === "text")
+					.map((c: any) => c.text)
+					.join("\n");
+
+				fireThenCallback(playbookState, toolCallId, {
+					step,
+					toolName: tool.name,
+					toolCallId,
+					text,
+					content: result.content ?? [],
+					isError: !!(result as any).isError,
+					details: result.details,
+					mocked: false,
+				});
+
+				return result;
+			} catch (err) {
+				const errMsg = err instanceof Error ? err.message : String(err);
+
+				try {
+					fireThenCallback(playbookState, toolCallId, {
+						step,
+						toolName: tool.name,
+						toolCallId,
+						text: errMsg,
+						content: [{ type: "text", text: errMsg }],
+						isError: true,
+						details: undefined,
+						mocked: false,
+					});
+				} catch {
+					/* best-effort callback */
+				}
+
+				if (propagateErrors) {
+					const diagnostic = formatToolError(step, tool.name, err);
+					throw new Error(diagnostic, { cause: err });
+				}
+
+				return {
+					content: [{ type: "text", text: errMsg }],
+					details: {},
+					isError: true,
+				};
+			}
+		},
+	} as AgentTool;
+}
+
+function fireThenCallback(
+	state: PlaybookState,
+	toolCallId: string,
+	record: ToolResultRecord,
+): void {
+	const callback =
+		state.pendingCallbacks.get(toolCallId) ??
+		state.pendingCallbacks.get(record.toolName);
+	const key = state.pendingCallbacks.has(toolCallId)
+		? toolCallId
+		: record.toolName;
+	if (callback) {
+		state.pendingCallbacks.delete(key);
+		try {
+			callback(record);
+		} catch (err) {
+			console.warn(
+				`[pi-test-harness] .then() callback error for ${record.toolName}: ${err}`,
+			);
+		}
+	}
+}

package/src/mock-ui.ts

@@ -0,0 +1,196 @@
+/**
+ * Mock UI context — intercepts ctx.ui.* calls from extensions.
+ * All calls are collected for assertions. Interactive methods return
+ * configured mock responses.
+ */
+
+import type { ExtensionUIContext } from "@earendil-works/pi-coding-agent";
+import type { MockUIConfig, UICallRecord } from "./types.js";
+
+/**
+ * Create a mock ExtensionUIContext that records all calls and returns
+ * configured responses.
+ */
+export function createMockUIContext(
+	config: MockUIConfig = {},
+	uiLog: UICallRecord[],
+): ExtensionUIContext {
+	function record(
+		method: string,
+		args: unknown[],
+		returnValue?: unknown,
+	): void {
+		uiLog.push({ method, args, returnValue });
+	}
+
+	const mockUI: ExtensionUIContext = {
+		async select(
+			title: string,
+			options: string[],
+			_opts?: any,
+		): Promise<string | undefined> {
+			let result: string | undefined;
+			const handler = config.select;
+			if (handler === undefined || handler === null) {
+				result = options[0]; // default: first item
+			} else if (typeof handler === "number") {
+				result = options[handler];
+			} else if (typeof handler === "string") {
+				result = options.find((o) => o === handler) ?? options[0];
+			} else if (typeof handler === "function") {
+				result = handler(title, options);
+			}
+			record("select", [title, options], result);
+			return result;
+		},
+
+		async confirm(
+			title: string,
+			message: string,
+			_opts?: any,
+		): Promise<boolean> {
+			let result: boolean;
+			const handler = config.confirm;
+			if (handler === undefined || handler === null) {
+				result = true; // default: approve
+			} else if (typeof handler === "boolean") {
+				result = handler;
+			} else if (typeof handler === "function") {
+				result = handler(title, message);
+			} else {
+				result = true;
+			}
+			record("confirm", [title, message], result);
+			return result;
+		},
+
+		async input(
+			title: string,
+			placeholder?: string,
+			_opts?: any,
+		): Promise<string | undefined> {
+			let result: string | undefined;
+			const handler = config.input;
+			if (handler === undefined || handler === null) {
+				result = "";
+			} else if (typeof handler === "string") {
+				result = handler;
+			} else if (typeof handler === "function") {
+				result = handler(title, placeholder);
+			}
+			record("input", [title, placeholder], result);
+			return result;
+		},
+
+		async editor(title: string, prefill?: string): Promise<string | undefined> {
+			let result: string | undefined;
+			const handler = config.editor;
+			if (handler === undefined || handler === null) {
+				result = "";
+			} else if (typeof handler === "string") {
+				result = handler;
+			} else if (typeof handler === "function") {
+				result = handler(title, prefill);
+			}
+			record("editor", [title, prefill], result);
+			return result;
+		},
+
+		notify(message: string, type?: "info" | "warning" | "error"): void {
+			record("notify", [message, type]);
+		},
+
+		onTerminalInput(): () => void {
+			return () => {};
+		},
+
+		setStatus(key: string, text: string | undefined): void {
+			record("setStatus", [key, text]);
+		},
+
+		setWorkingMessage(message?: string): void {
+			record("setWorkingMessage", [message]);
+		},
+
+		setWorkingVisible(visible: boolean): void {
+			record("setWorkingVisible", [visible]);
+		},
+
+		setWorkingIndicator(options?: {
+			frames?: string[];
+			intervalMs?: number;
+		}): void {
+			record("setWorkingIndicator", [options]);
+		},
+
+		setHiddenThinkingLabel(label?: string): void {
+			record("setHiddenThinkingLabel", [label]);
+		},
+
+		setWidget(key: string, content: any, _options?: any): void {
+			record("setWidget", [key, content]);
+		},
+
+		setFooter(...args: unknown[]): void {
+			record("setFooter", args);
+		},
+		setHeader(...args: unknown[]): void {
+			record("setHeader", args);
+		},
+
+		setTitle(title: string): void {
+			record("setTitle", [title]);
+		},
+
+		async custom<T>(): Promise<T> {
+			return undefined as never;
+		},
+
+		pasteToEditor(text: string): void {
+			record("pasteToEditor", [text]);
+		},
+		setEditorText(text: string): void {
+			record("setEditorText", [text]);
+		},
+		getEditorText(): string {
+			return "";
+		},
+
+		setEditorComponent(factory: any): void {
+			record("setEditorComponent", [factory]);
+		},
+
+		addAutocompleteProvider(factory: any): void {
+			record("addAutocompleteProvider", [factory]);
+		},
+
+		getEditorComponent(): any | undefined {
+			return undefined;
+		},
+
+		get theme(): any {
+			return {
+				fg: (_color: string, text: string) => text,
+				bold: (text: string) => text,
+				italic: (text: string) => text,
+				strikethrough: (text: string) => text,
+			};
+		},
+
+		getAllThemes(): any[] {
+			return [];
+		},
+		getTheme(): any {
+			return undefined;
+		},
+		setTheme(): any {
+			return { success: false, error: "Test mode" };
+		},
+		getToolsExpanded(): boolean {
+			return false;
+		},
+		setToolsExpanded(): void {},
+	};
+
+	return mockUI;
+}

package/src/pi-loader-parity.ts

@@ -0,0 +1,61 @@
+/**
+ * Extension-loader parity with Pi's shipped runtimes.
+ */
+
+/**
+ * Environment variable jiti reads when it decides whether to try the runtime's
+ * native require/import before applying its own transform.
+ * @internal
+ */
+const JITI_TRY_NATIVE = "JITI_TRY_NATIVE";
+
+/** Nesting depth of `withoutJitiNativeImport` calls in this process. */
+let suppressionDepth = 0;
+
+/**
+ * Run `fn` with jiti's native import/require fast-path disabled.
+ *
+ * **Why this exists**: Pi loads every extension entrypoint through jiti
+ * (`loadExtensionModule` in `core/extensions/loader.ts`). Which jiti options it
+ * passes depends on how Pi itself is running:
+ *
+ * ```ts
+ * isBunBinary || isNodeSeaBinary || isBundledNode
+ *   ? { virtualModules, tryNative: false } // compiled binary / bundled Node
+ *   : isTypeScriptSourceRuntime
+ *     ? { virtualModules, tsconfigPaths: true }
+ *     : { alias: getAliases() } // unbundled Node build
+ * ```
+ *
+ * The last branch — the one an in-process harness always takes, because it
+ * imports the package's compiled entrypoint — leaves `tryNative` at jiti's
+ * default, and that default is "enabled if Bun is detected". Under a Bun test
+ * runner this combination hands Pi a factory for a module whose body has not
+ * finished evaluating: an entrypoint with a module-level `await import(...)`
+ * reports as loaded before its top-level `const` is initialized, so the hoisted
+ * default export Pi calls throws `Cannot access 'x' before initialization`.
+ *
+ * Pi's shipped runtimes never take that path, so the harness pins the choice
+ * they make (`tryNative: false`) for the duration of the load. This keeps
+ * `extensionFactories`-style and path-loaded extensions in exactly the loader
+ * configuration the real CLI uses, instead of one that varies by test runner.
+ *
+ * Nesting is reference-counted so two sessions loading concurrently in one
+ * process cannot have one restore the variable while the other is mid-load.
+ */
+export async function withoutJitiNativeImport<T>(
+ fn: () => Promise<T>,
+): Promise<T> {
+ const previous = process.env[JITI_TRY_NATIVE];
+ suppressionDepth += 1;
+ process.env[JITI_TRY_NATIVE] = "0";
+ try {
+  return await fn();
+ } finally {
+  suppressionDepth -= 1;
+  if (suppressionDepth === 0) {
+   if (previous === undefined) delete process.env[JITI_TRY_NATIVE];
+   else process.env[JITI_TRY_NATIVE] = previous;
+  }
+ }
+}

package/src/playbook.ts

@@ -0,0 +1,189 @@
+/**
+ * PlaybookStreamFn — replaces the model with scripted responses.
+ *
+ * The playbook is a queue of actions. Each streamFn call dequeues the next
+ * action and returns it as an AssistantMessageEventStream.
+ */
+
+import {
+	createAssistantMessageEventStream,
+	type AssistantMessage,
+	type AssistantMessageEventStream,
+	type Context,
+	type Model,
+	type SimpleStreamOptions,
+} from "@earendil-works/pi-ai";
+import type { PlaybookAction, Turn, ToolResultRecord } from "./types.js";
+import { formatPlaybookDiagnostic } from "./diagnostics.js";
+
+// ── DSL builders ────────────────────────────────────────────
+
+/** Chainable call action builder */
+class CallAction {
+	readonly action: PlaybookAction;
+
+	constructor(toolName: string, params: Record<string, unknown> | (() => Record<string, unknown>)) {
+		this.action = { type: "call", toolName, params };
+	}
+
+	then(callback: (result: ToolResultRecord) => void): CallAction {
+		this.action.thenCallback = callback;
+		return this;
+	}
+}
+
+/**
+ * The model calls a tool.
+ * @param toolName Tool to call
+ * @param params Static params or function for late binding
+ */
+export function calls(
+	toolName: string,
+	params: Record<string, unknown> | (() => Record<string, unknown>) = {},
+): CallAction {
+	return new CallAction(toolName, params);
+}
+
+/**
+ * The model emits text. Agent loop ends for this turn.
+ */
+export function says(text: string): PlaybookAction {
+	return { type: "say", text };
+}
+
+/**
+ * Define one user→model turn.
+ * @param prompt The actual user prompt text
+ * @param actions What the model does in response (call/say sequence)
+ */
+export function when(prompt: string, actions: Array<CallAction | PlaybookAction>): Turn {
+	return {
+		prompt,
+		actions: actions.map((a) => (a instanceof CallAction ? a.action : a)),
+	};
+}
+
+// ── PlaybookStreamFn ────────────────────────────────────────
+
+function resolveParams(params: Record<string, unknown> | (() => Record<string, unknown>) | undefined): Record<string, unknown> {
+	if (!params) return {};
+	if (typeof params === "function") return params();
+	return params;
+}
+
+function createAssistantMessage(action: PlaybookAction, toolCallCounter: number): AssistantMessage {
+	const content: AssistantMessage["content"] = [];
+
+	if (action.type === "say") {
+		content.push({ type: "text", text: action.text ?? "" });
+	} else if (action.type === "call") {
+		const resolvedParams = resolveParams(action.params);
+		content.push({
+			type: "toolCall",
+			id: `playbook-tc-${toolCallCounter}`,
+			name: action.toolName!,
+			arguments: resolvedParams,
+		});
+	}
+
+	return {
+		role: "assistant",
+		content,
+		api: "openai-responses",
+		provider: "test",
+		model: "playbook",
+		usage: {
+			input: 0,
+			output: 0,
+			cacheRead: 0,
+			cacheWrite: 0,
+			totalTokens: 0,
+			cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+		},
+		stopReason: action.type === "call" ? "toolUse" : "stop",
+		timestamp: Date.now(),
+	};
+}
+
+export interface PlaybookState {
+	consumed: number;
+	remaining: number;
+	/** The action objects for each consumed step (for diagnostics) */
+	consumedActions: PlaybookAction[];
+	/** Callbacks pending for completed tool calls */
+	pendingCallbacks: Map<string, (result: ToolResultRecord) => void>;
+}
+
+export function createPlaybookStreamFn(turns: Turn[]): {
+	streamFn: (model: Model<any>, context: Context, options?: SimpleStreamOptions) => AssistantMessageEventStream;
+	state: PlaybookState;
+} {
+	// Flatten all turns into a single action queue
+	const queue: PlaybookAction[] = [];
+	for (const turn of turns) {
+		queue.push(...turn.actions);
+	}
+
+	const state: PlaybookState = {
+		consumed: 0,
+		remaining: queue.length,
+		consumedActions: [],
+		pendingCallbacks: new Map(),
+	};
+
+	let toolCallCounter = 0;
+
+	const streamFn = (
+		_model: Model<any>,
+		_context: Context,
+		_options?: SimpleStreamOptions,
+	): AssistantMessageEventStream => {
+		const stream = createAssistantMessageEventStream();
+		const action = queue.shift();
+
+		if (!action) {
+			// Playbook exhausted
+			const diagnostic = formatPlaybookDiagnostic("exhausted", state);
+			const fallback: AssistantMessage = {
+				role: "assistant",
+				content: [{ type: "text", text: `[PLAYBOOK EXHAUSTED] ${diagnostic}` }],
+				api: "openai-responses",
+				provider: "test",
+				model: "playbook",
+				usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
+				stopReason: "stop",
+				timestamp: Date.now(),
+			};
+			queueMicrotask(() => {
+				stream.push({ type: "done", reason: "stop", message: fallback });
+			});
+			return stream;
+		}
+
+		state.consumed++;
+		state.remaining = queue.length;
+		state.consumedActions.push(action);
+
+		if (action.type === "call") toolCallCounter++;
+		const message = createAssistantMessage(action, toolCallCounter);
+
+		// Register callback if present (keyed by tool call ID for uniqueness)
+		if (action.type === "call" && action.thenCallback) {
+			const tcContent = message.content.find((c) => c.type === "toolCall");
+			const tcId = tcContent && "id" in tcContent ? (tcContent as any).id : action.toolName!;
+			state.pendingCallbacks.set(tcId, action.thenCallback);
+		}
+
+		queueMicrotask(() => {
+			stream.push({
+				type: "done",
+				reason: message.stopReason === "toolUse" ? "toolUse" : "stop",
+				message,
+			});
+		});
+
+		return stream;
+	};
+
+	return { streamFn, state };
+}