@abdwhb-png/pi-test-harness 0.7.0

package/dist/session.js

@@ -0,0 +1,187 @@
+/**
+ * TestSession — orchestrates a test run with playbook, mock tools, and mock UI.
+ *
+ * 1. Creates a real pi environment (extensions, tools, hooks, session)
+ * 2. Replaces streamFn with playbook
+ * 3. Intercepts tool.execute() for mockTools
+ * 4. Injects mock UI context
+ * 5. Collects events
+ * 6. Runs conversation script
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import { createAgentSession, DefaultResourceLoader, SessionManager, SettingsManager, ModelRuntime, } from "@earendil-works/pi-coding-agent";
+import { createPlaybookStreamFn } from "./playbook.js";
+import { interceptToolExecution } from "./mock-tools.js";
+import { createMockUIContext } from "./mock-ui.js";
+import { createEventCollector } from "./events.js";
+import { formatPlaybookDiagnostic } from "./diagnostics.js";
+import { withoutJitiNativeImport } from "./pi-loader-parity.js";
+export async function createTestSession(options = {}) {
+    const propagateErrors = options.propagateErrors ?? true;
+    const ownsTmpDir = !options.cwd;
+    const cwd = options.cwd ?? fs.mkdtempSync(path.join(os.tmpdir(), "pi-test-harness-"));
+    if (!fs.existsSync(cwd)) {
+        fs.mkdirSync(cwd, { recursive: true });
+    }
+    const settingsManager = SettingsManager.inMemory();
+    const loader = new DefaultResourceLoader({
+        cwd,
+        agentDir: cwd, // Use cwd as agent dir to avoid touching real ~/.pi
+        settingsManager,
+        additionalExtensionPaths: options.extensions?.map((p) => path.resolve(cwd, p)) ?? [],
+        extensionFactories: options.extensionFactories,
+        systemPromptOverride: options.systemPrompt
+            ? () => options.systemPrompt
+            : undefined,
+    });
+    // Extensions load here. Pinned to the loader configuration Pi's shipped
+    // runtimes use — see withoutJitiNativeImport for why that matters.
+    await withoutJitiNativeImport(() => loader.reload());
+    // Create isolated ModelRuntime with auth under cwd and no persisted model catalog
+    const modelRuntime = await ModelRuntime.create({
+        authPath: path.join(cwd, "auth.json"),
+        modelsPath: null,
+    });
+    // Use a builtin model as placeholder (never actually called — playbook replaces streamFn)
+    const playbookModel = modelRuntime.getModel("openai", "gpt-4o");
+    if (!playbookModel) {
+        throw new Error("Model openai/gpt-4o not found in isolated ModelRuntime. " +
+            "This should not happen — builtin providers are always registered. " +
+            "Check that @earendil-works/pi-ai is installed.");
+    }
+    // Provide a dummy API key so AgentSession.prompt does not reject before
+    // the playbook replaces streamFunction. The key is never sent to any LLM.
+    // Pi 0.84 synchronizes runtime credentials with an offline model refresh.
+    await modelRuntime.setRuntimeApiKey("openai", "sk-test-harness-dummy");
+    const { session, extensionsResult } = await withoutJitiNativeImport(() => createAgentSession({
+        cwd,
+        agentDir: cwd,
+        model: playbookModel,
+        modelRuntime,
+        sessionManager: SessionManager.inMemory(),
+        settingsManager,
+        resourceLoader: loader,
+    }));
+    if (extensionsResult.errors.length > 0) {
+        session.dispose();
+        if (ownsTmpDir && fs.existsSync(cwd)) {
+            fs.rmSync(cwd, { recursive: true, force: true });
+        }
+        const errors = extensionsResult.errors
+            .map((e) => `  ${e.path}: ${e.error}`)
+            .join("\n");
+        throw new Error(`Extension load errors:\n${errors}`);
+    }
+    const events = createEventCollector();
+    let currentStep = 0;
+    let mockedToolNames = new Set();
+    // toolCallIds whose mock returned a ToolResult with isError:true — Pi 0.84
+    // hardcodes successful execute() as non-error, so records must consult this.
+    let mockedErrorToolCallIds = new Set();
+    session.subscribe((event) => {
+        events.all.push(event);
+        if (event.type === "tool_execution_start") {
+            const record = {
+                step: currentStep,
+                toolName: event.toolName,
+                input: event.args ?? {},
+                blocked: false,
+            };
+            events.toolCalls.push(record);
+        }
+        if (event.type === "tool_execution_end") {
+            const resultText = event.result?.content
+                ?.filter((c) => c.type === "text")
+                ?.map((c) => c.text)
+                ?.join("\n") ?? "";
+            if (event.isError) {
+                const lastCall = events.toolCalls[events.toolCalls.length - 1];
+                if (lastCall && lastCall.toolName === event.toolName) {
+                    if (resultText.includes("blocked") || resultText.includes("Plan mode")) {
+                        lastCall.blocked = true;
+                        lastCall.blockReason = resultText;
+                    }
+                }
+            }
+            // Record the final result (after afterToolCall modifications).
+            // Always push — each tool_execution_end has a unique toolCallId
+            // within a run, and the subscriber is the only source of results.
+            const isMocked = mockedToolNames.has(event.toolName);
+            events.toolResults.push({
+                step: currentStep,
+                toolName: event.toolName,
+                toolCallId: event.toolCallId,
+                text: resultText,
+                content: event.result?.content ?? [],
+                isError: event.isError || mockedErrorToolCallIds.has(event.toolCallId),
+                details: event.result?.details,
+                mocked: isMocked,
+            });
+        }
+        if (event.type === "message_end") {
+            events.messages.push(event.message);
+        }
+    });
+    let playbookState = null;
+    const mockUI = createMockUIContext(options.mockUI, events.ui);
+    await session.bindExtensions({
+        uiContext: mockUI,
+        onError: (err) => {
+            console.error(`[pi-test-harness] Extension error: ${err.event} — ${err.error}`);
+        },
+    });
+    const originalTools = [...session.agent.state.tools];
+    const testSession = {
+        session,
+        cwd,
+        events,
+        get playbook() {
+            return {
+                consumed: playbookState?.consumed ?? 0,
+                remaining: playbookState?.remaining ?? 0,
+            };
+        },
+        async run(...turns) {
+            const { streamFn, state } = createPlaybookStreamFn(turns);
+            playbookState = state;
+            // Assign playbook streamFn to the public Agent.streamFunction
+            session.agent.streamFunction = streamFn;
+            const effectiveMockTools = options.mockTools ?? {};
+            const currentTools = originalTools;
+            const { tools: interceptedTools, mockedNames, mockedErrorToolCallIds: errorIds, } = interceptToolExecution(currentTools, effectiveMockTools, state, propagateErrors);
+            mockedToolNames = mockedNames;
+            mockedErrorToolCallIds = errorIds;
+            session.agent.state.tools = interceptedTools;
+            for (const turn of turns) {
+                currentStep = state.consumed;
+                await session.prompt(turn.prompt);
+                await session.agent.waitForIdle();
+            }
+            if (state.remaining > 0) {
+                const allActions = turns.flatMap((t) => t.actions);
+                const remaining = allActions.slice(state.consumed);
+                const diagnostic = formatPlaybookDiagnostic("remaining", state, remaining);
+                throw new Error(diagnostic);
+            }
+        },
+        /**
+         * Dispose the test session and clean up the temp directory (if owned).
+         *
+         * Note: `session.dispose()` does NOT fire `session_shutdown`. That event is
+         * dispatched by pi at Node.js process exit. Extensions that open resources in
+         * `session_start` (e.g., SQLite databases) keep those resources open until the
+         * process exit. Use `safeRmSync` when cleaning up extension-owned files in
+         * afterEach hooks on Windows to avoid EPERM errors.
+         */
+        dispose() {
+            session.dispose();
+            if (ownsTmpDir && fs.existsSync(cwd)) {
+                fs.rmSync(cwd, { recursive: true, force: true });
+            }
+        },
+    };
+    return testSession;
+}
+//# sourceMappingURL=session.js.map

package/dist/session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.js","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EACN,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,eAAe,EAEf,YAAY,GACZ,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,sBAAsB,EAAsB,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAQhE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,UAA8B,EAAE;IAEhC,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC;IACxD,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;IAChC,MAAM,GAAG,GACR,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;IAE3E,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,eAAe,GAAG,eAAe,CAAC,QAAQ,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,qBAAqB,CAAC;QACxC,GAAG;QACH,QAAQ,EAAE,GAAG,EAAE,oDAAoD;QACnE,eAAe;QACf,wBAAwB,EACvB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;QAC3D,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,oBAAoB,EAAE,OAAO,CAAC,YAAY;YACzC,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,YAAa;YAC7B,CAAC,CAAC,SAAS;KACZ,CAAC,CAAC;IACH,wEAAwE;IACxE,mEAAmE;IACnE,MAAM,uBAAuB,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAErD,kFAAkF;IAClF,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC;QAC9C,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC;QACrC,UAAU,EAAE,IAAI;KAChB,CAAC,CAAC;IAEH,0FAA0F;IAC1F,MAAM,aAAa,GAAG,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAChE,IAAI,CAAC,aAAa,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CACd,0DAA0D;YACzD,oEAAoE;YACpE,gDAAgD,CACjD,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,YAAY,CAAC,gBAAgB,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAC;IAEvE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,MAAM,uBAAuB,CAAC,GAAG,EAAE,CACxE,kBAAkB,CAAC;QAClB,GAAG;QACH,QAAQ,EAAE,GAAG;QACb,KAAK,EAAE,aAAa;QACpB,YAAY;QACZ,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE;QACzC,eAAe;QACf,cAAc,EAAE,MAAM;KACtB,CAAC,CACF,CAAC;IAEF,IAAI,gBAAgB,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,IAAI,UAAU,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACtC,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM;aACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;aACrC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,oBAAoB,EAAE,CAAC;IACtC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,eAAe,GAAwB,IAAI,GAAG,EAAE,CAAC;IACrD,2EAA2E;IAC3E,6EAA6E;IAC7E,IAAI,sBAAsB,GAAwB,IAAI,GAAG,EAAE,CAAC;IAE5D,OAAO,CAAC,SAAS,CAAC,CAAC,KAAwB,EAAE,EAAE;QAC9C,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEvB,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAmB;gBAC9B,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,KAAK,EAAG,KAAa,CAAC,IAAI,IAAI,EAAE;gBAChC,OAAO,EAAE,KAAK;aACd,CAAC;YACF,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YACzC,MAAM,UAAU,GACf,KAAK,CAAC,MAAM,EAAE,OAAO;gBACpB,EAAE,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;gBACvC,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBACzB,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAErB,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC/D,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;oBACtD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;wBACxE,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;wBACxB,QAAQ,CAAC,WAAW,GAAG,UAAU,CAAC;oBACnC,CAAC;gBACF,CAAC;YACF,CAAC;YAED,+DAA+D;YAC/D,gEAAgE;YAChE,kEAAkE;YAClE,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC;gBACvB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE;gBACpC,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC;gBACtE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO;gBAC9B,MAAM,EAAE,QAAQ;aAChB,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YAClC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;IACF,CAAC,CAAC,CAAC;IAEH,IAAI,aAAa,GAAyB,IAAI,CAAC;IAE/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IAE9D,MAAM,OAAO,CAAC,cAAc,CAAC;QAC5B,SAAS,EAAE,MAAM;QACjB,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAChB,OAAO,CAAC,KAAK,CACZ,sCAAsC,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,KAAK,EAAE,CAChE,CAAC;QACH,CAAC;KACD,CAAC,CAAC;IAEH,MAAM,aAAa,GAAgB,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAElE,MAAM,WAAW,GAAgB;QAChC,OAAO;QACP,GAAG;QACH,MAAM;QAEN,IAAI,QAAQ;YACX,OAAO;gBACN,QAAQ,EAAE,aAAa,EAAE,QAAQ,IAAI,CAAC;gBACtC,SAAS,EAAE,aAAa,EAAE,SAAS,IAAI,CAAC;aACxC,CAAC;QACH,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,GAAG,KAAa;YACzB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAC1D,aAAa,GAAG,KAAK,CAAC;YAEtB,8DAA8D;YAC9D,OAAO,CAAC,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC;YAExC,MAAM,kBAAkB,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;YACnD,MAAM,YAAY,GAAG,aAAa,CAAC;YACnC,MAAM,EACL,KAAK,EAAE,gBAAgB,EACvB,WAAW,EACX,sBAAsB,EAAE,QAAQ,GAChC,GAAG,sBAAsB,CACzB,YAAY,EACZ,kBAAkB,EAClB,KAAK,EACL,eAAe,CACf,CAAC;YACF,eAAe,GAAG,WAAW,CAAC;YAC9B,sBAAsB,GAAG,QAAQ,CAAC;YAClC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,gBAAgB,CAAC;YAE7C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAC7B,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAClC,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YACnC,CAAC;YAED,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBACnD,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACnD,MAAM,UAAU,GAAG,wBAAwB,CAAC,WAAW,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC3E,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;YAC7B,CAAC;QACF,CAAC;QAED;;;;;;;;WAQG;QACH,OAAO;YACN,OAAO,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,UAAU,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtC,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,CAAC;QACF,CAAC;KACD,CAAC;IAEF,OAAO,WAAW,CAAC;AACpB,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,171 @@
+/**
+ * Shared types for pi-test-harness.
+ */
+import type { AgentSession, AgentSessionEvent } from "@earendil-works/pi-coding-agent";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+export interface PlaybookAction {
+    type: "call" | "say";
+    /** For "call": tool name */
+    toolName?: string;
+    /** For "call": static or late-bound params */
+    params?: Record<string, unknown> | (() => Record<string, unknown>);
+    /** For "say": text content */
+    text?: string;
+    /** Optional callback after tool execution */
+    thenCallback?: (result: ToolResultRecord) => void;
+}
+export interface Turn {
+    prompt: string;
+    actions: PlaybookAction[];
+}
+export interface ToolResult {
+    content: Array<{
+        type: string;
+        text: string;
+    }>;
+    details?: unknown;
+    isError?: boolean;
+}
+export type MockToolHandler = string | ToolResult | ((params: Record<string, unknown>) => string | ToolResult);
+export interface MockUIConfig {
+    confirm?: boolean | ((title: string, message: string) => boolean);
+    select?: number | string | ((title: string, items: string[]) => string | undefined);
+    input?: string | ((title: string, placeholder?: string) => string | undefined);
+    editor?: string | ((title: string, prefilled?: string) => string | undefined);
+}
+export interface ToolCallRecord {
+    step: number;
+    toolName: string;
+    input: Record<string, unknown>;
+    blocked: boolean;
+    blockReason?: string;
+}
+export interface ToolResultRecord {
+    step: number;
+    toolName: string;
+    toolCallId: string;
+    text: string;
+    content: Array<{
+        type: string;
+        text?: string;
+    }>;
+    isError: boolean;
+    details?: unknown;
+    mocked: boolean;
+}
+export interface UICallRecord {
+    method: string;
+    args: unknown[];
+    returnValue?: unknown;
+}
+export interface TestEvents {
+    all: AgentSessionEvent[];
+    toolCalls: ToolCallRecord[];
+    toolResults: ToolResultRecord[];
+    messages: AgentMessage[];
+    ui: UICallRecord[];
+    toolCallsFor(name: string): ToolCallRecord[];
+    toolResultsFor(name: string): ToolResultRecord[];
+    blockedCalls(): ToolCallRecord[];
+    uiCallsFor(method: string): UICallRecord[];
+    /** Ordered list of tool names as they were called */
+    toolSequence(): string[];
+}
+export interface TestSessionOptions {
+    /** Extension file paths to load */
+    extensions?: string[];
+    /** Extension factory functions (inline) */
+    extensionFactories?: Array<(pi: any) => void>;
+    /** Working directory (auto temp dir if omitted, cleaned on dispose) */
+    cwd?: string;
+    /** System prompt override */
+    systemPrompt?: string;
+    /** Mock tool execution (intercepts tool.execute()) */
+    mockTools?: Record<string, MockToolHandler>;
+    /** Mock UI responses */
+    mockUI?: MockUIConfig;
+    /** Abort on real tool throw (default: true) */
+    propagateErrors?: boolean;
+}
+export interface TestSession {
+    /** Run a conversation script */
+    run(...turns: Turn[]): Promise<void>;
+    /** Real session underneath */
+    session: AgentSession;
+    /** Working directory */
+    cwd: string;
+    /** Collected events */
+    events: TestEvents;
+    /** Playbook consumption state */
+    playbook: {
+        consumed: number;
+        remaining: number;
+    };
+    /** Cleanup */
+    dispose(): void;
+}
+/** Configuration for a single mock pi invocation response. */
+export interface MockPiCall {
+    /** Text output from the mock agent */
+    output?: string;
+    /** Exit code (default: 0) */
+    exitCode?: number;
+    /** Stderr output */
+    stderr?: string;
+    /** Delay in ms before responding */
+    delay?: number;
+    /** Raw JSONL events to emit instead of default message_end */
+    jsonl?: object[];
+    /** Files to write before exiting (path → content) */
+    writeFiles?: Record<string, string>;
+}
+/** Mock pi CLI for testing extensions that spawn pi as a subprocess. */
+export interface MockPi {
+    /** Create temp dir with pi shim, prepend to PATH */
+    install(): void;
+    /** Remove from PATH, delete temp dir */
+    uninstall(): void;
+    /** Queue a response for the next pi invocation */
+    onCall(response: MockPiCall): void;
+    /** Clear the response queue and reset the call counter */
+    reset(): void;
+    /** Number of times the mock pi has been invoked */
+    callCount(): number;
+    /** The temporary directory containing the queue and shim */
+    dir: string;
+}
+export interface SandboxOptions {
+    /** Package directory (runs npm pack) */
+    packageDir: string;
+    /** Expected resources after install */
+    expect?: {
+        extensions?: number;
+        tools?: string[];
+        skills?: number;
+    };
+    /** Optional smoke test in the sandbox */
+    smoke?: {
+        mockTools?: Record<string, MockToolHandler>;
+        script: Turn[];
+    };
+    /**
+     * Custom npm command and arguments for pack/install.
+     * Default: [process.platform === "win32" ? "npm.cmd" : "npm"]
+     * Use ["sfw", "npm"] to run through Socket Firewall, or provide
+     * an array like ["/usr/local/bin/node", "/path/to/npm_cli.js"]
+     * for a non-standard npm executable.
+     */
+    npmCommand?: string[];
+}
+export interface SandboxResult {
+    loaded: {
+        extensions: number;
+        extensionErrors: string[];
+        tools: string[];
+        skills: number;
+    };
+    smoke?: {
+        events: TestEvents;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EACX,YAAY,EACZ,iBAAiB,EACjB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAIlE,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;IACrB,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACnE,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6CAA6C;IAC7C,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,CAAC;CAClD;AAED,MAAM,WAAW,IAAI;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,cAAc,EAAE,CAAC;CAC1B;AAID,MAAM,WAAW,UAAU;IAC1B,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,eAAe,GACxB,MAAM,GACN,UAAU,GACV,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,MAAM,GAAG,UAAU,CAAC,CAAC;AAE9D,MAAM,WAAW,YAAY;IAC5B,OAAO,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC;IAClE,MAAM,CAAC,EACJ,MAAM,GACN,MAAM,GACN,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,MAAM,GAAG,SAAS,CAAC,CAAC;IAC5D,KAAK,CAAC,EACH,MAAM,GACN,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC,CAAC;IACjE,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC,CAAC;CAC9E;AAID,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,iBAAiB,EAAE,CAAC;IACzB,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,EAAE,EAAE,YAAY,EAAE,CAAC;IAEnB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,EAAE,CAAC;IAC7C,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,EAAE,CAAC;IACjD,YAAY,IAAI,cAAc,EAAE,CAAC;IACjC,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IAC3C,qDAAqD;IACrD,YAAY,IAAI,MAAM,EAAE,CAAC;CACzB;AAID,MAAM,WAAW,kBAAkB;IAClC,mCAAmC;IACnC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,2CAA2C;IAC3C,kBAAkB,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;IAC9C,uEAAuE;IACvE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC5C,wBAAwB;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IAEtB,+CAA+C;IAC/C,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC3B,gCAAgC;IAChC,GAAG,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,8BAA8B;IAC9B,OAAO,EAAE,YAAY,CAAC;IACtB,wBAAwB;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,uBAAuB;IACvB,MAAM,EAAE,UAAU,CAAC;IACnB,iCAAiC;IACjC,QAAQ,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAClD,cAAc;IACd,OAAO,IAAI,IAAI,CAAC;CAChB;AAID,8DAA8D;AAC9D,MAAM,WAAW,UAAU;IAC1B,sCAAsC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACpC;AAED,wEAAwE;AACxE,MAAM,WAAW,MAAM;IACtB,oDAAoD;IACpD,OAAO,IAAI,IAAI,CAAC;IAChB,wCAAwC;IACxC,SAAS,IAAI,IAAI,CAAC;IAClB,kDAAkD;IAClD,MAAM,CAAC,QAAQ,EAAE,UAAU,GAAG,IAAI,CAAC;IACnC,0DAA0D;IAC1D,KAAK,IAAI,IAAI,CAAC;IACd,mDAAmD;IACnD,SAAS,IAAI,MAAM,CAAC;IACpB,4DAA4D;IAC5D,GAAG,EAAE,MAAM,CAAC;CACZ;AAID,MAAM,WAAW,cAAc;IAC9B,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,MAAM,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,yCAAyC;IACzC,KAAK,CAAC,EAAE;QACP,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAC5C,MAAM,EAAE,IAAI,EAAE,CAAC;KACf,CAAC;IACF;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B,MAAM,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;KACf,CAAC;IACF,KAAK,CAAC,EAAE;QACP,MAAM,EAAE,UAAU,CAAC;KACnB,CAAC;CACF"}

package/dist/types.js

@@ -0,0 +1,5 @@
+/**
+ * Shared types for pi-test-harness.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/utils.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Utility helpers for pi-test-harness consumers.
+ */
+/**
+ * Remove a file, silently ignoring EPERM/EBUSY errors.
+ *
+ * **Why this exists**: On Windows, pi extensions that open SQLite databases
+ * do so in the `session_start` event handler. The corresponding close happens
+ * in `session_shutdown` — but that event fires at Node.js **process exit**,
+ * NOT when `session.dispose()` is called. This means DB files remain locked
+ * for the lifetime of the test runner process.
+ *
+ * Safe pattern in afterEach:
+ * ```ts
+ * afterEach(() => {
+ *   safeRmSync(dbPath);
+ *   safeRmSync(dbPath + "-wal");
+ *   safeRmSync(dbPath + "-shm");
+ * });
+ * ```
+ *
+ * Files are cleaned up by the OS when the process exits (or on next run).
+ * Using unique DB paths per test ensures isolation.
+ */
+/**
+ * Returns true for errno codes that represent a Windows file-lock condition.
+ * Exported for unit testing — not part of the public API contract.
+ * @internal
+ */
+export declare function _isLockedFileError(err: unknown): boolean;
+export declare function safeRmSync(filePath: string): void;
+//# sourceMappingURL=utils.d.ts.map

package/dist/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAGxD;AAED,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAQjD"}

package/dist/utils.js

@@ -0,0 +1,46 @@
+/**
+ * Utility helpers for pi-test-harness consumers.
+ */
+import { rmSync } from "node:fs";
+/**
+ * Remove a file, silently ignoring EPERM/EBUSY errors.
+ *
+ * **Why this exists**: On Windows, pi extensions that open SQLite databases
+ * do so in the `session_start` event handler. The corresponding close happens
+ * in `session_shutdown` — but that event fires at Node.js **process exit**,
+ * NOT when `session.dispose()` is called. This means DB files remain locked
+ * for the lifetime of the test runner process.
+ *
+ * Safe pattern in afterEach:
+ * ```ts
+ * afterEach(() => {
+ *   safeRmSync(dbPath);
+ *   safeRmSync(dbPath + "-wal");
+ *   safeRmSync(dbPath + "-shm");
+ * });
+ * ```
+ *
+ * Files are cleaned up by the OS when the process exits (or on next run).
+ * Using unique DB paths per test ensures isolation.
+ */
+/**
+ * Returns true for errno codes that represent a Windows file-lock condition.
+ * Exported for unit testing — not part of the public API contract.
+ * @internal
+ */
+export function _isLockedFileError(err) {
+    const code = err.code;
+    return code === "EPERM" || code === "EBUSY";
+}
+export function safeRmSync(filePath) {
+    try {
+        rmSync(filePath, { force: true });
+    }
+    catch (err) {
+        // Only swallow Windows file-lock errors. Everything else (permissions,
+        // bad path type, disk full) should still propagate so failures are visible.
+        if (!_isLockedFileError(err))
+            throw err;
+    }
+}
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAEjC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC9C,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;IACjD,OAAO,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,QAAgB;IAC1C,IAAI,CAAC;QACJ,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,uEAAuE;QACvE,4EAA4E;QAC5E,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC;YAAE,MAAM,GAAG,CAAC;IACzC,CAAC;AACF,CAAC"}

package/package.json

@@ -0,0 +1,84 @@
+{
+  "name": "@abdwhb-png/pi-test-harness",
+  "version": "0.7.0",
+  "description": "Test harness for pi extensions — in-process session testing, package install verification, and subprocess mocking",
+  "author": "Abdullah Alblushi (fork)",
+  "license": "MIT",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/abdwhb-png/pi-test-harness.git"
+  },
+  "homepage": "https://github.com/abdwhb-png/pi-test-harness",
+  "bugs": "https://github.com/abdwhb-png/pi-test-harness/issues",
+  "contributors": [
+    "Marc Fargas <marc@marcfargas.com> (https://github.com/marcfargas)",
+    "Abdullah Alblushi <abdwhb@gmail.com> (https://github.com/abdwhb)"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "pi",
+    "pi-package",
+    "testing",
+    "test-harness",
+    "mock",
+    "playbook"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "dist/**/*.d.ts.map",
+    "dist/**/*.js.map",
+    "dist/mock-pi-script.mjs",
+    "!dist/package/**",
+    "skills/**",
+    "src/**/*.ts",
+    "src/**/*.mjs",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && node -e \"require('fs').copyFileSync('src/mock-pi-script.mjs','dist/mock-pi-script.mjs')\" && node scripts/prepare-package.mjs",
+    "prepublishOnly": "npm run build",
+    "release": "changeset publish",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint src/ __tests__/",
+    "test:unit": "vitest run __tests__/playbook.test.ts __tests__/mock-pi.test.ts __tests__/resolve-executable.test.ts",
+    "test:integration": "vitest run __tests__/integration.test.ts __tests__/extension-loading.test.ts __tests__/sandbox.test.ts __tests__/fixes.test.ts __tests__/pi-085-compat.test.ts __tests__/prepare-package.test.ts",
+    "test:bun": "bun test --isolate scripts/extension-loading-smoke.test.ts",
+    "test:consumer": "node scripts/consumer-smoke.mjs"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-agent-core": "^0.85.0",
+    "@earendil-works/pi-ai": "^0.85.0",
+    "@earendil-works/pi-coding-agent": "^0.85.0"
+  },
+  "overrides": {
+    "read-yaml-file": "2.1.0"
+  },
+  "devDependencies": {
+    "@changesets/changelog-github": "^0.5.2",
+    "@changesets/cli": "2.31.1",
+    "@earendil-works/pi-agent-core": "0.85.1",
+    "@earendil-works/pi-ai": "0.85.1",
+    "@earendil-works/pi-coding-agent": "0.85.1",
+    "@eslint/js": "10.0.1",
+    "eslint": "10.8.0",
+    "globals": "^16.1.0",
+    "typebox": "1.3.7",
+    "typescript": "^5.8.3",
+    "typescript-eslint": "^8.32.1",
+    "vitest": "5.0.1"
+  }
+}