@pi-harness/pi-harness 0.1.13 → 0.1.14

package/packages/core/test/docker-sandbox.test.ts

@@ -0,0 +1,170 @@
+import { chmod, mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { delimiter, join } from "node:path";
+import { Context } from "@deepseek-ai/cordis";
+import { afterEach, describe, expect, test } from "vitest";
+import dockerSandboxPlugin from "../src/plugins/docker-sandbox.js";
+import { PiPluginUiRegistry, PiToolRegistry } from "../src/services.js";
+
+const temporaryDirectories: string[] = [];
+const originalPath = process.env.PATH;
+const originalLog = process.env.PI_HARNESS_FAKE_DOCKER_LOG;
+
+afterEach(async () => {
+  process.env.PATH = originalPath;
+  if (originalLog === undefined) delete process.env.PI_HARNESS_FAKE_DOCKER_LOG;
+  else process.env.PI_HARNESS_FAKE_DOCKER_LOG = originalLog;
+  await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })));
+});
+
+async function createRawFixture() {
+  const cwd = await mkdtemp(join(tmpdir(), "pi-harness-docker-sandbox-workspace-"));
+  const agentDir = await mkdtemp(join(tmpdir(), "pi-harness-docker-sandbox-agent-"));
+  temporaryDirectories.push(cwd, agentDir);
+  const context = new Context();
+  const tools = new PiToolRegistry();
+  const panels = new PiPluginUiRegistry();
+  context.provide("piHarnessLaunch", { cwd, agentDir, args: [], requestExit() {} });
+  context.provide("piTools", tools);
+  context.provide("piPluginUi", panels);
+  return { context, cwd, panels, tools };
+}
+
+async function createFixture() {
+  const fixture = await createRawFixture();
+  await fixture.context.plugin(dockerSandboxPlugin);
+  const tool = fixture.tools.snapshot().customTools.find((candidate) => candidate.name === "sandbox_exec");
+  if (tool === undefined) throw new Error("sandbox_exec was not registered");
+  return { ...fixture, tool };
+}
+
+async function installFakeDocker(cwd: string): Promise<string> {
+  const bin = join(cwd, "bin");
+  const log = join(cwd, "docker.log");
+  await mkdir(bin);
+  const executable = join(bin, "docker");
+  await writeFile(
+    executable,
+    `#!/usr/bin/env node
+const fs = require("node:fs");
+fs.appendFileSync(process.env.PI_HARNESS_FAKE_DOCKER_LOG, JSON.stringify(process.argv.slice(2)) + "\\n");
+if (process.argv[2] === "run") process.stdout.write("界".repeat(5000) + "\\u001b[31mBAD\\u001b[0m\\0");
+`,
+    "utf8",
+  );
+  await chmod(executable, 0o755);
+  process.env.PATH = `${bin}${delimiter}${originalPath ?? ""}`;
+  process.env.PI_HARNESS_FAKE_DOCKER_LOG = log;
+  return log;
+}
+
+describe("Docker sandbox production boundaries", () => {
+  test("uses the enforced isolation argv and bounds captured output by UTF-8 bytes", async () => {
+    if (process.platform === "win32") return;
+    const fixture = await createFixture();
+    const log = await installFakeDocker(fixture.cwd);
+    try {
+      const result = await fixture.tool.execute("run", { command: ["printf", "ok"], image: "alpine:3.20" }, undefined, undefined, {} as never);
+      const details = result.details as { output: string; status: string };
+      expect(details.status).toBe("completed");
+      expect(Buffer.byteLength(details.output, "utf8")).toBeLessThanOrEqual(12_000);
+      expect(details.output).toContain("BAD");
+      expect(details.output).not.toContain("\u001b");
+      expect(details.output).not.toContain("\u0000");
+      const calls = (await readFile(log, "utf8"))
+        .trim()
+        .split("\n")
+        .map((line) => JSON.parse(line) as string[]);
+      expect(calls[0]).toEqual(["image", "inspect", "alpine:3.20"]);
+      expect(calls[1]).toEqual(
+        expect.arrayContaining([
+          "run",
+          "--pull=never",
+          "--network",
+          "none",
+          "--read-only",
+          "--cap-drop",
+          "ALL",
+          "--memory",
+          "512m",
+          "--cpus",
+          "1",
+          "--pids-limit",
+          "256",
+        ]),
+      );
+    } finally {
+      await fixture.context.fiber.dispose();
+    }
+  });
+
+  test("declares a strict sequential schema and rolls back when panel registration fails", async () => {
+    const fixture = await createFixture();
+    try {
+      expect(fixture.tool.executionMode).toBe("sequential");
+      const parameters = fixture.tool.parameters as { type: unknown; additionalProperties: unknown };
+      expect(parameters.type).toBe("object");
+      expect(parameters.additionalProperties).toBe(false);
+      await fixture.context.fiber.dispose();
+      const second = await createRawFixture();
+      second.panels.register({ id: "docker-sandbox-panel", pluginId: "fixture", title: "Fixture", read: () => ({}) });
+      await expect(second.context.plugin(dockerSandboxPlugin)).rejects.toThrow(/already registered.*docker-sandbox-panel/iu);
+      expect(second.tools.snapshot().customTools).toEqual([]);
+      await second.context.fiber.dispose();
+    } finally {
+      await fixture.context.fiber.dispose();
+    }
+  });
+
+  test("rejects blank images and argv entries that exceed byte limits", async () => {
+    const fixture = await createFixture();
+    try {
+      await expect(fixture.tool.execute("blank", { command: ["echo"], image: "   " }, undefined, undefined, {} as never)).rejects.toThrow(
+        /image.*non-empty|invalid.*image/iu,
+      );
+      await expect(fixture.tool.execute("bytes", { command: ["界".repeat(10_000)] }, undefined, undefined, {} as never)).rejects.toThrow(
+        /argument.*16384.*bytes/iu,
+      );
+    } finally {
+      await fixture.context.fiber.dispose();
+    }
+  });
+
+  test("rejects command arrays with inherited, symbol, or extra properties", async () => {
+    const fixture = await createFixture();
+    try {
+      const inheritedParameters = Object.create({ command: ["echo"] }) as { command: string[] };
+      await expect(fixture.tool.execute("inherited-parameters", inheritedParameters, undefined, undefined, {} as never)).rejects.toThrow(
+        /parameters.*plain object/iu,
+      );
+      const inherited = Object.create({ 0: "echo" }) as string[];
+      Object.defineProperty(inherited, "length", { value: 1, enumerable: false });
+      await expect(fixture.tool.execute("inherited", { command: inherited }, undefined, undefined, {} as never)).rejects.toThrow(/command.*array|argument/iu);
+      const extra = ["echo"] as string[] & { extra?: string };
+      extra.extra = "unexpected";
+      await expect(fixture.tool.execute("extra", { command: extra }, undefined, undefined, {} as never)).rejects.toThrow(/command.*property|unknown/iu);
+      const symbol = ["echo"] as string[] & { [key: symbol]: boolean };
+      symbol[Symbol("extra")] = true;
+      await expect(fixture.tool.execute("symbol", { command: symbol }, undefined, undefined, {} as never)).rejects.toThrow(/command.*property|unknown/iu);
+    } finally {
+      await fixture.context.fiber.dispose();
+    }
+  });
+
+  test("checks lifecycle cancellation before reading raw parameters", async () => {
+    const fixture = await createFixture();
+    let accessed = false;
+    const params = {} as { command?: string[] };
+    Object.defineProperty(params, "command", {
+      enumerable: true,
+      get() {
+        accessed = true;
+        throw new Error("Docker parameter accessor executed");
+      },
+    });
+    await fixture.context.fiber.dispose();
+
+    await expect(fixture.tool.execute("disposed", params, undefined, undefined, {} as never)).rejects.toThrow("Docker sandbox plugin disposed");
+    expect(accessed).toBe(false);
+  });
+});

package/packages/core/test/fail-logger.test.ts

@@ -0,0 +1,363 @@
+import { Context } from "@deepseek-ai/cordis";
+import { afterEach, describe, expect, test, vi } from "vitest";
+import failLoggerPlugin from "../src/plugins/fail-logger.js";
+import { PiPluginUiRegistry } from "../src/services.js";
+
+interface FailurePanelData {
+  total: number;
+  observed: number;
+  dropped: number;
+  capacity: number;
+  failures: Array<{ time: string; source: string; message: string; occurrences: number }>;
+}
+
+const contexts: Context[] = [];
+
+afterEach(async () => {
+  await Promise.all(contexts.splice(0).map(async (context) => context.fiber.dispose()));
+});
+
+async function setup(): Promise<{ context: Context; panels: PiPluginUiRegistry }> {
+  const context = new Context();
+  contexts.push(context);
+  const panels = new PiPluginUiRegistry();
+  context.provide("piPluginUi", panels);
+  await context.plugin(failLoggerPlugin);
+  return { context, panels };
+}
+
+async function panel(panels: PiPluginUiRegistry): Promise<FailurePanelData> {
+  const snapshot = await panels.snapshot();
+  return snapshot[0]?.data as FailurePanelData;
+}
+
+describe("failure logger plugin", () => {
+  test("deduplicates matching failures across intervening events and counts occurrences", async () => {
+    const { context, panels } = await setup();
+    context.emit("pi/extension-error", { extensionPath: "a.ts", event: "load", error: "same failure" });
+    context.emit("pi/extension-error", { extensionPath: "b.ts", event: "load", error: "other failure" });
+    context.emit("pi/extension-error", { extensionPath: "a.ts", event: "tool", error: "same failure" });
+
+    const data = await panel(panels);
+    expect(data).toMatchObject({ total: 2, observed: 3, dropped: 0, capacity: 50 });
+    expect(data.failures).toMatchObject([
+      { source: "extension", message: "a.ts: same failure", occurrences: 2 },
+      { source: "extension", message: "b.ts: other failure", occurrences: 1 },
+    ]);
+    expect(data.failures.every((failure) => Number.isFinite(Date.parse(failure.time)))).toBe(true);
+  });
+
+  test("saturates occurrence counts at the largest safe integer", async () => {
+    const { context, panels } = await setup();
+    context.emit("pi/extension-error", { extensionPath: "repeat.ts", event: "load", error: "same failure" });
+    const originalGet = Reflect.get(Map.prototype, "get") as (this: Map<unknown, unknown>, key: unknown) => unknown;
+    const get = vi.spyOn(Map.prototype, "get").mockImplementation(function (this: Map<unknown, unknown>, key: unknown) {
+      const value = originalGet.call(this, key);
+      if (key === "extension:repeat.ts: same failure" && typeof value === "object" && value !== null && "occurrences" in value) {
+        (value as { occurrences: number }).occurrences = Number.MAX_SAFE_INTEGER;
+      }
+      return value;
+    } as never);
+    try {
+      context.emit("pi/extension-error", { extensionPath: "repeat.ts", event: "load", error: "same failure" });
+    } finally {
+      get.mockRestore();
+    }
+
+    expect((await panel(panels)).failures[0]?.occurrences).toBe(Number.MAX_SAFE_INTEGER);
+  });
+
+  test("summarizes hostile and oversized failures without running accessors or throwing", async () => {
+    const { context, panels } = await setup();
+    let accessed = false;
+    const accessor: Record<string, unknown> = {};
+    Object.defineProperty(accessor, "error", {
+      enumerable: true,
+      get() {
+        accessed = true;
+        throw new Error("getter executed");
+      },
+    });
+    expect(() => context.emit("pi/extension-error", accessor as never)).not.toThrow();
+    expect(accessed).toBe(false);
+
+    const cyclic: Record<string, unknown> = { code: "E_FAIL", attempt: 42n };
+    cyclic.self = cyclic;
+    expect(() => context.emit("pi/extension-error", { extensionPath: "cycle.ts", event: "load", error: cyclic } as never)).not.toThrow();
+    let enumerated = false;
+    const nonEnumerated = new Proxy(
+      { code: "E_PROXY" },
+      {
+        ownKeys() {
+          enumerated = true;
+          throw new Error("failure payload was enumerated");
+        },
+      },
+    );
+    expect(() => context.emit("pi/extension-error", { extensionPath: "proxy.ts", event: "load", error: nonEnumerated } as never)).not.toThrow();
+    expect(enumerated).toBe(false);
+    let constructorAccessed = false;
+    const nestedError = new Error("nested failure");
+    Object.defineProperty(nestedError, "constructor", {
+      get() {
+        constructorAccessed = true;
+        throw new Error("constructor getter executed");
+      },
+    });
+    expect(() =>
+      context.emit("pi/extension-error", { extensionPath: "nested.ts", event: "load", error: { code: "E_NESTED", cause: nestedError } } as never),
+    ).not.toThrow();
+    expect(constructorAccessed).toBe(false);
+    expect(() =>
+      context.emit("pi/extension-error", {
+        extensionPath: "p".repeat(2_000),
+        event: "load",
+        error: "m".repeat(10_000),
+      }),
+    ).not.toThrow();
+
+    const data = await panel(panels);
+    expect(data.failures).toHaveLength(5);
+    expect(data.failures.some((failure) => failure.message.includes("[Accessor]"))).toBe(true);
+    expect(data.failures.some((failure) => failure.message.includes("E_FAIL") && failure.message.includes("42"))).toBe(true);
+    expect(data.failures.some((failure) => failure.message.includes("E_PROXY"))).toBe(true);
+    expect(data.failures.some((failure) => failure.message.includes("nested failure"))).toBe(true);
+    expect(data.failures.every((failure) => failure.message.length <= 2_048)).toBe(true);
+  });
+
+  test("previews proxy arrays without property reads", async () => {
+    const { context, panels } = await setup();
+    const propertyReads: PropertyKey[] = [];
+    const payload = new Proxy([{ code: "E_ARRAY" }], {
+      get(_target, key) {
+        propertyReads.push(key);
+        throw new Error(`array property read: ${String(key)}`);
+      },
+    });
+
+    expect(() => context.emit("pi/extension-error", { extensionPath: "array.ts", event: "load", error: payload } as never)).not.toThrow();
+    expect(propertyReads).toEqual([]);
+    expect((await panel(panels)).failures[0]?.message).toContain("E_ARRAY");
+  });
+
+  test("previews proxied errors without prototype inspection", async () => {
+    const { context, panels } = await setup();
+    let prototypeInspections = 0;
+    const payload = new Proxy(new Error("proxied failure"), {
+      getPrototypeOf(target) {
+        prototypeInspections += 1;
+        return Reflect.getPrototypeOf(target);
+      },
+    });
+
+    expect(() => context.emit("pi/extension-error", { extensionPath: "proxy-error.ts", event: "load", error: payload } as never)).not.toThrow();
+    expect(prototypeInspections).toBe(0);
+    expect((await panel(panels)).failures[0]?.message).toBe("proxy-error.ts: proxied failure");
+  });
+
+  test("sanitizes NUL characters from failure paths and messages", async () => {
+    const { context, panels } = await setup();
+    context.emit("pi/extension-error", { extensionPath: "bad\0path.ts", event: "load", error: "bad\0message" });
+
+    const message = (await panel(panels)).failures[0]?.message;
+    expect(message).toBe("bad�path.ts: bad�message");
+    expect(message).not.toContain("\0");
+  });
+
+  test("keeps the newest 50 unique records and reports capacity eviction", async () => {
+    const { context, panels } = await setup();
+    for (let index = 0; index < 60; index += 1) {
+      context.emit("pi/extension-error", { extensionPath: "capacity.ts", event: "load", error: `failure-${index}` });
+    }
+
+    const data = await panel(panels);
+    expect(data).toMatchObject({ total: 50, observed: 60, dropped: 10, capacity: 50 });
+    expect(data.failures[0]?.message).toBe("capacity.ts: failure-59");
+    expect(data.failures.at(-1)?.message).toBe("capacity.ts: failure-10");
+  });
+
+  test("records agent and compaction failures with their actual event sources", async () => {
+    const { context, panels } = await setup();
+    context.emit("pi/session-event", {
+      type: "agent_end",
+      willRetry: false,
+      messages: [{ role: "assistant", stopReason: "error", errorMessage: "agent failed" }],
+    } as never);
+    context.emit("pi/session-event", { type: "compaction_end", errorMessage: "compaction failed" } as never);
+
+    expect((await panel(panels)).failures).toMatchObject([
+      { source: "compaction", message: "compaction failed", occurrences: 1 },
+      { source: "agent", message: "agent failed", occurrences: 1 },
+    ]);
+  });
+
+  test("does not report an aborted compaction as a failure", async () => {
+    const { context, panels } = await setup();
+    context.emit("pi/session-event", {
+      type: "compaction_end",
+      reason: "manual",
+      result: undefined,
+      aborted: true,
+      willRetry: false,
+      errorMessage: "Operation was cancelled",
+    } as never);
+
+    expect(await panel(panels)).toMatchObject({ total: 0, observed: 0, failures: [] });
+  });
+
+  test("finds the most recent failed assistant before trailing non-assistant messages", async () => {
+    const { context, panels } = await setup();
+    context.emit("pi/session-event", {
+      type: "agent_end",
+      willRetry: false,
+      messages: [
+        { role: "assistant", stopReason: "error", errorMessage: "older failure" },
+        { role: "assistant", stopReason: "error", errorMessage: "latest failure" },
+        { role: "toolResult", content: [{ type: "text", text: "tool failed too" }] },
+        { role: "custom", customType: "diagnostic" },
+      ],
+    } as never);
+
+    expect((await panel(panels)).failures).toMatchObject([{ source: "agent", message: "latest failure", occurrences: 1 }]);
+  });
+
+  test("bounds reverse message inspection without proxy property reads", async () => {
+    const { context, panels } = await setup();
+    const rawMessages = new Array<unknown>(10_001);
+    rawMessages[0] = { role: "assistant", stopReason: "error", errorMessage: "outside scan window" };
+    const propertyReads: PropertyKey[] = [];
+    let descriptorInspections = 0;
+    const messages = new Proxy(rawMessages, {
+      get(_target, key) {
+        propertyReads.push(key);
+        throw new Error(`messages property read: ${String(key)}`);
+      },
+      getOwnPropertyDescriptor(target, key) {
+        descriptorInspections += 1;
+        return Reflect.getOwnPropertyDescriptor(target, key);
+      },
+    });
+
+    context.emit("pi/session-event", { type: "agent_end", willRetry: false, messages } as never);
+    expect(propertyReads).toEqual([]);
+    expect(descriptorInspections).toBe(10_001);
+    expect(await panel(panels)).toMatchObject({ total: 0, observed: 0, failures: [] });
+  });
+
+  test("uses a stable fallback for an empty assistant error message", async () => {
+    const { context, panels } = await setup();
+    context.emit("pi/session-event", {
+      type: "agent_end",
+      willRetry: false,
+      messages: [{ role: "assistant", stopReason: "error", errorMessage: "   " }],
+    } as never);
+
+    expect((await panel(panels)).failures).toMatchObject([{ source: "agent", message: "Agent turn failed", occurrences: 1 }]);
+  });
+
+  test("ignores hostile session events without invoking accessors", async () => {
+    const { context, panels } = await setup();
+    const accessed: string[] = [];
+    const hostileEvent: Record<string, unknown> = {};
+    for (const key of ["type", "messages", "errorMessage"]) {
+      Object.defineProperty(hostileEvent, key, {
+        get() {
+          accessed.push(key);
+          throw new Error(`${key} getter executed`);
+        },
+      });
+    }
+
+    expect(() => context.emit("pi/session-event", hostileEvent as never)).not.toThrow();
+    expect(accessed).toEqual([]);
+
+    const hostileMessages = { type: "agent_end" };
+    Object.defineProperty(hostileMessages, "messages", {
+      get() {
+        accessed.push("messages");
+        throw new Error("messages getter executed");
+      },
+    });
+    const hostileCompaction = { type: "compaction_end" };
+    Object.defineProperty(hostileCompaction, "errorMessage", {
+      get() {
+        accessed.push("errorMessage");
+        throw new Error("errorMessage getter executed");
+      },
+    });
+    const hostileMessage = { role: "assistant" };
+    Object.defineProperty(hostileMessage, "stopReason", {
+      get() {
+        accessed.push("stopReason");
+        throw new Error("stopReason getter executed");
+      },
+    });
+
+    expect(() => context.emit("pi/session-event", hostileMessages as never)).not.toThrow();
+    expect(() => context.emit("pi/session-event", { type: "agent_end", messages: [hostileMessage] } as never)).not.toThrow();
+    expect(() => context.emit("pi/session-event", hostileCompaction as never)).not.toThrow();
+    expect(accessed).toEqual([]);
+    expect(await panel(panels)).toMatchObject({ total: 0, observed: 0, dropped: 0, failures: [] });
+  });
+
+  test("rolls back event listeners when panel registration fails", async () => {
+    const context = new Context();
+    const panels = new PiPluginUiRegistry();
+    context.provide("piPluginUi", panels);
+    const disposeDuplicate = panels.register({
+      id: "fail-logger-panel",
+      pluginId: "duplicate",
+      title: "Duplicate",
+      read: () => ({}),
+    });
+    let inspections = 0;
+    const hostile = new Proxy(
+      {},
+      {
+        getOwnPropertyDescriptor() {
+          inspections += 1;
+          return undefined;
+        },
+      },
+    );
+    try {
+      await expect(context.plugin(failLoggerPlugin)).rejects.toThrow(/already registered/iu);
+      context.emit("pi/extension-error", hostile as never);
+      context.emit("pi/session-event", hostile as never);
+      expect(inspections).toBe(0);
+    } finally {
+      disposeDuplicate();
+      await context.fiber.dispose();
+    }
+  });
+
+  test("removes its panel and listeners on disposal", async () => {
+    const { context, panels } = await setup();
+    await context.fiber.dispose();
+    let inspections = 0;
+    const hostile = new Proxy(
+      {},
+      {
+        getOwnPropertyDescriptor() {
+          inspections += 1;
+          return undefined;
+        },
+      },
+    );
+
+    context.emit("pi/extension-error", hostile as never);
+    context.emit("pi/session-event", hostile as never);
+    expect(inspections).toBe(0);
+    await expect(panels.snapshot()).resolves.toEqual([]);
+  });
+
+  test("returns detached failure objects from every panel snapshot", async () => {
+    const { context, panels } = await setup();
+    context.emit("pi/extension-error", { extensionPath: "isolation.ts", event: "load", error: "original" });
+    const first = await panel(panels);
+    first.failures[0]!.message = "mutated";
+
+    const second = await panel(panels);
+    expect(second.failures[0]?.message).toBe("isolation.ts: original");
+  });
+});