failproofai 1.0.7-beta.1 → 1.0.7-beta.2

package/.next/standalone/sdk/typescript/test/sandbox.test.ts

@@ -0,0 +1,189 @@
+import { describe, expect, it } from "vitest";
+
+import { EvalResult, Score } from "../src/evaluator/authoring.js";
+import { sessionTranscriptFromWire } from "../src/evaluator/protocol.js";
+import {
+  EvaluationSandboxUnavailable,
+  EvaluationTimeout,
+  UnsafeEvaluatorSource,
+  compileCondition,
+  compileEvaluator,
+  sourceChecksum,
+} from "../src/evaluator/source.js";
+import { transcriptWire } from "./helpers.js";
+
+/**
+ * The `worker_threads` sandbox, exercised for real.
+ *
+ * It is not the correctness boundary — `expression.ts` is — but it is the only
+ * thing that bounds CPU, heap and wall clock, and the only thing that can KILL
+ * a running evaluation. Every one of these cases is a way a permitted
+ * expression could otherwise take the worker down with it.
+ */
+
+const session = sessionTranscriptFromWire(
+  transcriptWire([
+    { type: "tool_use", payload: { tool_name: "search" } },
+    { type: "tool_result", payload: { error: "timeout" } },
+    { type: "tool_result", payload: { output: "ok" } },
+  ]),
+);
+
+describe("running a managed evaluation", () => {
+  it("returns a rebuilt EvalResult from the worker", async () => {
+    const evaluate = compileEvaluator(
+      "EvalResult({ score: Score(1 - session.eventsOfType('tool_result')" +
+        ".filter(e => e.payload.error != null).length / session.count('tool_result')), " +
+        "reasoning: `saw ${session.eventCount} events` })",
+      { evalKey: "tool_success_rate" },
+    );
+    const result = await evaluate(session);
+    expect(result).toBeInstanceOf(EvalResult);
+    expect(result.score).toBeInstanceOf(Score);
+    expect(result.score!.value).toBe(0.5);
+    expect(result.reasoning).toBe("saw 3 events");
+    // The rebuilt result is a real one, so the authoring bounds apply to it.
+    expect(result.resultItems("tool_success_rate")).toHaveLength(1);
+  });
+
+  it("runs a managed condition in the same sandbox", async () => {
+    expect(await compileCondition("session.count('tool_use') > 0")(session)).toBe(true);
+    const structured = await compileCondition("ConditionResult(false, 'no_tools')")(session);
+    expect(structured).toMatchObject({ applicable: false, reasonCode: "no_tools" });
+  });
+
+  it("carries a metric and an assertion across the boundary", async () => {
+    const evaluate = compileEvaluator(
+      "EvalResult({ metrics: { tool_calls: Metric(session.count('tool_use'), { unit: 'count' }) }, " +
+        "assertions: { used_a_tool: Assertion(session.count('tool_use') > 0) } })",
+      { evalKey: "tool_calls" },
+    );
+    const result = await evaluate(session);
+    const items = result.resultItems("tool_calls");
+    expect(items.map((item) => [item.resultKey, item.resultKind, item.numericValue ?? item.boolValue])).toEqual([
+      ["tool_calls", "metric", 1],
+      ["used_a_tool", "assertion", true],
+    ]);
+  });
+});
+
+describe("failures come back as data, not as a crash", () => {
+  it("preserves the error's name across the worker boundary", async () => {
+    const evaluate = compileEvaluator("EvalResult({ score: Score(session.nope.deeper) })");
+    await expect(evaluate(session)).rejects.toThrow(TypeError);
+  });
+
+  it("reports a result that is not an EvalResult", async () => {
+    await expect(compileEvaluator("42")(session)).rejects.toThrow(/must return an EvalResult/);
+  });
+
+  it("reports a condition that is not a boolean or a ConditionResult", async () => {
+    await expect(compileCondition("42")(session)).rejects.toThrow(
+      /must return a boolean or a ConditionResult/,
+    );
+  });
+
+  it("rejects unsafe source BEFORE a worker is started", () => {
+    // Compilation is synchronous and happens in the parent, so a poison
+    // definition costs nothing and dead-letters as one failed run.
+    expect(() => compileEvaluator("(x => x).constructor")).toThrow(UnsafeEvaluatorSource);
+  });
+});
+
+describe("resource bounds", () => {
+  it("kills an evaluation that overruns its wall clock", async () => {
+    // Under the step ceiling this would run for a very long time; the worker is
+    // terminated either way, and both arrive as an EvaluationTimeout.
+    const evaluate = compileEvaluator(
+      "EvalResult({ score: Score(Array.from({ length: 1 }).length) })",
+      { timeoutSeconds: 0.001 },
+    );
+    await expect(evaluate(session)).rejects.toThrow(EvaluationTimeout);
+  }, 20_000);
+
+  it("bounds an unbounded recursion inside the worker", async () => {
+    const evaluate = compileEvaluator("(f => f(f))(f => f(f))", { timeoutSeconds: 10 });
+    await expect(evaluate(session)).rejects.toThrow(EvaluationTimeout);
+  }, 30_000);
+
+  it("refuses to evaluate at all when the sandbox cannot be started", async () => {
+    // Fails CLOSED. An evaluation that cannot be terminated is not one we are
+    // willing to start, so an unusable sandbox is a refusal — never a fallback
+    // to running tenant source in the worker's own thread.
+    const previous = process.env.FAILPROOFAI_SDK_SANDBOX_WORKER;
+    process.env.FAILPROOFAI_SDK_SANDBOX_WORKER = "/nonexistent/failproofai-sandbox-worker.js";
+    try {
+      const evaluate = compileEvaluator("EvalResult({ score: Score(1) })");
+      await expect(evaluate(session)).rejects.toThrow();
+      // And specifically NOT by quietly returning a result.
+      await expect(evaluate(session)).rejects.not.toBeInstanceOf(EvalResult);
+    } finally {
+      process.env.FAILPROOFAI_SDK_SANDBOX_WORKER = previous;
+    }
+  });
+
+  it("names the override when it cannot locate the worker at all", () => {
+    // `EvaluationSandboxUnavailable` carries the remedy, because the situation
+    // it describes — a bundled install whose published layout is gone — is one
+    // the operator can only fix if they are told which variable to set.
+    const error = new EvaluationSandboxUnavailable("x");
+    expect(error.name).toBe("EvaluationSandboxUnavailable");
+  });
+
+  it("clamps a server-supplied timeout to the ceiling", async () => {
+    // A large `timeout_seconds` must not be able to remove the bound. The
+    // evaluation itself is instant; what matters is that it still completes,
+    // i.e. the clamp did not turn a huge number into an invalid one.
+    const evaluate = compileEvaluator("EvalResult({ score: Score(1) })", {
+      timeoutSeconds: 86_400,
+    });
+    await expect(evaluate(session)).resolves.toBeInstanceOf(EvalResult);
+  });
+});
+
+describe("isolation", () => {
+  it("gives the worker no environment to read", async () => {
+    // The language cannot reach `process` at all, so this is defence in depth —
+    // it means a future gap could not be escalated into credential theft.
+    const previous = process.env.FAILPROOFAI_EVALUATOR_TOKEN;
+    process.env.FAILPROOFAI_EVALUATOR_TOKEN = "a-cross-tenant-credential";
+    try {
+      expect(() => compileEvaluator("process.env")).toThrow(UnsafeEvaluatorSource);
+      expect(() => compileEvaluator("Object.keys(process.env)")).toThrow(UnsafeEvaluatorSource);
+    } finally {
+      if (previous === undefined) delete process.env.FAILPROOFAI_EVALUATOR_TOKEN;
+      else process.env.FAILPROOFAI_EVALUATOR_TOKEN = previous;
+    }
+  });
+
+  it("does not let one evaluation see another's state", async () => {
+    const first = compileEvaluator("EvalResult({ score: Score(0.25) })");
+    const second = compileEvaluator("EvalResult({ score: Score(0.75) })");
+    const [a, b] = await Promise.all([first(session), second(session)]);
+    expect(a.score!.value).toBe(0.25);
+    expect(b.score!.value).toBe(0.75);
+  });
+
+  it("runs more evaluations than the concurrency cap without losing any", async () => {
+    // The semaphore bounds how many sandboxes exist at once; the extras queue
+    // rather than pile up memory, and every one still returns.
+    const results = await Promise.all(
+      Array.from({ length: 8 }, (_, index) =>
+        compileEvaluator(`EvalResult({ score: Score(${(index / 10).toFixed(1)}) })`)(session),
+      ),
+    );
+    expect(results.map((result) => result.score!.value)).toEqual([
+      0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7,
+    ]);
+  }, 30_000);
+});
+
+describe("sourceChecksum", () => {
+  it("covers the condition and the evaluator together, separated unambiguously", () => {
+    // A separator that could appear in either source would let two different
+    // pairs hash the same, which is what the checksum exists to prevent.
+    expect(sourceChecksum("a", "b")).not.toBe(sourceChecksum("ab", ""));
+    expect(sourceChecksum(null, "b")).toBe(sourceChecksum("", "b"));
+    expect(sourceChecksum("a", "b")).toMatch(/^sha256:[0-9a-f]{64}$/);
+  });
+});

package/.next/standalone/sdk/typescript/test/scopes.test.ts

@@ -0,0 +1,271 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { current, propagate } from "../src/context.js";
+import { agent, session, toolCall } from "../src/scopes.js";
+import { flushed, useSpool } from "./helpers.js";
+import type { Spool } from "./helpers.js";
+
+let spool: Spool;
+
+beforeEach(() => {
+  spool = useSpool();
+});
+afterEach(async () => {
+  await spool.cleanup();
+});
+
+const typesOf = (events: Array<Record<string, unknown>>): string[] =>
+  events.map((event) => String(event.type));
+
+describe("session", () => {
+  it("binds identity and emits nothing on its own", async () => {
+    const id = await session(async (sessionId) => {
+      expect(current().sessionId).toBe(sessionId);
+      return sessionId;
+    });
+    expect(id).toMatch(/^[0-9a-f]{32}$/);
+    expect(await flushed(spool)).toEqual([]);
+  });
+
+  it("inherits an already-bound session rather than splitting the run in two", async () => {
+    await session({ sessionId: "outer" }, async () => {
+      await session(async (inner) => {
+        expect(inner).toBe("outer");
+      });
+    });
+  });
+
+  it("unbinds on the way out", async () => {
+    await session({ sessionId: "s" }, () => undefined);
+    expect(current().sessionId).toBeNull();
+  });
+});
+
+describe("agent", () => {
+  it("brackets the block with agent_start and agent_end", async () => {
+    await agent("planner", { goal: "g" }, () => undefined);
+    const events = await flushed(spool);
+    expect(typesOf(events)).toEqual(["agent_start", "agent_end"]);
+    expect(events[0]!.goal).toBe("g");
+    expect(events[1]!.outcome).toBe("success");
+  });
+
+  it("emits error BEFORE agent_end, and marks the outcome 'failed' not 'failure'", async () => {
+    await expect(
+      agent("planner", async () => {
+        throw new TypeError("boom");
+      }),
+    ).rejects.toThrow("boom");
+
+    const events = await flushed(spool);
+    expect(typesOf(events)).toEqual(["agent_start", "error", "agent_end"]);
+    expect(events[1]!.error_type).toBe("TypeError");
+    expect(events[1]!.message).toBe("boom");
+    expect(typeof events[1]!.traceback).toBe("string");
+    // The server only counts error|failed|timeout|rejected as a failure.
+    expect(events[2]!.outcome).toBe("failed");
+  });
+
+  it("names the error by its class when a subclass leaves `name` as 'Error'", async () => {
+    // openai's BadRequestError (and many SDKs' errors) never set `name`, so
+    // `error.name` reads "Error" and the class was lost from the Errors surface.
+    class BadRequestError extends Error {}
+    await expect(
+      agent("planner", async () => {
+        await toolCall("t", () => {
+          throw new BadRequestError("model does not exist");
+        });
+      }),
+    ).rejects.toThrow("model does not exist");
+    const events = await flushed(spool);
+    expect(events.find((e) => e.type === "tool_result")!.error).toBe("BadRequestError: model does not exist");
+    expect(events.find((e) => e.type === "error")!.error_type).toBe("BadRequestError");
+  });
+
+  it("treats an AbortError as cancellation: no error event, outcome 'cancelled'", async () => {
+    const abort = new Error("stopped");
+    abort.name = "AbortError";
+    await expect(
+      agent("planner", async () => {
+        throw abort;
+      }),
+    ).rejects.toThrow("stopped");
+
+    const events = await flushed(spool);
+    expect(typesOf(events)).toEqual(["agent_start", "agent_end"]);
+    expect(events[1]!.outcome).toBe("cancelled");
+  });
+
+  it("nests, with parent_id taken from the enclosing agent", async () => {
+    await agent("supervisor", async () => {
+      await agent("worker", () => undefined);
+    });
+    const starts = (await flushed(spool)).filter((event) => event.type === "agent_start");
+    expect(starts[0]!.parent_id).toBeUndefined();
+    expect(starts[1]!.agent_id).toBe("worker");
+    expect(starts[1]!.parent_id).toBe("supervisor");
+  });
+
+  it("does NOT inherit a parent across an explicit new session", async () => {
+    // The span tree is keyed by session, so a cross-session parent would render
+    // as a dangling reference or get grafted onto an unrelated agent.
+    await agent("server", { sessionId: "boot" }, async () => {
+      await agent("handler", { sessionId: "request-1" }, () => undefined);
+    });
+    const handler = (await flushed(spool)).find(
+      (event) => event.type === "agent_start" && event.agent_id === "handler",
+    )!;
+    expect(handler.session_id).toBe("request-1");
+    expect(handler.parent_id).toBeUndefined();
+  });
+
+  it("honours an explicit parentId of null as a forced root", async () => {
+    await agent("outer", async () => {
+      await agent("inner", { parentId: null }, () => undefined);
+    });
+    const inner = (await flushed(spool)).find(
+      (event) => event.type === "agent_start" && event.agent_id === "inner",
+    )!;
+    expect(inner.parent_id).toBeUndefined();
+  });
+
+  it("stays synchronous for a synchronous body", () => {
+    const value = agent("sync", () => 42);
+    expect(value).toBe(42);
+  });
+
+  it("leaves no frame behind when agent_start itself throws", () => {
+    expect(() => agent("bad", { type: "reserved" }, () => undefined)).toThrow(/Reserved field/);
+    expect(current().sessionId).toBeNull();
+    expect(current().depth).toBe(0);
+  });
+});
+
+describe("toolCall", () => {
+  it("records the body's resolved value as the output", async () => {
+    await session({ sessionId: "s" }, async () => {
+      const hits = await toolCall("search", { input: { q: "x" } }, async () => ["a", "b"]);
+      expect(hits).toEqual(["a", "b"]);
+    });
+    const result = (await flushed(spool)).find((event) => event.type === "tool_result")!;
+    expect(result.output).toEqual(["a", "b"]);
+    expect(result.tool_name).toBe("search");
+  });
+
+  it("prefers an explicitly assigned output over the return value", async () => {
+    await session({ sessionId: "s" }, async () => {
+      await toolCall("search", async (call) => {
+        call.output = { chosen: true };
+        return "ignored";
+      });
+    });
+    const result = (await flushed(spool)).find((event) => event.type === "tool_result")!;
+    expect(result.output).toEqual({ chosen: true });
+  });
+
+  it("records a failure on the leaf and emits NO error event", async () => {
+    await session({ sessionId: "s" }, async () => {
+      await expect(
+        toolCall("search", async () => {
+          throw new RangeError("nope");
+        }),
+      ).rejects.toThrow("nope");
+    });
+    const events = await flushed(spool);
+    expect(typesOf(events)).toEqual(["tool_use", "tool_result"]);
+    expect(events[1]!.error).toBe("RangeError: nope");
+  });
+
+  it("closes a cancelled tool call with no error string", async () => {
+    const abort = new Error("cancelled");
+    abort.name = "AbortError";
+    await session({ sessionId: "s" }, async () => {
+      await expect(
+        toolCall("search", async () => {
+          throw abort;
+        }),
+      ).rejects.toThrow();
+    });
+    const result = (await flushed(spool)).find((event) => event.type === "tool_result")!;
+    expect("error" in result).toBe(false);
+  });
+
+  it("resolves identity once, at entry, so a nested scope cannot move the result", async () => {
+    await agent("outer", async () => {
+      await toolCall("search", async () => {
+        await agent("inner", () => undefined);
+      });
+    });
+    const result = (await flushed(spool)).find((event) => event.type === "tool_result")!;
+    expect(result.agent_id).toBe("outer");
+  });
+});
+
+describe("using-style scopes", () => {
+  it("emits the same events as the callback form", async () => {
+    {
+      using span = agent.open("planner", { goal: "g" });
+      expect(span.agentId).toBe("planner");
+      using call = toolCall.open("search", { input: { q: "x" } });
+      call.call.output = "done";
+    }
+    const events = await flushed(spool);
+    expect(typesOf(events)).toEqual(["agent_start", "tool_use", "tool_result", "agent_end"]);
+    expect(events[2]!.output).toBe("done");
+    expect(events[3]!.outcome).toBe("success");
+    expect(current().sessionId).toBeNull();
+  });
+
+  it("records a failure the block caught, via fail()", async () => {
+    {
+      using span = agent.open("planner");
+      span.fail(new Error("handled"));
+    }
+    const events = await flushed(spool);
+    expect(events.at(-1)!.outcome).toBe("failed");
+  });
+});
+
+describe("propagate", () => {
+  it("carries identity into a callback invoked outside the scope", async () => {
+    let stored: (() => void) | null = null;
+    await session({ sessionId: "s1" }, async () => {
+      await agent("planner", () => {
+        stored = propagate(() => {
+          expect(current().sessionId).toBe("s1");
+          expect(current().agentId).toBe("planner");
+        });
+      });
+    });
+    expect(current().sessionId).toBeNull();
+    stored!();
+  });
+
+  it("binds the same identity on every invocation, not the previous call's leftovers", async () => {
+    const seen: Array<string | null> = [];
+    const wrapped = await session({ sessionId: "s1" }, () =>
+      propagate(() => {
+        seen.push(current().sessionId);
+      }),
+    );
+    wrapped();
+    wrapped();
+    expect(seen).toEqual(["s1", "s1"]);
+  });
+});
+
+describe("concurrency", () => {
+  it("keeps two concurrent runs on separate identities", async () => {
+    await Promise.all([
+      agent("a", { sessionId: "sa" }, async () => {
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        expect(current().sessionId).toBe("sa");
+      }),
+      agent("b", { sessionId: "sb" }, async () => {
+        expect(current().sessionId).toBe("sb");
+      }),
+    ]);
+    const starts = (await flushed(spool)).filter((event) => event.type === "agent_start");
+    expect(new Set(starts.map((event) => event.session_id))).toEqual(new Set(["sa", "sb"]));
+  });
+});

package/.next/standalone/sdk/typescript/test/setup.ts

@@ -0,0 +1,19 @@
+import { afterAll } from "vitest";
+
+import { runtime } from "../src/runtime.js";
+
+/**
+ * Close the process-wide writer when a test FILE finishes.
+ *
+ * Vitest gives each file a fresh module registry but one process, so every file
+ * that touches `runtime.ts` builds another writer with another flush timer and
+ * another `exit` listener. Left open they accumulate across the run, Node warns
+ * about the listener count, and the last file's assertions race timers the
+ * first file started.
+ *
+ * `close()` is the API the writer already exposes for exactly this.
+ */
+afterAll(async () => {
+  await runtime.writer.flushNow();
+  runtime.writer.close();
+});

package/.next/standalone/sdk/typescript/test/skill-snippets.test.ts

@@ -0,0 +1,73 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join, relative } from "node:path";
+
+import ts from "typescript";
+import { describe, expect, it } from "vitest";
+
+import * as failproofai from "../src/index.js";
+import * as evaluator from "../src/evaluator/index.js";
+
+/**
+ * The `failproofai-sdk` skill teaches this package too, and its TypeScript blocks
+ * are instructions an agent copies into someone's real agent loop — code under
+ * test, like the Python blocks `sdk/python/tests/test_skill_snippets.py` guards.
+ *
+ * Two checks, both cheap and both aimed at drift rather than style: every
+ * ```ts block parses, and every SDK name a block calls — `failproofai.x`,
+ * `failproofai.event.x`, a named import from `@failproofai/sdk/evaluator` —
+ * still exists. A rename in `src/` otherwise leaves the skill teaching a call
+ * that throws `is not a function` in the customer's process.
+ */
+
+const SKILL = join(__dirname, "..", "..", "python", "skill");
+
+function markdownFiles(dir: string): string[] {
+  return readdirSync(dir).flatMap((name) => {
+    const path = join(dir, name);
+    if (statSync(path).isDirectory()) return markdownFiles(path);
+    return name.endsWith(".md") ? [path] : [];
+  });
+}
+
+function tsBlocks(path: string): string[] {
+  const text = readFileSync(path, "utf8");
+  return [...text.matchAll(/```(?:ts|typescript)[^\n]*\n([\s\S]*?)```/g)].map((m) => m[1]!);
+}
+
+const FILES = markdownFiles(SKILL).filter((path) => tsBlocks(path).length > 0);
+const BLOCKS = FILES.flatMap((path) =>
+  tsBlocks(path).map((code, index) => ({ id: `${relative(SKILL, path)} #${index}`, code })),
+);
+
+describe("the failproofai-sdk skill's TypeScript snippets", () => {
+  it("has snippets to check (a path typo would make every test below vacuous)", () => {
+    expect(BLOCKS.length).toBeGreaterThanOrEqual(8);
+  });
+
+  it.each(BLOCKS)("$id parses", ({ code }) => {
+    const out = ts.transpileModule(code, {
+      reportDiagnostics: true,
+      compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 },
+    });
+    const errors = (out.diagnostics ?? []).map((d) => ts.flattenDiagnosticMessageText(d.messageText, "\n"));
+    expect(errors).toEqual([]);
+  });
+
+  it.each(BLOCKS)("$id calls only names the SDK exports", ({ code }) => {
+    const missing: string[] = [];
+    for (const [, name] of code.matchAll(/\bfailproofai\.(?!event\.)(\w+)/g)) {
+      if (!(name! in failproofai)) missing.push(`failproofai.${name}`);
+    }
+    for (const [, name] of code.matchAll(/\bfailproofai\.event\.(\w+)/g)) {
+      if (typeof (failproofai.event as unknown as Record<string, unknown>)[name!] !== "function") {
+        missing.push(`failproofai.event.${name}`);
+      }
+    }
+    for (const [, names] of code.matchAll(/import\s*\{([^}]*)\}\s*from\s*"@failproofai\/sdk\/evaluator"/g)) {
+      for (const name of names!.split(",").map((n) => n.trim()).filter(Boolean)) {
+        if (!(name in evaluator)) missing.push(`@failproofai/sdk/evaluator: ${name}`);
+      }
+    }
+    expect(missing).toEqual([]);
+  });
+});

package/.next/standalone/sdk/typescript/test/spool-contract.test.ts

@@ -0,0 +1,124 @@
+import { readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+  expandUser,
+  failproofaiCustomAgentsDir,
+  getBaseDir,
+  legacyAgenteyeDir,
+  setBaseDir,
+} from "../src/resolver.js";
+
+/**
+ * Four implementations resolve where this SDK's spool lives, and they must
+ * agree or the SDK writes where no daemon reads — with NO error on either side,
+ * which is the whole reason this file exists:
+ *
+ *   * this package's `resolver.ts`
+ *   * the Python SDK's `failproofai_sdk/_resolver.py`
+ *   * the daemon's `crates/fpai-collect/src/config.rs`
+ *   * the CLI's `src/hooks/fp-home.ts`
+ *
+ * These assertions read the OTHER THREE from disk rather than restating them.
+ * A test that spelled the path out a fifth time would pass while the daemon
+ * watched somewhere else.
+ */
+
+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
+
+function read(relative: string): string {
+  return readFileSync(join(repoRoot, relative), "utf8");
+}
+
+afterEach(() => {
+  setBaseDir(null);
+});
+
+describe("agreement with the other implementations", () => {
+  it("matches the daemon's custom_agents_events_dir", () => {
+    const rust = read("crates/fpai-collect/src/config.rs");
+    // `home.join("custom-agents").join("events")`
+    expect(rust).toMatch(/\.join\("custom-agents"\)\s*\.join\("events"\)/);
+    expect(join(failproofaiCustomAgentsDir(), "events")).toBe(
+      join(homedir(), ".failproofai", "custom-agents", "events"),
+    );
+  });
+
+  it("matches the CLI's customAgentsDir", () => {
+    const hooks = read("src/hooks/fp-home.ts");
+    expect(hooks).toContain('atHome(home, "custom-agents")');
+    expect(hooks).toMatch(/process\.env\.FAILPROOFAI_HOME\s*\|\|\s*resolve\(homedir\(\), "\.failproofai"\)/);
+  });
+
+  it("matches the Python SDK's resolver", () => {
+    const python = read("sdk/python/failproofai_sdk/_resolver.py");
+    expect(python).toContain('base / "custom-agents"');
+    expect(python).toContain('os.environ.get("FAILPROOFAI_HOME")');
+  });
+
+  it("keeps the daemon watching the legacy root, so an older SDK is not stranded", () => {
+    const rust = read("crates/fpai-collect/src/config.rs");
+    expect(rust).toContain("agenteye_events_dir()");
+    expect(legacyAgenteyeDir()).toBe(join(homedir(), ".agenteye"));
+  });
+});
+
+describe("resolution order", () => {
+  it("prefers an explicit base directory over everything", () => {
+    setBaseDir("/tmp/somewhere-explicit");
+    expect(getBaseDir()).toBe("/tmp/somewhere-explicit");
+  });
+
+  it("moves with FAILPROOFAI_HOME but never leaves it", () => {
+    const previous = process.env.FAILPROOFAI_HOME;
+    process.env.FAILPROOFAI_HOME = "/opt/fp";
+    try {
+      // The `custom-agents` segment is appended unconditionally: the variable
+      // MOVES the umbrella, it cannot take the spool outside it.
+      expect(failproofaiCustomAgentsDir()).toBe(join("/opt/fp", "custom-agents"));
+    } finally {
+      if (previous === undefined) delete process.env.FAILPROOFAI_HOME;
+      else process.env.FAILPROOFAI_HOME = previous;
+    }
+  });
+
+  it("is NOT redirected by AGENTEYE_HOME", () => {
+    const previous = process.env.AGENTEYE_HOME;
+    process.env.AGENTEYE_HOME = "/tmp/agenteye-elsewhere";
+    try {
+      // A redirect with no confirmation and no error means batches land in a
+      // directory nothing reads, and an unread spool is indistinguishable from
+      // an idle one. The Python SDK removed this for the same reason.
+      expect(getBaseDir()).not.toContain("agenteye-elsewhere");
+    } finally {
+      if (previous === undefined) delete process.env.AGENTEYE_HOME;
+      else process.env.AGENTEYE_HOME = previous;
+    }
+  });
+});
+
+describe("tilde expansion", () => {
+  it("expands a leading ~, so the documented migration bridge does not create one", () => {
+    // `configure({ baseDir: "~/.agenteye" })` is the documented explicit bridge.
+    // Without expansion, the writer's recursive mkdir cheerfully creates a
+    // directory literally named `~` under the process's cwd and spools into it:
+    // nothing on the machine watches that path, so 100% of the telemetry is
+    // lost.
+    setBaseDir("~/.agenteye");
+    expect(getBaseDir()).toBe(join(homedir(), ".agenteye"));
+    expect(getBaseDir().startsWith("~")).toBe(false);
+  });
+
+  it("leaves a tilde that is not a home reference alone", () => {
+    expect(expandUser("/var/lib/~weird")).toBe("/var/lib/~weird");
+  });
+
+  it("resolves a relative base directory against the cwd", () => {
+    setBaseDir("relative/spool");
+    expect(getBaseDir()).toBe(resolve("relative/spool"));
+  });
+});