crewhaus 0.1.3

package/src/justification-gate.test.ts

@@ -0,0 +1,292 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { ProviderAdapter } from "@crewhaus/adapter-anthropic";
+import { openAuditLog, verify } from "@crewhaus/audit-log";
+import type { JustificationJudge } from "@crewhaus/permission-engine";
+import { runChatLoop } from "@crewhaus/runtime-core";
+import { buildTool } from "@crewhaus/tool-builder";
+import { z } from "zod";
+import {
+  InvalidJudgeChoiceError,
+  createJustificationJudge,
+  justificationAuditDir,
+  openJustificationAuditSink,
+  resolveJudgeChoice,
+} from "./justification-gate";
+
+// ---------------------------------------------------------------------------
+// Pure judge-choice resolution (flag > spec > rule-based) + invalid handling.
+// ---------------------------------------------------------------------------
+
+describe("resolveJudgeChoice", () => {
+  test("defaults to rule-based when neither flag nor spec supply a judge", () => {
+    expect(resolveJudgeChoice(undefined, undefined)).toBe("rule-based");
+    expect(resolveJudgeChoice(undefined, {})).toBe("rule-based");
+  });
+
+  test("the spec's security.justification.judge is used when the flag is absent", () => {
+    expect(resolveJudgeChoice(undefined, { judge: "claude" })).toBe("claude");
+  });
+
+  test("the --justification-judge flag overrides the spec", () => {
+    // spec says claude, flag says rule-based → flag wins.
+    expect(resolveJudgeChoice("rule-based", { judge: "claude" })).toBe("rule-based");
+    expect(resolveJudgeChoice("claude", { judge: "rule-based" })).toBe("claude");
+  });
+
+  test("an unrecognised value throws InvalidJudgeChoiceError with the allowed list", () => {
+    expect(() => resolveJudgeChoice("gpt-omniscient", undefined)).toThrow(InvalidJudgeChoiceError);
+    try {
+      resolveJudgeChoice("gpt-omniscient", undefined);
+    } catch (err) {
+      expect(err).toBeInstanceOf(InvalidJudgeChoiceError);
+      expect((err as InvalidJudgeChoiceError).choice).toBe("gpt-omniscient");
+      expect((err as InvalidJudgeChoiceError).message).toContain("rule-based, claude");
+    }
+  });
+});
+
+describe("createJustificationJudge", () => {
+  test("returns undefined for rule-based (caller falls back to ruleBasedJustificationJudge)", async () => {
+    expect(await createJustificationJudge("rule-based", undefined)).toBeUndefined();
+  });
+
+  test("returns a callable JustificationJudge for claude (lazy import resolves)", async () => {
+    // Proves the CLI's lazy import of @crewhaus/justification-judge-claude +
+    // the anthropic adapter resolves and yields a function. The judge's
+    // verdict behaviour (incl. fail-closed) is exhaustively covered in the
+    // judge package's own stubbed tests; here we only assert the handoff
+    // produces the functional interface runChatLoop consumes. A dummy key
+    // satisfies the adapter constructor — it is never called (no turn runs).
+    const prev = process.env["ANTHROPIC_API_KEY"];
+    process.env["ANTHROPIC_API_KEY"] = "test-no-call";
+    try {
+      const judge = await createJustificationJudge("claude", "claude-haiku-4-5");
+      expect(typeof judge).toBe("function");
+    } finally {
+      // Assignment (not `delete`) to restore — matches the env-restore pattern
+      // used elsewhere (runtime-core/src/index.test.ts) and avoids noDelete.
+      process.env["ANTHROPIC_API_KEY"] = prev;
+    }
+  });
+
+  test("a non-Anthropic judge model resolves via the model-router and carries the STRIPPED wire model", async () => {
+    // `local/<m>@<url>` resolves the real OpenAI-compatible adapter with NO
+    // API key required, so this exercises the router path end-to-end. The
+    // endpoint is unreachable (port 1) — the judge fails CLOSED, and its
+    // verdict's judgeModel must carry the stripped wire id `<model> (error)`,
+    // NOT the full prefixed router string the old hardcoded
+    // createAnthropicAdapter() path passed verbatim.
+    const judge = await createJustificationJudge(
+      "claude",
+      "local/justification-judge-model@http://127.0.0.1:1/v1",
+    );
+    expect(typeof judge).toBe("function");
+    if (judge === undefined) throw new Error("unreachable");
+    const verdict = await judge({
+      toolName: "SendMessage",
+      justification: "post the summary",
+      sessionGoal: "summarize the channel",
+      input: { text: "hi" },
+    });
+    expect(verdict.allow).toBe(false); // fail-closed on transport error
+    expect(verdict.judgeModel).toBe("justification-judge-model (error)");
+  });
+});
+
+describe("openJustificationAuditSink", () => {
+  test("returns undefined when disabled", async () => {
+    expect(await openJustificationAuditSink({ cwd: "/tmp", enabled: false })).toBeUndefined();
+  });
+
+  test("roots the audit dir at <cwd>/.crewhaus/audit", () => {
+    expect(justificationAuditDir("/work")).toBe(join("/work", ".crewhaus", "audit"));
+  });
+});
+
+// ---------------------------------------------------------------------------
+// End-to-end: the CLI-resolved judge + CLI-opened durable sink actually
+// govern a real gate decision, and the durable, hash-chained
+// `permission_justification_evaluated` record records the judge identity.
+// This closes the CLI->runtime handoff gap (the prior CLI test only proved
+// the lazy import resolved; the judge was never called because stdin closed
+// before any turn).
+// ---------------------------------------------------------------------------
+
+/** One-turn stub adapter: emits a `tool_use` for the justification-gated
+ *  `sendmessage` tool (carrying a justification), then plain text so the
+ *  single-turn loop terminates. No network — deterministic. */
+function makeGatedToolUseAdapter(justification: string): ProviderAdapter {
+  let i = 0;
+  return {
+    providerId: "anthropic",
+    features: {
+      caching: "explicit",
+      tool_use: true,
+      vision: true,
+      thinking: true,
+      web_search: true,
+    },
+    estimateTokens: () => 0,
+    stream: () => {
+      const isFirst = i === 0;
+      i += 1;
+      return (async function* () {
+        yield { kind: "message_start", usage: { input: 10, output: 0 } } as const;
+        if (isFirst) {
+          yield {
+            kind: "content_block_start",
+            index: 0,
+            block: { type: "tool_use", id: "toolu_cli", name: "sendmessage", input: {} },
+          } as const;
+          yield {
+            kind: "content_block_delta",
+            index: 0,
+            delta: {
+              type: "input_json_delta",
+              partial_json: JSON.stringify({ body: "ack", justification }),
+            },
+          } as const;
+          yield { kind: "content_block_stop", index: 0 } as const;
+          yield {
+            kind: "message_delta",
+            stopReason: "tool_use",
+            usage: { input: 10, output: 5 },
+          } as const;
+        } else {
+          yield {
+            kind: "content_block_start",
+            index: 0,
+            block: { type: "text", text: "" },
+          } as const;
+          yield {
+            kind: "content_block_delta",
+            index: 0,
+            delta: { type: "text_delta", text: "done" },
+          } as const;
+          yield { kind: "content_block_stop", index: 0 } as const;
+          yield {
+            kind: "message_delta",
+            stopReason: "end_turn",
+            usage: { input: 10, output: 5 },
+          } as const;
+        }
+        yield { kind: "message_stop" } as const;
+      })();
+    },
+  };
+}
+
+describe("justification gate end-to-end (CLI sink + judge -> durable audit)", () => {
+  test("a CLI-opened audit sink records a verify()-clean permission_justification_evaluated record carrying the judge identity", async () => {
+    const cwd = mkdtempSync(join(tmpdir(), "crewhaus-just-gate-"));
+    try {
+      // The judge the CLI would resolve — here a stub with a distinctive model
+      // id so the assertion on the DURABLE record's judgeModel is exact and
+      // deterministic (no live model). The CLI's real claude-judge factory is
+      // covered above; the judge package's verdict logic is covered in its own
+      // suite. This test owns the handoff: CLI sink + a judge -> a real gate
+      // decision -> a durable, hash-chained record.
+      const judge: JustificationJudge = async () => ({
+        allow: true,
+        reason: "justification is consistent with the acknowledgement goal",
+        confidence: 0.83,
+        judgeModel: "claude-haiku-4-5",
+      });
+
+      // Open the sink exactly the way `crewhaus run` does (rooted at
+      // <cwd>/.crewhaus/audit), via the CLI helper under test.
+      const sink = await openJustificationAuditSink({ cwd, enabled: true });
+      expect(sink).toBeDefined();
+
+      const sendMessage = buildTool({
+        name: "sendmessage",
+        description: "send a message (justification-gated)",
+        inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
+        requireJustification: true,
+        execute: async () => "sent",
+      });
+
+      const JUSTIFICATION = "acknowledge the user's support ticket per the session goal";
+      await runChatLoop({
+        model: "test-model",
+        instructions: "Acknowledge support tickets the user points you at.",
+        _adapter: makeGatedToolUseAdapter(JUSTIFICATION),
+        singleTurn: true,
+        seedMessages: [{ role: "user", content: "ack the ticket" }],
+        tools: [sendMessage],
+        permissionMode: "bypass",
+        justificationJudge: judge,
+        // biome-ignore lint/style/noNonNullAssertion: asserted defined above.
+        justificationAuditSink: sink!,
+      });
+
+      // The durable, hash-chained chain must be intact.
+      const auditDir = justificationAuditDir(cwd);
+      const v = await verify(auditDir);
+      expect(v.ok).toBe(true);
+      expect(v.recordsChecked).toBe(1);
+
+      // Read the record back and assert the kind, verbatim justification, and
+      // judge identity — proving the CLI-resolved judge governed a real gate
+      // decision that reached the durable audit log.
+      const log = await openAuditLog({ rootDir: auditDir });
+      const records = [];
+      for await (const r of log.read()) records.push(r);
+      expect(records).toHaveLength(1);
+      const rec = records[0];
+      expect(rec?.kind).toBe("permission_justification_evaluated");
+      const payload = rec?.payload as {
+        toolName: string;
+        justification: string;
+        verdict: string;
+        judgeModel: string;
+        confidence?: number;
+      };
+      expect(payload.toolName).toBe("sendmessage");
+      expect(payload.verdict).toBe("allow");
+      expect(payload.judgeModel).toBe("claude-haiku-4-5");
+      // Stored VERBATIM (the audit kind's contract), not a `reason` substring.
+      expect(payload.justification).toBe(JUSTIFICATION);
+      expect(payload.confidence).toBe(0.83);
+    } finally {
+      rmSync(cwd, { recursive: true, force: true });
+    }
+  });
+
+  test("--no-justification-audit (enabled:false) writes no durable record", async () => {
+    const cwd = mkdtempSync(join(tmpdir(), "crewhaus-just-gate-off-"));
+    try {
+      const sink = await openJustificationAuditSink({ cwd, enabled: false });
+      expect(sink).toBeUndefined();
+
+      const sendMessage = buildTool({
+        name: "sendmessage",
+        description: "send a message (justification-gated)",
+        inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
+        requireJustification: true,
+        execute: async () => "sent",
+      });
+
+      await runChatLoop({
+        model: "test-model",
+        instructions: "Acknowledge support tickets the user points you at.",
+        _adapter: makeGatedToolUseAdapter("ack the ticket per the goal"),
+        singleTurn: true,
+        seedMessages: [{ role: "user", content: "ack the ticket" }],
+        tools: [sendMessage],
+        permissionMode: "bypass",
+        // no judge => rule-based default; no sink => no durable record.
+      });
+
+      // Nothing was written under the audit dir (verify on a missing dir is ok/0).
+      const v = await verify(justificationAuditDir(cwd));
+      expect(v.ok).toBe(true);
+      expect(v.recordsChecked).toBe(0);
+    } finally {
+      rmSync(cwd, { recursive: true, force: true });
+    }
+  });
+});

package/src/justification-gate.ts

@@ -0,0 +1,124 @@
+import { join } from "node:path";
+import type { JustificationJudge } from "@crewhaus/permission-engine";
+import type { JustificationAuditSink } from "@crewhaus/runtime-core";
+
+/**
+ * FR-004 — Pillar 3 intent-gate wiring for `crewhaus run`, factored out of
+ * the entry file `index.ts` (which runs a top-level argv switch and so cannot
+ * be imported by a test without executing the CLI). Side-effect-free and
+ * directly unit-testable, mirroring `scope-audit.ts`.
+ *
+ * Two halves of the gate the CLI threads into `runChatLoop`:
+ *
+ *   1. The JUDGE that scores a justification against the compile-time session
+ *      goal — `"rule-based"` (the deterministic default; the CLI leaves
+ *      `justificationJudge` unset so runtime-core falls back to
+ *      `ruleBasedJustificationJudge`) or `"claude"` (the model-backed
+ *      `@crewhaus/justification-judge-claude`).
+ *
+ *   2. The durable AUDIT SINK the gate appends a
+ *      `permission_justification_evaluated` record to — a real
+ *      `@crewhaus/audit-log` rooted at `<cwd>/.crewhaus/audit`. This is the
+ *      hash-chained, tamper-evident artifact the AuditKind was created for;
+ *      the trace-bus `permission_decision` event is ephemeral.
+ *
+ * Resolution precedence for the judge:
+ *   `--justification-judge` flag > spec `security.justification.judge` >
+ *   `"rule-based"`.
+ */
+
+export type JudgeChoice = "rule-based" | "claude";
+
+const VALID_JUDGE_CHOICES: ReadonlyArray<JudgeChoice> = ["rule-based", "claude"];
+
+/** Thrown by `resolveJudgeChoice` on an unrecognised choice. The CLI entry
+ *  file catches it and routes the message through `die()`; tests assert on
+ *  `.choice` / `.message` without the process exiting. */
+export class InvalidJudgeChoiceError extends Error {
+  override readonly name = "InvalidJudgeChoiceError";
+  constructor(readonly choice: string) {
+    super(`invalid --justification-judge "${choice}" — allowed: ${VALID_JUDGE_CHOICES.join(", ")}`);
+  }
+}
+
+function isJudgeChoice(s: string): s is JudgeChoice {
+  return (VALID_JUDGE_CHOICES as ReadonlyArray<string>).includes(s);
+}
+
+/**
+ * Pure resolution of the judge choice from the flag value + spec block.
+ * `flagValue` is the raw `--justification-judge` value (or undefined when the
+ * flag is absent). Throws `InvalidJudgeChoiceError` for any value outside the
+ * allowed set so the caller can `die()` with a friendly message.
+ */
+export function resolveJudgeChoice(
+  flagValue: string | undefined,
+  securityJustification: { judge?: JudgeChoice; model?: string } | undefined,
+): JudgeChoice {
+  const raw = flagValue ?? securityJustification?.judge ?? "rule-based";
+  if (!isJudgeChoice(raw)) throw new InvalidJudgeChoiceError(raw);
+  return raw;
+}
+
+/**
+ * Build the `JustificationJudge` for a resolved choice. Returns `undefined`
+ * for `"rule-based"` so the caller omits `justificationJudge` from
+ * `runChatLoop` (runtime-core then uses `ruleBasedJustificationJudge`, the
+ * documented default for tests/offline runs). For `"claude"` it lazily
+ * imports the model-router + `@crewhaus/justification-judge-claude` so
+ * that model-backed code only loads when actually selected.
+ *
+ * The judge model is resolved through the model-router, so the spec's
+ * `security.justification.model` accepts the full router grammar
+ * (claude-*, openai/*, gemini/*, bedrock/*, local/<m>@<url>) — the judge
+ * package only needs a `ProviderAdapter`. The wire model is the
+ * resolution's *stripped* modelId; the previous hardcoded
+ * `createAnthropicAdapter()` + verbatim model string broke every
+ * non-Anthropic judge with model-not-found.
+ */
+export async function createJustificationJudge(
+  choice: JudgeChoice,
+  model: string | undefined,
+): Promise<JustificationJudge | undefined> {
+  if (choice === "rule-based") return undefined;
+  const { resolveModel } = await import("@crewhaus/model-router");
+  const { createClaudeJustificationJudge } = await import("@crewhaus/justification-judge-claude");
+  const resolution = await resolveModel(model ?? "claude-haiku-4-5");
+  return createClaudeJustificationJudge({
+    adapter: resolution.adapter,
+    model: resolution.modelId,
+  });
+}
+
+/** Directory the run path roots the durable justification audit log at. */
+export function justificationAuditDir(cwd: string): string {
+  return join(cwd, ".crewhaus", "audit");
+}
+
+export type OpenJustificationAuditSinkOptions = {
+  /** Working directory; the sink is rooted at `<cwd>/.crewhaus/audit`. */
+  readonly cwd: string;
+  /**
+   * When false, no durable sink is opened (the gate keeps only its ephemeral
+   * trace-bus event). Defaults to true: `crewhaus run` writes the durable,
+   * hash-chained `permission_justification_evaluated` record by default so
+   * the intent gate's decisions are tamper-evident. The `--no-justification-audit`
+   * flag flips this off for ephemeral/offline runs.
+   */
+  readonly enabled?: boolean;
+};
+
+/**
+ * Open the durable `JustificationAuditSink` the gate appends to. Returns
+ * `undefined` when disabled (so the caller spreads nothing into
+ * `runChatLoop` and the durable path is a no-op). The returned object is a
+ * real `@crewhaus/audit-log` `AuditLog`, which structurally satisfies the
+ * `JustificationAuditSink` seam (its `append({ kind, payload })`).
+ */
+export async function openJustificationAuditSink(
+  opts: OpenJustificationAuditSinkOptions,
+): Promise<JustificationAuditSink | undefined> {
+  if (opts.enabled === false) return undefined;
+  const { openAuditLog } = await import("@crewhaus/audit-log");
+  return await openAuditLog({ rootDir: justificationAuditDir(opts.cwd) });
+}

package/src/scope-audit.test.ts

@@ -0,0 +1,207 @@
+import { describe, expect, test } from "bun:test";
+import { OUTWARD_TOOL_NAMES } from "@crewhaus/tool-builder";
+import type { RegisteredTool, ToolIoCapability } from "@crewhaus/tool-catalog";
+import { fetch as fetchTool } from "@crewhaus/tool-fetch";
+import { imageGenerate } from "@crewhaus/tool-image-generation";
+import { webFetch, webSearch } from "@crewhaus/tool-web";
+import { auditSpecToolNames, auditToolScopes, collectToolNames } from "./scope-audit";
+
+// auditToolScopes reads `.name`, `.scope`, and `.ioCapability`; construct
+// minimal doubles so the test can express the exact triples under audit —
+// including the dangerous "outward name forced internal" case that no
+// built-in fixture can produce (every built-in outward tool is external).
+function mkTool(
+  name: string,
+  scope: "internal" | "external",
+  ioCapability?: ToolIoCapability,
+): RegisteredTool {
+  return { name, scope, ...(ioCapability ? { ioCapability } : {}) } as unknown as RegisteredTool;
+}
+
+describe("auditToolScopes — FR-002 strict scope gate", () => {
+  test("flags an outward-named tool left at scope:'internal' (the red path)", () => {
+    const findings = auditToolScopes([mkTool("Fetch", "internal")]);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.toolName).toBe("Fetch");
+    expect(findings[0]?.reason).toContain('scope is "internal"');
+    expect(findings[0]?.reason).toContain('expected "external"');
+  });
+
+  test("flags a namespaced MCP tool forced internal (prefix rule)", () => {
+    const findings = auditToolScopes([mkTool("mcp__slack__send", "internal")]);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.toolName).toBe("mcp__slack__send");
+  });
+
+  test("does NOT flag an outward tool that is correctly external", () => {
+    expect(auditToolScopes([mkTool("Fetch", "external")])).toHaveLength(0);
+  });
+
+  test("does NOT flag a pure-compute internal tool", () => {
+    expect(auditToolScopes([mkTool("read", "internal")])).toHaveLength(0);
+  });
+
+  test("reports every mis-scoped tool, leaving correct ones unflagged", () => {
+    const findings = auditToolScopes([
+      mkTool("Fetch", "internal"), // flagged
+      mkTool("WebSearch", "external"), // ok
+      mkTool("SendMessage", "internal"), // flagged
+      mkTool("read", "internal"), // ok (internal compute)
+    ]);
+    expect(findings.map((f) => f.toolName).sort()).toEqual(["Fetch", "SendMessage"]);
+  });
+
+  test("an empty tool set produces no findings", () => {
+    expect(auditToolScopes([])).toHaveLength(0);
+  });
+
+  // FR-002 mechanism-2 residual: the capability-driven path. These are the
+  // cases the prior name-only audit could NOT reach — a custom buildTool tool
+  // with a NOVEL name that opens a socket / spawns a process. Now that the
+  // tool declares `ioCapability`, the audit binds scope:"external" to it.
+  test("flags a NOVEL-named custom tool that declares ioCapability:'network' but is internal", () => {
+    const findings = auditToolScopes([mkTool("SomeCustomSocketTool", "internal", "network")]);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.toolName).toBe("SomeCustomSocketTool");
+    expect(findings[0]?.reason).toContain('ioCapability "network"');
+    expect(findings[0]?.reason).toContain('scope is "internal"');
+  });
+
+  test("flags a NOVEL-named custom tool that declares ioCapability:'process' but is internal", () => {
+    const findings = auditToolScopes([mkTool("RunDaemon", "internal", "process")]);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.reason).toContain('ioCapability "process"');
+  });
+
+  test("does NOT flag an io-capable custom tool that correctly set scope:'external'", () => {
+    expect(auditToolScopes([mkTool("SomeCustomSocketTool", "external", "network")])).toHaveLength(
+      0,
+    );
+  });
+
+  test("the capability path fires even for a name the outward-name heuristic would miss", () => {
+    // Proves the gate is no longer a no-op for user-authored tools: the name
+    // is not in OUTWARD_TOOL_NAMES and is not mcp__-prefixed, yet it is flagged.
+    const findings = auditToolScopes([mkTool("totally_internal_looking", "internal", "network")]);
+    expect(findings).toHaveLength(1);
+  });
+
+  test("a tool with NEITHER capability nor outward name is not flagged (documented residual)", () => {
+    // The irreducible limit of a static annotation check: a tool that touches
+    // the network but declares neither its capability nor an outward name is
+    // invisible to the audit. Pinned so the boundary is explicit, not implied.
+    expect(auditToolScopes([mkTool("opaque", "internal")])).toHaveLength(0);
+  });
+});
+
+describe("auditSpecToolNames — FR-002 compile --strict spec-level gate", () => {
+  // Resolver doubles standing in for the offline built-in tool map.
+  const resolveNone = (_name: string): RegisteredTool | undefined => undefined;
+
+  test("flags an unresolved mcp__ sink referenced by a spec (the live red path)", () => {
+    // This is the exact case the adversarial review proved exited 0: a spec
+    // referencing an MCP tool the offline map cannot resolve. --strict now
+    // refuses it because the sink's external scope is unverifiable offline.
+    const findings = auditSpecToolNames(["mcp__evil__exfiltrate"], resolveNone);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.toolName).toBe("mcp__evil__exfiltrate");
+    expect(findings[0]?.reason).toContain("unverifiable offline");
+  });
+
+  test("flags an unresolved outward-by-name built-in (e.g. SendMessage) referenced in a spec", () => {
+    const findings = auditSpecToolNames(["SendMessage"], resolveNone);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.toolName).toBe("SendMessage");
+  });
+
+  test("does NOT flag an unknown NON-outward custom name (offline gate has nothing to assert)", () => {
+    // Documented boundary: a name the offline map doesn't know and whose name
+    // carries no outward signal is left to the live doctor audit / runtime.
+    expect(auditSpecToolNames(["SomeCustomSocketTool"], resolveNone)).toHaveLength(0);
+    expect(auditSpecToolNames(["read", "bash"], resolveNone)).toHaveLength(0);
+  });
+
+  test("defers to the per-tool audit for RESOLVED names (external built-in passes)", () => {
+    const resolve = (name: string) =>
+      name === "fetch" ? mkTool("Fetch", "external", "network") : undefined;
+    expect(auditSpecToolNames(["fetch"], resolve)).toHaveLength(0);
+  });
+
+  test("flags a resolved built-in that an override forced back to internal", () => {
+    const resolve = (name: string) =>
+      name === "fetch" ? mkTool("Fetch", "internal", "network") : undefined;
+    const findings = auditSpecToolNames(["fetch"], resolve);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.toolName).toBe("Fetch");
+  });
+
+  test("mixes resolved-clean, resolved-dirty, and unresolved-outward names correctly", () => {
+    const resolve = (name: string): RegisteredTool | undefined => {
+      if (name === "fetch") return mkTool("Fetch", "external", "network"); // clean
+      if (name === "webFetch") return mkTool("WebFetch", "internal", "network"); // dirty
+      if (name === "read") return mkTool("read", "internal"); // clean internal
+      return undefined; // mcp__x__y → unresolved outward
+    };
+    const findings = auditSpecToolNames(["fetch", "webFetch", "read", "mcp__x__y"], resolve);
+    expect(findings.map((f) => f.toolName).sort()).toEqual(["WebFetch", "mcp__x__y"]);
+  });
+});
+
+describe("collectToolNames — variant-agnostic IR tool extraction", () => {
+  test("collects top-level ir.tools", () => {
+    const ir = { target: "cli", tools: ["fetch", "read"] };
+    expect(collectToolNames(ir).sort()).toEqual(["fetch", "read"]);
+  });
+
+  test("collects nested tools (steps / sub-agents) and dedups", () => {
+    const ir = {
+      target: "workflow",
+      tools: ["fetch"],
+      steps: [{ tools: ["read", "fetch"] }, { tools: ["bash"] }],
+      subAgents: [{ tools: ["webSearch"] }],
+    };
+    expect(collectToolNames(ir).sort()).toEqual(["bash", "fetch", "read", "webSearch"]);
+  });
+
+  test("ignores non-string entries and non-tools keys", () => {
+    const ir = { tools: ["fetch", 42, null], notTools: ["ignored"] };
+    expect(collectToolNames(ir)).toEqual(["fetch"]);
+  });
+
+  test("returns empty for a toolless IR", () => {
+    expect(collectToolNames({ target: "cli", agent: {} })).toEqual([]);
+  });
+});
+
+// FR-002 minor gap #3 — pin the loadToolMap key ↔ RegisteredTool.name ↔
+// scope ↔ ioCapability alignment for the offline-resolvable outward built-ins.
+// collectToolNames yields camelCase KEYS (fetch/webFetch/webSearch/
+// imageGenerate); the audit keys on the PascalCase .name PROPERTY. The strict
+// resolve path only lines up because loadToolMap resolves key→RegisteredTool
+// first. This test asserts the source-of-truth: each outward built-in's .name
+// is in OUTWARD_TOOL_NAMES AND it ships scope:"external" AND it declares
+// ioCapability — so a future refactor that breaks any leg fails here rather
+// than silently making the strict audit match nothing.
+describe("FR-002 invariant — outward built-ins are correctly self-describing", () => {
+  // Imported from the real packages (the same modules loadToolMap() imports),
+  // keyed by the camelCase loadToolMap key they are registered under.
+  const outwardBuiltins: Record<string, RegisteredTool> = {
+    fetch: fetchTool,
+    webFetch,
+    webSearch,
+    imageGenerate,
+  };
+
+  test.each(Object.entries(outwardBuiltins))(
+    "loadToolMap key %s resolves to an external, io-capable, outward-named tool",
+    (_key, tool) => {
+      expect(OUTWARD_TOOL_NAMES.has(tool.name)).toBe(true);
+      expect(tool.scope).toBe("external");
+      expect(tool.ioCapability).toBe("network");
+    },
+  );
+
+  test("each resolved outward built-in passes auditToolScopes (no finding)", () => {
+    expect(auditToolScopes(Object.values(outwardBuiltins))).toHaveLength(0);
+  });
+});