@crewhaus/tool-harness-synthesizer 0.1.0 → 0.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewhaus/tool-harness-synthesizer",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "description": "Track D / §55 — Thompson-sampled tree search over candidate verifier functions for skills and tools. Smaller LLM + synthesized verifier beats larger LLM (AutoHarness, arxiv 2603.03329).",
   "main": "src/index.ts",
@@ -12,14 +12,15 @@
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/errors": "0.0.0",
-    "@crewhaus/prompt-optimizer": "0.0.0"
+    "@crewhaus/errors": "0.1.2",
+    "@crewhaus/prompt-optimizer": "0.1.2",
+    "@crewhaus/sandbox": "0.1.2"
   },
   "license": "Apache-2.0",
   "author": {
     "name": "Max Meier",
-    "email": "max@studiomax.io",
-    "url": "https://studiomax.io"
+    "email": "max@crewhaus.ai",
+    "url": "https://crewhaus.ai"
   },
   "repository": {
     "type": "git",
@@ -31,12 +32,7 @@
     "url": "https://github.com/crewhaus/factory/issues"
   },
   "publishConfig": {
-    "access": "restricted"
+    "access": "public"
   },
-  "files": [
-    "src",
-    "README.md",
-    "LICENSE",
-    "NOTICE"
-  ]
+  "files": ["src", "README.md", "LICENSE", "NOTICE"]
 }

package/src/index.test.ts

@@ -1,11 +1,41 @@
 import { describe, expect, test } from "bun:test";
+import type { OptimizerState } from "@crewhaus/prompt-optimizer";
+import type { Sandbox, SandboxExecOptions, SandboxExecResult } from "@crewhaus/sandbox";
 import {
   HarnessSynthesizerError,
+  VerifierMutationProvider,
   type VerifierSample,
   runVerifier,
   synthesizeVerifier,
   thompsonPick,
 } from "./index";
+import { VERIFIER_SENTINEL, evalVerifierPayload, runVerifierInSandbox } from "./sandboxed-eval";
+
+/**
+ * Test double for `@crewhaus/sandbox`. Reports `backend: "docker"` so it
+ * passes `runVerifier`'s fail-closed gate (the real `noop` backend is
+ * rejected), and reproduces the in-container harness output by running the
+ * SAME `evalVerifierPayload` routine in-process. Deterministic, no docker
+ * daemon. The verifier strings under test are trusted, so evaluating them
+ * in-process here (rather than in a real jail) is safe.
+ */
+class FakeDockerSandbox implements Sandbox {
+  readonly backend = "docker" as const;
+  readonly calls: SandboxExecOptions[] = [];
+  async exec(opts: SandboxExecOptions): Promise<SandboxExecResult> {
+    this.calls.push(opts);
+    const payload = JSON.parse(opts.stdin ?? "{}");
+    const result = evalVerifierPayload(payload);
+    return {
+      stdout: VERIFIER_SENTINEL + JSON.stringify(result),
+      stderr: "",
+      exitCode: 0,
+      timedOut: false,
+      durationMs: 1,
+    };
+  }
+  async close(): Promise<void> {}
+}
 
 const evenSamples: VerifierSample[] = [
   { input: null, output: 0, expected: true },
@@ -16,45 +46,68 @@ const evenSamples: VerifierSample[] = [
 ];
 
 describe("runVerifier", () => {
-  test("scores a correct verifier at 1.0", () => {
-    const r = runVerifier("return typeof output === 'number' && output % 2 === 0", evenSamples);
+  const sandbox = new FakeDockerSandbox();
+
+  test("scores a correct verifier at 1.0", async () => {
+    const r = await runVerifier(
+      "return typeof output === 'number' && output % 2 === 0",
+      evenSamples,
+      { sandbox },
+    );
     expect(r.heuristic).toBe(1);
     expect(r.errors).toBe(0);
     expect(r.verdicts).toEqual([true, false, true, false, true]);
   });
 
-  test("scores a constant-true verifier at the majority class", () => {
-    const r = runVerifier("return true", evenSamples);
+  test("scores a constant-true verifier at the majority class", async () => {
+    const r = await runVerifier("return true", evenSamples, { sandbox });
     // 3 of 5 expected: true → score 0.6
     expect(r.heuristic).toBe(0.6);
   });
 
-  test("captures runtime errors without throwing", () => {
-    const r = runVerifier("throw new Error('boom')", evenSamples);
+  test("captures runtime errors without throwing", async () => {
+    const r = await runVerifier("throw new Error('boom')", evenSamples, { sandbox });
     expect(r.errors).toBe(5);
     expect(r.heuristic).toBe(0.4); // false vs expected: 2 of 5 are expected false
   });
 
-  test("throws on uncompilable code", () => {
-    expect(() => runVerifier("not valid javascript {{{", evenSamples)).toThrow(
+  test("throws on uncompilable code", async () => {
+    await expect(runVerifier("not valid javascript {{{", evenSamples, { sandbox })).rejects.toThrow(
+      HarnessSynthesizerError,
+    );
+  });
+
+  test("fails closed on a noop (non-isolating) sandbox", async () => {
+    const noop: Sandbox = {
+      backend: "noop",
+      exec: async () => {
+        throw new Error("must not run untrusted code under noop");
+      },
+      close: async () => {},
+    };
+    await expect(runVerifier("return true", evenSamples, { sandbox: noop })).rejects.toThrow(
       HarnessSynthesizerError,
     );
   });
+
+  test("ships code + label-free samples to node:22-alpine via stdin", async () => {
+    const sb = new FakeDockerSandbox();
+    await runVerifier("return true", evenSamples, { sandbox: sb });
+    const call = sb.calls[0];
+    expect(call?.image).toBe("node:22-alpine");
+    expect(call?.argv?.[0]).toBe("node");
+    const payload = JSON.parse(call?.stdin ?? "{}");
+    expect(payload.code).toBe("return true");
+    expect(payload.samples).toHaveLength(5);
+    // The jail must never receive the labels it is scored against.
+    expect(JSON.stringify(payload)).not.toContain("expected");
+  });
 });
 
 describe("thompsonPick", () => {
   test("returns 0 for a single candidate", () => {
     const idx = thompsonPick(
-      [
-        {
-          id: "x",
-          code: "return true",
-          score: 1,
-          heuristic: 1,
-          alpha: 10,
-          beta: 1,
-        },
-      ],
+      [{ id: "x", code: "return true", score: 1, heuristic: 1, alpha: 10, beta: 1 }],
       () => 0.5,
     );
     expect(idx).toBe(0);
@@ -65,20 +118,21 @@ describe("thompsonPick", () => {
       { id: "a", code: "1", score: 0.1, heuristic: 0.1, alpha: 1, beta: 9 },
       { id: "b", code: "1", score: 0.9, heuristic: 0.9, alpha: 9, beta: 1 },
     ];
-    // RNG always 0.5 — Marsaglia normal is degenerate; we just verify it
-    // doesn't crash and returns a valid index.
     const idx = thompsonPick(nodes, () => 0.5);
     expect([0, 1]).toContain(idx);
   });
 });
 
 describe("synthesizeVerifier", () => {
+  const sandbox = new FakeDockerSandbox();
+
   test("returns immediately when a seed already meets target", async () => {
     const result = await synthesizeVerifier({
       seedCandidates: ["return typeof output === 'number' && output % 2 === 0"],
       samples: evenSamples,
       refiner: async () => "throw new Error('should not be called')",
       target: 1.0,
+      sandbox,
     });
     expect(result.converged).toBe(true);
     expect(result.iterations).toBe(0);
@@ -86,9 +140,6 @@ describe("synthesizeVerifier", () => {
   });
 
   test("converges via refiner when seed is poor", async () => {
-    // Start with a constant-true seed; refiner produces the correct
-    // verifier on the first call. This proves the search loop wires
-    // refiner → score → pool update correctly.
     const result = await synthesizeVerifier({
       seedCandidates: ["return true"],
       samples: evenSamples,
@@ -96,6 +147,7 @@ describe("synthesizeVerifier", () => {
       target: 1.0,
       maxIterations: 3,
       rng: () => 0.5,
+      sandbox,
     });
     expect(result.converged).toBe(true);
     expect(result.best.heuristic).toBe(1);
@@ -109,11 +161,34 @@ describe("synthesizeVerifier", () => {
       target: 1.0,
       maxIterations: 3,
       rng: () => 0.5,
+      sandbox,
     });
     expect(result.converged).toBe(false);
     expect(result.best.heuristic).toBeGreaterThanOrEqual(0.6);
   });
 
+  test("picks the higher-heuristic seed when multiple seeds are supplied", async () => {
+    // Two seeds → the `pool.reduce` comparator runs (with one seed reduce
+    // returns the lone element without invoking the callback). The perfect
+    // seed must win and short-circuit at iteration 0.
+    const result = await synthesizeVerifier({
+      seedCandidates: [
+        "return false", // heuristic 0.4
+        "return typeof output === 'number' && output % 2 === 0", // heuristic 1.0
+      ],
+      samples: evenSamples,
+      refiner: async () => "throw new Error('should not be called')",
+      target: 1.0,
+      rng: () => 0.5,
+      sandbox,
+    });
+    expect(result.converged).toBe(true);
+    expect(result.iterations).toBe(0);
+    expect(result.best.id).toBe("seed_1");
+    expect(result.best.heuristic).toBe(1);
+    expect(result.trajectory).toHaveLength(2);
+  });
+
   test("throws on empty seed candidates", async () => {
     await expect(
       synthesizeVerifier({
@@ -134,3 +209,223 @@ describe("synthesizeVerifier", () => {
     ).rejects.toThrow(HarnessSynthesizerError);
   });
 });
+
+/**
+ * A sandbox whose container call returns a caller-supplied result verbatim, so
+ * tests can drive `runVerifierInSandbox`'s error branches (timeout, nonzero
+ * exit, empty/garbled output) deterministically without a real container.
+ */
+class ScriptedSandbox implements Sandbox {
+  readonly backend = "docker" as const;
+  readonly calls: SandboxExecOptions[] = [];
+  constructor(private readonly result: SandboxExecResult) {}
+  async exec(opts: SandboxExecOptions): Promise<SandboxExecResult> {
+    this.calls.push(opts);
+    return this.result;
+  }
+  async close(): Promise<void> {}
+}
+
+const ioSamples: VerifierSample[] = [{ input: null, output: 0, expected: true }];
+
+describe("VerifierMutationProvider", () => {
+  const baseState = (prompt: string): OptimizerState =>
+    ({
+      iteration: 0,
+      best: { id: "c0", prompt, mutations: [], score: 0.5 },
+      trajectory: [],
+      trainSet: [],
+      devSet: [],
+    }) as OptimizerState;
+
+  test("next() runs the inner search and appends a verifier annotation", async () => {
+    const sandbox = new FakeDockerSandbox();
+    const provider = new VerifierMutationProvider(
+      evenSamples,
+      // Seed is already perfect, so the inner search converges at iteration 0.
+      async () => "return false",
+      ["return typeof output === 'number' && output % 2 === 0"],
+      4,
+      sandbox,
+    );
+    expect(provider.name).toBe("verifier-synthesis");
+    const result = await provider.next(baseState("BASE PROMPT"));
+    expect(result.prompt.startsWith("BASE PROMPT")).toBe(true);
+    expect(result.prompt).toContain("[verifier seed_0, h=1.000]");
+    expect(result.mutations).toEqual([{ kind: "rephrase-instruction" }]);
+    expect(result.rationale).toContain("verifier-synthesis pass 1");
+    expect(result.rationale).toContain("(converged)");
+  });
+
+  test("next() increments the synthesis pass counter across calls", async () => {
+    const sandbox = new FakeDockerSandbox();
+    const provider = new VerifierMutationProvider(
+      evenSamples,
+      async () => "return typeof output === 'number' && output % 2 === 0",
+      ["return typeof output === 'number' && output % 2 === 0"],
+      4,
+      sandbox,
+    );
+    const first = await provider.next(baseState("P"));
+    const second = await provider.next(baseState("P"));
+    expect(first.rationale).toContain("pass 1");
+    expect(second.rationale).toContain("pass 2");
+  });
+
+  test("omits the sandbox option when none is injected (default backend path)", async () => {
+    // No sandbox passed → synthesizeVerifier creates the default backend. Pin it
+    // to `noop` so the fail-closed gate throws BEFORE any container is spawned —
+    // this exercises the `this.sandbox === undefined` branch with zero real I/O.
+    const prev = process.env["CREWHAUS_SANDBOX"];
+    process.env["CREWHAUS_SANDBOX"] = "noop";
+    try {
+      const provider = new VerifierMutationProvider(
+        evenSamples,
+        async () => "return true",
+        ["return true"],
+        1,
+      );
+      await expect(provider.next(baseState("P"))).rejects.toThrow(HarnessSynthesizerError);
+    } finally {
+      // Restore exactly: an unset var must be *removed*, not set to the string
+      // "undefined" (which `readEnvBackend` would reject). `Reflect.deleteProperty`
+      // does this without tripping biome's noDelete rule.
+      if (prev === undefined) Reflect.deleteProperty(process.env, "CREWHAUS_SANDBOX");
+      else process.env["CREWHAUS_SANDBOX"] = prev;
+    }
+  });
+
+  test("non-converged inner search omits the (converged) marker", async () => {
+    const sandbox = new FakeDockerSandbox();
+    const provider = new VerifierMutationProvider(
+      evenSamples,
+      async () => "return false", // 0.4 — never reaches 1.0
+      ["return false"],
+      2,
+      sandbox,
+    );
+    const result = await provider.next(baseState("P"));
+    expect(result.rationale).not.toContain("(converged)");
+  });
+});
+
+describe("runVerifierInSandbox error branches", () => {
+  test("throws on non-JSON-serializable samples (BigInt)", async () => {
+    const sandbox = new FakeDockerSandbox();
+    const bad: VerifierSample[] = [{ input: 1n, output: 0, expected: true }];
+    await expect(runVerifierInSandbox(sandbox, "return true", bad)).rejects.toThrow(
+      /not JSON-serializable/,
+    );
+    // Nothing was shipped to the jail.
+    expect(sandbox.calls).toHaveLength(0);
+  });
+
+  test("throws when the harness times out", async () => {
+    const sandbox = new ScriptedSandbox({
+      stdout: "",
+      stderr: "",
+      exitCode: 0,
+      timedOut: true,
+      durationMs: 1,
+    });
+    await expect(
+      runVerifierInSandbox(sandbox, "return true", ioSamples, { timeoutMs: 1234 }),
+    ).rejects.toThrow(/timed out after 1234ms/);
+  });
+
+  test("throws with stderr tail when the harness exits nonzero and emits no result", async () => {
+    const sandbox = new ScriptedSandbox({
+      stdout: "no sentinel here",
+      stderr: "fatal: allocation failure",
+      exitCode: 137,
+      timedOut: false,
+      durationMs: 1,
+    });
+    await expect(runVerifierInSandbox(sandbox, "return true", ioSamples)).rejects.toThrow(
+      /exited 137: fatal: allocation failure/,
+    );
+  });
+
+  test("throws 'no result' when exit is clean but no sentinel is present", async () => {
+    const sandbox = new ScriptedSandbox({
+      stdout: "garbage without the sentinel",
+      stderr: "",
+      exitCode: 0,
+      timedOut: false,
+      durationMs: 1,
+    });
+    await expect(runVerifierInSandbox(sandbox, "return true", ioSamples)).rejects.toThrow(
+      /produced no result/,
+    );
+  });
+
+  test("throws on a verdict-count mismatch", async () => {
+    // Sentinel present, valid JSON, but two verdicts for one sample.
+    const sandbox = new ScriptedSandbox({
+      stdout: `${VERIFIER_SENTINEL}${JSON.stringify({ verdicts: [true, false], errors: 0 })}`,
+      stderr: "",
+      exitCode: 0,
+      timedOut: false,
+      durationMs: 1,
+    });
+    await expect(runVerifierInSandbox(sandbox, "return true", ioSamples)).rejects.toThrow(
+      /returned 2 verdicts for 1 samples/,
+    );
+  });
+
+  test("surfaces a compileError reported by the harness", async () => {
+    const sandbox = new ScriptedSandbox({
+      stdout: `${VERIFIER_SENTINEL}${JSON.stringify({ compileError: "Unexpected token" })}`,
+      stderr: "",
+      exitCode: 0,
+      timedOut: false,
+      durationMs: 1,
+    });
+    await expect(runVerifierInSandbox(sandbox, "bad {{", ioSamples)).rejects.toThrow(
+      /did not compile: Unexpected token/,
+    );
+  });
+});
+
+describe("parseHarnessResult edge cases (via runVerifierInSandbox)", () => {
+  test("treats a non-JSON result line after the sentinel as no result", async () => {
+    // Exercises the JSON.parse catch inside parseHarnessResult (returns undefined).
+    const sandbox = new ScriptedSandbox({
+      stdout: `${VERIFIER_SENTINEL}this-is-not-json`,
+      stderr: "",
+      exitCode: 0,
+      timedOut: false,
+      durationMs: 1,
+    });
+    await expect(runVerifierInSandbox(sandbox, "return true", ioSamples)).rejects.toThrow(
+      /produced no result/,
+    );
+  });
+
+  test("treats a sentinel-framed object without a verdicts array as no result", async () => {
+    // Valid JSON object, but neither a compileError string nor a verdicts array
+    // → parseHarnessResult falls through to its final `return undefined`.
+    const sandbox = new ScriptedSandbox({
+      stdout: `${VERIFIER_SENTINEL}${JSON.stringify({ verdicts: "not-an-array" })}`,
+      stderr: "",
+      exitCode: 0,
+      timedOut: false,
+      durationMs: 1,
+    });
+    await expect(runVerifierInSandbox(sandbox, "return true", ioSamples)).rejects.toThrow(
+      /produced no result/,
+    );
+  });
+});
+
+describe("runVerifier (live docker — gated by CREWHAUS_VERIFIER_LIVE_DOCKER=1)", () => {
+  test("scores a basic verifier in a real container", async () => {
+    if (process.env["CREWHAUS_VERIFIER_LIVE_DOCKER"] !== "1") return; // skipped by default
+    const r = await runVerifier(
+      "return typeof output === 'number' && output % 2 === 0",
+      evenSamples,
+    );
+    expect(r.heuristic).toBe(1);
+    expect(r.errors).toBe(0);
+  });
+});

package/src/index.ts

@@ -25,12 +25,15 @@
  * variant that proposes verifier-aware prompt edits.
  *
  * v0 ships:
- *   - `synthesizeVerifier(spec)` — pure tree search over candidate
- *     verifier code strings (the LLM call is supplied by the caller
- *     so this package stays pure)
+ *   - `synthesizeVerifier(spec)` — tree search over candidate verifier
+ *     code strings (the LLM call is supplied by the caller so this
+ *     package stays pure with respect to the model)
  *   - `thompsonPick(nodes)` — Thompson sampling over tree nodes
  *   - `VerifierMutationProvider` — adapter to plug verifier search
  *     into the existing optimizer
+ *   - `runVerifier(code, samples)` — scores a candidate by running it
+ *     inside a locked-down `@crewhaus/sandbox` container; the candidate
+ *     code never executes on this host (see its SECURITY note).
  *
  * Cited paper: AutoHarness (arxiv 2603.03329, Lou et al., 2026-03).
  */
@@ -40,6 +43,8 @@ import type {
   OptimizerState,
   ProviderMutation,
 } from "@crewhaus/prompt-optimizer";
+import { type Sandbox, createSandbox } from "@crewhaus/sandbox";
+import { runVerifierInSandbox } from "./sandboxed-eval";
 
 export class HarnessSynthesizerError extends CrewhausError {
   override readonly name = "HarnessSynthesizerError";
@@ -54,6 +59,11 @@ export class HarnessSynthesizerError extends CrewhausError {
  * (false) this sample. Both classes are required for non-degenerate
  * search — a verifier that returns `true` for everything passes the
  * `expected: true` set perfectly.
+ *
+ * NOTE: `input`/`output` cross an isolation boundary (see `runVerifier`)
+ * and are marshaled as JSON, so they must be JSON-serializable. The
+ * verifier observes structured copies, not live host references — an
+ * intentional consequence of running it out-of-process.
  */
 export type VerifierSample = {
   readonly input: unknown;
@@ -66,8 +76,15 @@ export type VerifierSample = {
  * achieved on the last evaluation. `code` is a function body string
  * with the signature `(input: unknown, output: unknown) => boolean`.
  * It's stored as a string so the search can mutate it and feed it
- * back to the LLM. Execution happens via `runVerifier` which
- * compiles + invokes safely in a sandboxed Function call.
+ * back to the LLM.
+ *
+ * SECURITY: `code` is attacker-influenceable (caller seed candidates +
+ * refiner output, which a future model-backed refiner derives from
+ * skill/tool I/O that can carry injected content). It is NEVER compiled
+ * or invoked on this host. Execution happens via `runVerifier`, which
+ * evaluates the candidate inside a locked-down `@crewhaus/sandbox`
+ * container with no ambient authority — see the SECURITY note on
+ * `runVerifier`.
  */
 export type VerifierCandidate = {
   readonly id: string;
@@ -85,50 +102,81 @@ export type VerifierCandidate = {
  * returns a new code string. In production, this is a model call; for
  * testing it's a deterministic rule-based mutation. Either way, the
  * signature is the same.
+ *
+ * SECURITY: the string this returns is executed (in isolation — see
+ * `runVerifier`), so a model-backed refiner is effectively running
+ * model output as code. The container is the trust boundary, NOT the
+ * model — never relax `runVerifier`'s isolation on the assumption that
+ * refiner output is trustworthy.
  */
 export type RefinerFn = (
   current: VerifierCandidate,
   failures: ReadonlyArray<VerifierSample>,
 ) => Promise<string>;
 
+/** Isolation options shared by `runVerifier` and the inner search. */
+export type RunVerifierOptions = {
+  /**
+   * Isolation backend. Defaults to `createSandbox()` (honors the
+   * `CREWHAUS_SANDBOX` env; `docker` by default). The non-isolating
+   * `noop` backend is REFUSED — see the SECURITY note on `runVerifier`.
+   */
+  readonly sandbox?: Sandbox;
+  /** Per-evaluation wall-clock budget (ms). Default: 10_000. */
+  readonly timeoutMs?: number;
+  /** Container image. Default: `node:22-alpine` (on the sandbox allowlist). */
+  readonly image?: string;
+};
+
 /**
  * The Critic: runs `code` against a sample set and returns the per-
- * sample verdict and the heuristic value. Pure; deterministic given
- * the same code + samples.
+ * sample verdict and the heuristic value. Deterministic given the same
+ * code + samples (modulo the verifier's own determinism).
+ *
+ * SECURITY (FR-007): `code` is untrusted and is NOT run in this process.
+ * `runVerifier` ships `{ code, samples }` to a locked-down
+ * `@crewhaus/sandbox` container (`--network none`, `--read-only`, no
+ * host env, cpu/mem caps, wall-clock kill, `no-new-privileges`) and
+ * scores the returned verdicts here on the host — the container never
+ * receives `expected`. Inside the jail the candidate is compiled with
+ * the runtime's dynamic-function constructor and invoked over the
+ * samples; an exfiltration/RCE attempt (e.g. a refiner emitting a
+ * network call) is contained by the jail. FAIL CLOSED: when no real
+ * isolation backend is available (the `noop` backend, or none),
+ * `runVerifier` throws rather than executing untrusted code unsandboxed,
+ * mirroring the `requiresSandbox` floor enforced elsewhere in the repo.
+ *
+ * Behavior contract: per-sample runtime errors are caught (verdict
+ * `false`, `errors++`), matching the prior in-process semantics. Code
+ * that fails to compile, times out, or otherwise crashes the harness
+ * throws a `HarnessSynthesizerError`.
  */
-export function runVerifier(
+export async function runVerifier(
   code: string,
   samples: ReadonlyArray<VerifierSample>,
-): {
+  opts: RunVerifierOptions = {},
+): Promise<{
   readonly verdicts: ReadonlyArray<boolean>;
   readonly heuristic: number;
   readonly errors: number;
-} {
-  let fn: (input: unknown, output: unknown) => boolean;
+}> {
+  const ownsSandbox = opts.sandbox === undefined;
+  const sandbox = opts.sandbox ?? createSandbox();
   try {
-    // Code must be the body of a function with parameters (input, output).
-    // We wrap defensively so callers can pass either a body or a complete
-    // expression returning a function.
-    fn = new Function("input", "output", `${code}`) as (input: unknown, output: unknown) => boolean;
-  } catch (err) {
-    throw new HarnessSynthesizerError(`verifier code did not compile: ${(err as Error).message}`);
-  }
-  const verdicts: boolean[] = [];
-  let correct = 0;
-  let errors = 0;
-  for (const s of samples) {
-    let v: boolean;
-    try {
-      v = Boolean(fn(s.input, s.output));
-    } catch {
-      v = false;
-      errors++;
+    // FR-007 — fail closed: never run untrusted verifier code without
+    // a real isolation boundary.
+    if (sandbox.backend === "noop") {
+      throw new HarnessSynthesizerError(
+        "refusing to evaluate verifier code in a noop sandbox (no isolation); set CREWHAUS_SANDBOX=docker|podman",
+      );
     }
-    verdicts.push(v);
-    if (v === s.expected) correct++;
+    return await runVerifierInSandbox(sandbox, code, samples, {
+      ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
+      ...(opts.image !== undefined ? { image: opts.image } : {}),
+    });
+  } finally {
+    if (ownsSandbox) await sandbox.close();
   }
-  const heuristic = samples.length === 0 ? 0 : correct / samples.length;
-  return { verdicts, heuristic, errors };
 }
 
 /**
@@ -210,6 +258,14 @@ export type SynthesizeOptions = {
   readonly target?: number;
   /** RNG for Thompson sampling. Default: Math.random. */
   readonly rng?: () => number;
+  /**
+   * Isolation backend reused across every evaluation in the inner search.
+   * Defaults to one `createSandbox()` per call (closed when the search
+   * ends). See `runVerifier` for the isolation/fail-closed contract.
+   */
+  readonly sandbox?: Sandbox;
+  /** Per-evaluation wall-clock budget (ms) forwarded to `runVerifier`. */
+  readonly timeoutMs?: number;
 };
 
 export type SynthesizeResult = {
@@ -223,6 +279,9 @@ export type SynthesizeResult = {
  * Run the tree search. Returns the best candidate found, the
  * iteration count, and whether the target heuristic was reached.
  * Pure with respect to randomness: pass `rng` for determinism.
+ *
+ * One isolation backend is created (or reused, if `opts.sandbox` is
+ * supplied) for the whole search and closed on exit — see `runVerifier`.
  */
 export async function synthesizeVerifier(opts: SynthesizeOptions): Promise<SynthesizeResult> {
   if (opts.seedCandidates.length === 0) {
@@ -235,66 +294,79 @@ export async function synthesizeVerifier(opts: SynthesizeOptions): Promise<Synth
   const rng = opts.rng ?? Math.random;
   const maxIter = opts.maxIterations ?? 16;
 
-  // Initialize the candidate pool from seeds.
-  const pool: VerifierCandidate[] = [];
-  for (let i = 0; i < opts.seedCandidates.length; i++) {
-    const code = opts.seedCandidates[i] as string;
-    const { heuristic } = runVerifier(code, opts.samples);
-    pool.push({
-      id: `seed_${i}`,
-      code,
-      score: heuristic,
-      heuristic,
-      // Beta starts uniform; update with observed correct/incorrect counts.
-      alpha: 1 + Math.round(heuristic * opts.samples.length),
-      beta: 1 + Math.round((1 - heuristic) * opts.samples.length),
-    });
-  }
-  const trajectory: VerifierCandidate[] = [...pool];
-
-  // Early exit if a seed already satisfies the target.
-  let best = pool.reduce((a, b) => (a.heuristic >= b.heuristic ? a : b));
-  if (best.heuristic >= target) {
-    return { best, iterations: 0, converged: true, trajectory };
-  }
+  // One isolation backend reused across the whole inner search; close it
+  // on exit only if we created it (the caller owns an injected one).
+  const ownsSandbox = opts.sandbox === undefined;
+  const sandbox = opts.sandbox ?? createSandbox();
+  const runOpts: RunVerifierOptions = {
+    sandbox,
+    ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
+  };
 
-  for (let iter = 0; iter < maxIter; iter++) {
-    const pickIdx = thompsonPick(pool, rng);
-    const parent = pool[pickIdx] as VerifierCandidate;
-    // Compute concrete failures for the refiner.
-    const { verdicts } = runVerifier(parent.code, opts.samples);
-    const failures: VerifierSample[] = [];
-    for (let i = 0; i < opts.samples.length; i++) {
-      const s = opts.samples[i] as VerifierSample;
-      const v = verdicts[i] as boolean;
-      if (v !== s.expected) failures.push(s);
-    }
-    let newCode: string;
-    try {
-      newCode = await opts.refiner(parent, failures);
-    } catch (err) {
-      throw new HarnessSynthesizerError(
-        `refiner threw on iteration ${iter}: ${(err as Error).message}`,
-        err,
-      );
+  try {
+    // Initialize the candidate pool from seeds.
+    const pool: VerifierCandidate[] = [];
+    for (let i = 0; i < opts.seedCandidates.length; i++) {
+      const code = opts.seedCandidates[i] as string;
+      const { heuristic } = await runVerifier(code, opts.samples, runOpts);
+      pool.push({
+        id: `seed_${i}`,
+        code,
+        score: heuristic,
+        heuristic,
+        // Beta starts uniform; update with observed correct/incorrect counts.
+        alpha: 1 + Math.round(heuristic * opts.samples.length),
+        beta: 1 + Math.round((1 - heuristic) * opts.samples.length),
+      });
     }
-    const { heuristic } = runVerifier(newCode, opts.samples);
-    const child: VerifierCandidate = {
-      id: `cand_${iter}`,
-      code: newCode,
-      score: heuristic,
-      heuristic,
-      alpha: 1 + Math.round(heuristic * opts.samples.length),
-      beta: 1 + Math.round((1 - heuristic) * opts.samples.length),
-    };
-    pool.push(child);
-    trajectory.push(child);
-    if (heuristic > best.heuristic) best = child;
+    const trajectory: VerifierCandidate[] = [...pool];
+
+    // Early exit if a seed already satisfies the target.
+    let best = pool.reduce((a, b) => (a.heuristic >= b.heuristic ? a : b));
     if (best.heuristic >= target) {
-      return { best, iterations: iter + 1, converged: true, trajectory };
+      return { best, iterations: 0, converged: true, trajectory };
     }
+
+    for (let iter = 0; iter < maxIter; iter++) {
+      const pickIdx = thompsonPick(pool, rng);
+      const parent = pool[pickIdx] as VerifierCandidate;
+      // Compute concrete failures for the refiner.
+      const { verdicts } = await runVerifier(parent.code, opts.samples, runOpts);
+      const failures: VerifierSample[] = [];
+      for (let i = 0; i < opts.samples.length; i++) {
+        const s = opts.samples[i] as VerifierSample;
+        const v = verdicts[i] as boolean;
+        if (v !== s.expected) failures.push(s);
+      }
+      let newCode: string;
+      try {
+        newCode = await opts.refiner(parent, failures);
+      } catch (err) {
+        throw new HarnessSynthesizerError(
+          `refiner threw on iteration ${iter}: ${(err as Error).message}`,
+          err,
+        );
+      }
+      const { heuristic } = await runVerifier(newCode, opts.samples, runOpts);
+      const child: VerifierCandidate = {
+        id: `cand_${iter}`,
+        code: newCode,
+        score: heuristic,
+        heuristic,
+        alpha: 1 + Math.round(heuristic * opts.samples.length),
+        beta: 1 + Math.round((1 - heuristic) * opts.samples.length),
+      };
+      pool.push(child);
+      trajectory.push(child);
+      if (heuristic > best.heuristic) best = child;
+      if (best.heuristic >= target) {
+        return { best, iterations: iter + 1, converged: true, trajectory };
+      }
+    }
+    return { best, iterations: maxIter, converged: false, trajectory };
+  } finally {
+    if (ownsSandbox) await sandbox.close();
   }
-  return { best, iterations: maxIter, converged: false, trajectory };
 }
 
 /**
@@ -310,6 +382,11 @@ export async function synthesizeVerifier(opts: SynthesizeOptions): Promise<Synth
  * a freshly-synthesized verifier persisted to .crewhaus/verifiers/.
  * (CLI `--mutator verifier-synthesis` wiring is a follow-up; the CLI
  * today exposes `rule-based` and `claude` only.)
+ *
+ * SECURITY: the inner search executes refiner-produced code. Pass a
+ * `sandbox` (or rely on the default) — `synthesizeVerifier`/`runVerifier`
+ * isolate every evaluation and fail closed without a real backend. A
+ * model-backed `refiner` is only safe to wire because of that jail.
  */
 export class VerifierMutationProvider implements MutationProvider {
   readonly name = "verifier-synthesis";
@@ -320,6 +397,7 @@ export class VerifierMutationProvider implements MutationProvider {
     private readonly refiner: RefinerFn,
     private readonly seedCandidates: ReadonlyArray<string>,
     private readonly maxInnerIterations: number = 4,
+    private readonly sandbox?: Sandbox,
   ) {}
 
   async next(state: OptimizerState): Promise<ProviderMutation> {
@@ -329,6 +407,7 @@ export class VerifierMutationProvider implements MutationProvider {
       samples: this.samples,
       refiner: this.refiner,
       maxIterations: this.maxInnerIterations,
+      ...(this.sandbox !== undefined ? { sandbox: this.sandbox } : {}),
     });
     const annotation = `\n\n[verifier ${result.best.id}, h=${result.best.heuristic.toFixed(3)}]`;
     return {

package/src/sandboxed-eval.ts

@@ -0,0 +1,192 @@
+/**
+ * FR-007 — sandboxed evaluation of untrusted verifier code.
+ *
+ * This module is the ONLY place candidate verifier code is compiled and
+ * run. It is `tool-harness-synthesizer`'s trust boundary: `code` strings
+ * are attacker-influenceable (caller seeds + refiner output), so they are
+ * NEVER executed on this host. `runVerifierInSandbox` ships them to a
+ * locked-down `@crewhaus/sandbox` container (`--network none`,
+ * `--read-only`, no host env, cpu/mem caps, wall-clock kill,
+ * `no-new-privileges`) and scores the returned verdicts on the host —
+ * the container never receives the `expected` labels.
+ *
+ * The container runs `VERIFIER_HARNESS` (a fixed `node -e` script). The
+ * untrusted `code` is delivered as STDIN DATA, never interpolated into
+ * that script string — so it cannot break out of the harness.
+ */
+import type { Sandbox } from "@crewhaus/sandbox";
+import { HarnessSynthesizerError, type VerifierSample } from "./index";
+
+/** Sentinel that frames the harness's single JSON result line on stdout. */
+export const VERIFIER_SENTINEL = "__CREWHAUS_VERIFIER__";
+
+const VERIFIER_IMAGE = "node:22-alpine";
+const DEFAULT_TIMEOUT_MS = 10_000;
+
+/** Input/output pair handed to the verifier (labels stripped). */
+type VerifierIO = { readonly input: unknown; readonly output: unknown };
+
+/** Payload marshaled across the isolation boundary. */
+type VerifierPayload = { readonly code: string; readonly samples: ReadonlyArray<VerifierIO> };
+
+/** Per-sample result from inside the jail (or the in-process test double). */
+export type VerifierEvalResult =
+  | { readonly verdicts: ReadonlyArray<boolean>; readonly errors: number }
+  | { readonly compileError: string };
+
+/**
+ * Compile `payload.code` and run it over `payload.samples`, collecting a
+ * boolean verdict per sample. A compile failure returns `{ compileError }`;
+ * a per-sample runtime throw is caught (verdict `false`, `errors++`).
+ *
+ * Single source of truth for verifier scoring: it runs both (a) inside the
+ * container — `VERIFIER_HARNESS` embeds this function's own source via
+ * `.toString()` — and (b) in unit tests via a fake sandbox. It MUST stay
+ * self-contained: reference no imports or module scope, or the in-container
+ * copy throws a ReferenceError.
+ */
+export function evalVerifierPayload(payload: VerifierPayload): VerifierEvalResult {
+  let fn: (input: unknown, output: unknown) => unknown;
+  try {
+    fn = new Function("input", "output", String(payload.code)) as (
+      input: unknown,
+      output: unknown,
+    ) => unknown;
+  } catch (err) {
+    return { compileError: err instanceof Error ? err.message : String(err) };
+  }
+  const verdicts: boolean[] = [];
+  let errors = 0;
+  const samples = payload.samples || [];
+  for (let i = 0; i < samples.length; i++) {
+    try {
+      const s = samples[i] as VerifierIO;
+      verdicts.push(Boolean(fn(s.input, s.output)));
+    } catch {
+      verdicts.push(false);
+      errors++;
+    }
+  }
+  return { verdicts, errors };
+}
+
+/**
+ * The in-container harness (`node -e <this>`). Reads `{ code, samples }`
+ * from stdin, runs `evalVerifierPayload` (embedded verbatim), and writes
+ * `SENTINEL + JSON` once on stdout. Untrusted `code` is stdin DATA, never
+ * part of this script.
+ */
+const VERIFIER_HARNESS = [
+  '"use strict";',
+  `const SENTINEL = ${JSON.stringify(VERIFIER_SENTINEL)};`,
+  `const evalVerifierPayload = ${evalVerifierPayload.toString()};`,
+  'let raw = "";',
+  'try { raw = require("fs").readFileSync(0, "utf8"); } catch (e) { raw = ""; }',
+  "let payload;",
+  "try { payload = JSON.parse(raw); }",
+  'catch (e) { process.stdout.write(SENTINEL + JSON.stringify({ compileError: "invalid harness payload" })); process.exit(0); }',
+  "process.stdout.write(SENTINEL + JSON.stringify(evalVerifierPayload(payload)));",
+].join("\n");
+
+function parseHarnessResult(stdout: string): VerifierEvalResult | undefined {
+  const idx = stdout.lastIndexOf(VERIFIER_SENTINEL);
+  if (idx === -1) return undefined;
+  const tail = stdout.slice(idx + VERIFIER_SENTINEL.length);
+  const line = tail.split("\n", 1)[0] ?? "";
+  let obj: unknown;
+  try {
+    obj = JSON.parse(line);
+  } catch {
+    return undefined;
+  }
+  if (typeof obj !== "object" || obj === null) return undefined;
+  if (
+    "compileError" in obj &&
+    typeof (obj as { compileError: unknown }).compileError === "string"
+  ) {
+    return { compileError: (obj as { compileError: string }).compileError };
+  }
+  const rec = obj as { verdicts?: unknown; errors?: unknown };
+  if (Array.isArray(rec.verdicts)) {
+    return {
+      verdicts: rec.verdicts.map((v) => Boolean(v)),
+      errors: typeof rec.errors === "number" ? rec.errors : 0,
+    };
+  }
+  return undefined;
+}
+
+/**
+ * Marshal `{ code, samples }` (labels stripped) into the jail, run the
+ * harness, parse the framed result, and score verdicts against the
+ * host-held `expected` labels.
+ *
+ * Throws `HarnessSynthesizerError` on: non-serializable samples, compile
+ * failure, timeout, non-zero harness exit, or unparseable output. Caught
+ * per-sample runtime errors are reflected in `errors` (not thrown).
+ */
+export async function runVerifierInSandbox(
+  sandbox: Sandbox,
+  code: string,
+  samples: ReadonlyArray<VerifierSample>,
+  opts: { readonly timeoutMs?: number; readonly image?: string } = {},
+): Promise<{
+  readonly verdicts: ReadonlyArray<boolean>;
+  readonly heuristic: number;
+  readonly errors: number;
+}> {
+  // Strip `expected` — the jail produces verdicts; the host scores them.
+  const payload: VerifierPayload = {
+    code,
+    samples: samples.map((s) => ({ input: s.input, output: s.output })),
+  };
+  let stdin: string;
+  try {
+    // JSON.stringify throws on BigInt and circular references — surface
+    // that as a clear error rather than shipping a broken payload.
+    stdin = JSON.stringify(payload);
+  } catch (err) {
+    throw new HarnessSynthesizerError(
+      `verifier samples are not JSON-serializable: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const result = await sandbox.exec({
+    image: opts.image ?? VERIFIER_IMAGE,
+    argv: ["node", "-e", VERIFIER_HARNESS],
+    stdin,
+    timeoutMs,
+  });
+
+  if (result.timedOut) {
+    throw new HarnessSynthesizerError(`verifier evaluation timed out after ${timeoutMs}ms`);
+  }
+  const parsed = parseHarnessResult(result.stdout);
+  if (parsed === undefined) {
+    if (result.exitCode !== 0) {
+      throw new HarnessSynthesizerError(
+        `verifier harness exited ${result.exitCode}: ${result.stderr.slice(-500)}`,
+      );
+    }
+    throw new HarnessSynthesizerError("verifier harness produced no result");
+  }
+  if ("compileError" in parsed) {
+    throw new HarnessSynthesizerError(`verifier code did not compile: ${parsed.compileError}`);
+  }
+  if (parsed.verdicts.length !== samples.length) {
+    throw new HarnessSynthesizerError(
+      `verifier harness returned ${parsed.verdicts.length} verdicts for ${samples.length} samples`,
+    );
+  }
+
+  let correct = 0;
+  const verdicts: boolean[] = [];
+  for (let i = 0; i < samples.length; i++) {
+    const v = parsed.verdicts[i] === true;
+    verdicts.push(v);
+    if (v === (samples[i] as VerifierSample).expected) correct++;
+  }
+  const heuristic = samples.length === 0 ? 0 : correct / samples.length;
+  return { verdicts, heuristic, errors: parsed.errors };
+}