@hassangameryt/freeflow 0.3.0 → 0.4.0

package/router/dist/benchmarks/codex-qa-benchmarks.js

@@ -3,459 +3,498 @@ import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { performance } from "node:perf_hooks";
-import { approximateTokens, defaultJsonRunReportPath, escapeMarkdownTableCell as escapeTable, formatPercent, latencySummary, normalizeIterations, parseBenchmarkCliArgs, reductionPercent, writeBenchmarkReportPair, } from "./benchmark-harness.js";
+import {
+  approximateTokens,
+  defaultJsonRunReportPath,
+  escapeMarkdownTableCell as escapeTable,
+  formatPercent,
+  latencySummary,
+  normalizeIterations,
+  parseBenchmarkCliArgs,
+  reductionPercent,
+  writeBenchmarkReportPair,
+} from "./benchmark-harness.js";
 import { freeflowSearch } from "../tools/search.js";
-const SANDBOX_DOC_PATH = "docs/codex-cli-agent-harness/2026-06-12-pass-3-sandboxing-and-permissions.md";
+const SANDBOX_DOC_PATH = "router/evals/fixtures/output-router-corpus/sandbox-permissions.md";
 const GENERATED_GRAPH_PATH = "graphify-out/graph.html";
-const SANDBOX_QUESTION = "Find the Sandbox Permissions section, report file/lines, and explain UseDefault, RequireEscalated, and WithAdditionalPermissions.";
-const SANDBOX_QUERY = "Sandbox Permissions SandboxPermissions UseDefault RequireEscalated WithAdditionalPermissions Plain-language meaning";
+const SANDBOX_QUESTION =
+  "Find the Sandbox Permissions section, report file/lines, and explain UseDefault, RequireEscalated, and WithAdditionalPermissions.";
+const SANDBOX_QUERY =
+  "Sandbox Permissions SandboxPermissions UseDefault RequireEscalated WithAdditionalPermissions Plain-language meaning";
 const SANDBOX_REQUIRED_EVIDENCE = [
-    "SandboxPermissions is a per-command request shape",
-    "UseDefault",
-    "Run with the turn's normal sandbox",
-    "RequireEscalated",
-    "Request unsandboxed execution",
-    "WithAdditionalPermissions",
-    "Stay sandboxed but widen permissions for this one command",
+  "SandboxPermissions is a per-command request shape",
+  "UseDefault",
+  "Run with the turn's normal sandbox",
+  "RequireEscalated",
+  "Request unsandboxed execution",
+  "WithAdditionalPermissions",
+  "Stay sandboxed but widen permissions for this one command",
 ];
 const SANDBOX_REQUIRED_ANSWER = [
-    "UseDefault: run with the turn's normal sandbox",
-    "RequireEscalated: request unsandboxed execution",
-    "WithAdditionalPermissions: stay sandboxed but widen permissions for this one command",
+  "UseDefault: run with the turn's normal sandbox",
+  "RequireEscalated: request unsandboxed execution",
+  "WithAdditionalPermissions: stay sandboxed but widen permissions for this one command",
 ];
 export async function runCodexQaBenchmarks(options = {}) {
-    const iterations = normalizeIterations(options.iterations, 1);
-    const fixture = await createSandboxPermissionsFixture();
-    try {
-        const results = [];
-        const modeResults = [];
-        for (const mode of ["native-broad-search-proxy", "improved-freeflow-router"]) {
-            modeResults.push(await runModeIterations(fixture, mode, iterations));
-        }
-        results.push({
-            id: fixture.id,
-            title: fixture.title,
-            question: fixture.question,
-            expected: fixture.expected,
-            results: modeResults,
-        });
-        return {
-            generatedAt: options.generatedAt ?? new Date().toISOString(),
-            iterations,
-            summary: summarizeCodexQaReport(results),
-            fixtures: results,
-            skippedExternalTools: skippedExternalTools(),
-        };
-    }
-    finally {
-        await fixture.cleanup();
+  const iterations = normalizeIterations(options.iterations, 1);
+  const fixture = await createSandboxPermissionsFixture();
+  try {
+    const results = [];
+    const modeResults = [];
+    for (const mode of ["native-broad-search-proxy", "improved-freeflow-router"]) {
+      modeResults.push(await runModeIterations(fixture, mode, iterations));
     }
+    results.push({
+      id: fixture.id,
+      title: fixture.title,
+      question: fixture.question,
+      expected: fixture.expected,
+      results: modeResults,
+    });
+    return {
+      generatedAt: options.generatedAt ?? new Date().toISOString(),
+      iterations,
+      summary: summarizeCodexQaReport(results),
+      fixtures: results,
+      skippedExternalTools: skippedExternalTools(),
+    };
+  } finally {
+    await fixture.cleanup();
+  }
 }
 export async function writeCodexQaBenchmarkReports(report, markdownReportPath, options = {}) {
-    return writeBenchmarkReportPair({
-        report,
-        markdownReportPath,
-        jsonReportPath: options.jsonReportPath ?? defaultJsonRunReportPath(markdownReportPath),
-        renderMarkdown: renderCodexQaBenchmarkReport,
-    });
+  return writeBenchmarkReportPair({
+    report,
+    markdownReportPath,
+    jsonReportPath: options.jsonReportPath ?? defaultJsonRunReportPath(markdownReportPath),
+    renderMarkdown: renderCodexQaBenchmarkReport,
+  });
 }
 export function renderCodexQaBenchmarkReport(report) {
-    const lines = [
-        "# Codex Structured Q&A Macro Benchmark Report - Iteration 1",
-        "",
-        `Date: ${report.generatedAt.slice(0, 10)}`,
-        "",
-        "## Scope",
-        "",
-        "Fast, deterministic Stage 1 macro benchmark for the Codex CLI agent-harness research corpus. The benchmark asks one structured Q&A question derived from the Sandbox Permissions pass and grades retrieval-backed answer quality, citation quality, evidence quality, context size, and latency.",
-        "",
-        "The fixture uses existing Freeflow research docs as oracle scaffolding and includes a generated `graphify-out` decoy to preserve the original broad-retrieval failure shape. Upstream Codex source citation comparison is marked skipped until a source snapshot is explicitly supplied.",
-        "",
-        "## Command",
-        "",
-        "```sh",
-        "npm run bench:router:codex-qa",
-        "```",
-        "",
-        "The CLI writes machine-readable JSON under `evals/runs/output-router/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
-        "",
-        "## Summary",
-        "",
-        `- Iterations per mode: ${report.iterations}`,
-        `- Fixtures: ${report.summary.fixtures}`,
-        `- Improved Freeflow Router gated pass: ${report.summary.improved.passed}/${report.summary.fixtures}`,
-        `- Native broad-search proxy pass: ${report.summary.nativeBaseline.passed}/${report.summary.fixtures}`,
-        `- Sandbox failure fixed: ${report.summary.sandboxFailureFixed ? "yes" : "no"}`,
-        `- Generated false positives observed: ${report.summary.generatedFalsePositiveCount}/${report.summary.modeResults} mode results`,
-        `- Improved answer/citation/evidence: ${report.summary.improved.answerCorrect}/${report.summary.fixtures} answer, ${report.summary.improved.citationCorrect}/${report.summary.fixtures} citation, ${report.summary.improved.evidenceCorrect}/${report.summary.fixtures} evidence`,
-        `- Improved weighted byte/token reduction: ${formatPercent(report.summary.improved.weightedByteReductionPercent)} / ${formatPercent(report.summary.improved.weightedTokenReductionPercent)} (${report.summary.improved.totalRawBytes}/${report.summary.improved.totalRawTokensApprox} raw to ${report.summary.improved.totalContextBytes}/${report.summary.improved.totalContextTokensApprox} context)`,
-        "",
-        "## Results",
-        "",
-        "| fixture | mode | correctness | checks | path | lines | raw bytes/tokens | context bytes/tokens | byte/token reduction | latency p50/p95 ms | proxy calls | answer | notes |",
-        "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- |",
-    ];
-    for (const fixture of report.fixtures) {
-        for (const result of fixture.results) {
-            lines.push([
-                fixture.id,
-                result.mode,
-                result.correctness.passed ? "pass" : "fail",
-                `answer ${mark(result.correctness.answerCorrect)} citation ${mark(result.correctness.citationCorrect)} evidence ${mark(result.correctness.evidenceCorrect)} gen-fp ${mark(!result.correctness.generatedFalsePositive)}`,
-                result.actualPath ?? "-",
-                result.actualLines ?? "-",
-                `${result.rawBytes}/${result.rawTokensApprox}`,
-                `${result.contextBytes}/${result.contextTokensApprox}`,
-                `${formatPercent(result.byteReductionPercent)}/${formatPercent(result.tokenReductionPercent)}`,
-                `${result.latencyMs.p50.toFixed(2)}/${result.latencyMs.p95.toFixed(2)}`,
-                String(result.proxyCalls),
-                result.answer,
-                result.notes.join("; "),
-            ].map((cell) => escapeTable(String(cell))).join(" | ").replace(/^/, "|").replace(/$/, " |"));
-        }
-    }
-    lines.push("", "## Fixture Questions", "");
-    for (const fixture of report.fixtures) {
-        lines.push(`- ${fixture.id}: ${fixture.question}`);
-        lines.push(`  - expected doc: \`${fixture.expected.path}\``);
-        lines.push(`  - upstream source citation comparison: ${fixture.expected.sourceCitationStatus}`);
-    }
-    lines.push("", "## Skipped External Comparators", "");
-    for (const skipped of report.skippedExternalTools) {
-        lines.push(`- ${skipped.name}: ${skipped.reason}`);
+  const lines = [
+    "# Codex Structured Q&A Macro Benchmark Report - Iteration 1",
+    "",
+    `Date: ${report.generatedAt.slice(0, 10)}`,
+    "",
+    "## Scope",
+    "",
+    "Fast, deterministic Stage 1 macro benchmark for the neutral Output Router research corpus. The benchmark asks one structured Q&A question from the Sandbox Permissions fixture and grades retrieval-backed answer quality, citation quality, evidence quality, context size, and latency.",
+    "",
+    "The fixture uses existing Freeflow research docs as oracle scaffolding and includes a generated `graphify-out` decoy to preserve the original broad-retrieval failure shape. Upstream Codex source citation comparison is marked skipped until a source snapshot is explicitly supplied.",
+    "",
+    "## Command",
+    "",
+    "```sh",
+    "npm run bench:router:codex-qa",
+    "```",
+    "",
+    "The CLI writes machine-readable JSON under `router/evals/runs/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
+    "",
+    "## Summary",
+    "",
+    `- Iterations per mode: ${report.iterations}`,
+    `- Fixtures: ${report.summary.fixtures}`,
+    `- Improved Freeflow Router gated pass: ${report.summary.improved.passed}/${report.summary.fixtures}`,
+    `- Native broad-search proxy pass: ${report.summary.nativeBaseline.passed}/${report.summary.fixtures}`,
+    `- Sandbox failure fixed: ${report.summary.sandboxFailureFixed ? "yes" : "no"}`,
+    `- Generated false positives observed: ${report.summary.generatedFalsePositiveCount}/${report.summary.modeResults} mode results`,
+    `- Improved answer/citation/evidence: ${report.summary.improved.answerCorrect}/${report.summary.fixtures} answer, ${report.summary.improved.citationCorrect}/${report.summary.fixtures} citation, ${report.summary.improved.evidenceCorrect}/${report.summary.fixtures} evidence`,
+    `- Improved weighted byte/token reduction: ${formatPercent(report.summary.improved.weightedByteReductionPercent)} / ${formatPercent(report.summary.improved.weightedTokenReductionPercent)} (${report.summary.improved.totalRawBytes}/${report.summary.improved.totalRawTokensApprox} raw to ${report.summary.improved.totalContextBytes}/${report.summary.improved.totalContextTokensApprox} context)`,
+    "",
+    "## Results",
+    "",
+    "| fixture | mode | correctness | checks | path | lines | raw bytes/tokens | context bytes/tokens | byte/token reduction | latency p50/p95 ms | proxy calls | answer | notes |",
+    "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- |",
+  ];
+  for (const fixture of report.fixtures) {
+    for (const result of fixture.results) {
+      lines.push(
+        [
+          fixture.id,
+          result.mode,
+          result.correctness.passed ? "pass" : "fail",
+          `answer ${mark(result.correctness.answerCorrect)} citation ${mark(result.correctness.citationCorrect)} evidence ${mark(result.correctness.evidenceCorrect)} gen-fp ${mark(!result.correctness.generatedFalsePositive)}`,
+          result.actualPath ?? "-",
+          result.actualLines ?? "-",
+          `${result.rawBytes}/${result.rawTokensApprox}`,
+          `${result.contextBytes}/${result.contextTokensApprox}`,
+          `${formatPercent(result.byteReductionPercent)}/${formatPercent(result.tokenReductionPercent)}`,
+          `${result.latencyMs.p50.toFixed(2)}/${result.latencyMs.p95.toFixed(2)}`,
+          String(result.proxyCalls),
+          result.answer,
+          result.notes.join("; "),
+        ]
+          .map((cell) => escapeTable(String(cell)))
+          .join(" | ")
+          .replace(/^/, "|")
+          .replace(/$/, " |"),
+      );
     }
-    return `${lines.join("\n")}\n`;
+  }
+  lines.push("", "## Fixture Questions", "");
+  for (const fixture of report.fixtures) {
+    lines.push(`- ${fixture.id}: ${fixture.question}`);
+    lines.push(`  - expected doc: \`${fixture.expected.path}\``);
+    lines.push(`  - upstream source citation comparison: ${fixture.expected.sourceCitationStatus}`);
+  }
+  lines.push("", "## Skipped External Comparators", "");
+  for (const skipped of report.skippedExternalTools) {
+    lines.push(`- ${skipped.name}: ${skipped.reason}`);
+  }
+  return `${lines.join("\n")}\n`;
 }
 async function runModeIterations(fixture, mode, iterations) {
-    const latencies = [];
-    let firstObservation;
-    for (let index = 0; index < iterations; index += 1) {
-        const startedAt = performance.now();
-        const observation = mode === "native-broad-search-proxy"
-            ? await nativeBroadSearchObservation(fixture)
-            : await improvedFreeflowObservation(fixture);
-        latencies.push(performance.now() - startedAt);
-        firstObservation ??= observation;
-    }
-    if (!firstObservation) {
-        throw new Error("Codex Q&A benchmark produced no observation");
-    }
-    const correctness = gradeObservation(fixture, firstObservation);
-    return {
-        mode,
-        toolPathUsed: firstObservation.toolPathUsed,
-        skipped: false,
-        proxyCalls: firstObservation.proxyCalls,
-        rawBytes: firstObservation.rawBytes,
-        rawTokensApprox: approximateTokens(firstObservation.rawBytes),
-        contextBytes: firstObservation.contextBytes,
-        contextTokensApprox: approximateTokens(firstObservation.contextBytes),
-        byteReductionPercent: reductionPercent(firstObservation.rawBytes, firstObservation.contextBytes),
-        tokenReductionPercent: reductionPercent(approximateTokens(firstObservation.rawBytes), approximateTokens(firstObservation.contextBytes)),
-        latencyMs: latencySummary(latencies),
-        ...(firstObservation.actualPath !== undefined ? { actualPath: firstObservation.actualPath } : {}),
-        ...(firstObservation.actualLines !== undefined ? { actualLines: firstObservation.actualLines } : {}),
-        answer: firstObservation.answer,
-        evidenceExcerpt: firstObservation.evidenceExcerpt,
-        correctness,
-        notes: firstObservation.notes,
-    };
+  const latencies = [];
+  let firstObservation;
+  for (let index = 0; index < iterations; index += 1) {
+    const startedAt = performance.now();
+    const observation =
+      mode === "native-broad-search-proxy"
+        ? await nativeBroadSearchObservation(fixture)
+        : await improvedFreeflowObservation(fixture);
+    latencies.push(performance.now() - startedAt);
+    firstObservation ??= observation;
+  }
+  if (!firstObservation) {
+    throw new Error("Codex Q&A benchmark produced no observation");
+  }
+  const correctness = gradeObservation(fixture, firstObservation);
+  return {
+    mode,
+    toolPathUsed: firstObservation.toolPathUsed,
+    skipped: false,
+    proxyCalls: firstObservation.proxyCalls,
+    rawBytes: firstObservation.rawBytes,
+    rawTokensApprox: approximateTokens(firstObservation.rawBytes),
+    contextBytes: firstObservation.contextBytes,
+    contextTokensApprox: approximateTokens(firstObservation.contextBytes),
+    byteReductionPercent: reductionPercent(firstObservation.rawBytes, firstObservation.contextBytes),
+    tokenReductionPercent: reductionPercent(
+      approximateTokens(firstObservation.rawBytes),
+      approximateTokens(firstObservation.contextBytes),
+    ),
+    latencyMs: latencySummary(latencies),
+    ...(firstObservation.actualPath !== undefined ? { actualPath: firstObservation.actualPath } : {}),
+    ...(firstObservation.actualLines !== undefined ? { actualLines: firstObservation.actualLines } : {}),
+    answer: firstObservation.answer,
+    evidenceExcerpt: firstObservation.evidenceExcerpt,
+    correctness,
+    notes: firstObservation.notes,
+  };
 }
 async function improvedFreeflowObservation(fixture) {
-    const queryResult = await freeflowSearch({
-        action: "query",
+  const queryResult = await freeflowSearch({
+    action: "query",
+    source: { kind: "repo", root: fixture.root },
+    query: fixture.query,
+    preserve: "important",
+    topK: 1,
+  });
+  const seedEvidence = queryResult.evidence?.[0];
+  const expandedResult = seedEvidence
+    ? await freeflowSearch({
+        action: "expand",
         source: { kind: "repo", root: fixture.root },
-        query: fixture.query,
+        evidence: seedEvidence,
+        expansion: "lines_30",
         preserve: "important",
-        topK: 1,
-    });
-    const seedEvidence = queryResult.evidence?.[0];
-    const expandedResult = seedEvidence
-        ? await freeflowSearch({
-            action: "expand",
-            source: { kind: "repo", root: fixture.root },
-            evidence: seedEvidence,
-            expansion: "lines_30",
-            preserve: "important",
-        })
-        : undefined;
-    const evidence = expandedResult?.evidence?.[0] ?? seedEvidence;
-    const evidenceExcerpt = evidence?.excerpt ?? "";
-    return {
-        toolPathUsed: "freeflowSearch repo query + expand",
-        proxyCalls: expandedResult ? 2 : 1,
-        rawBytes: await repoRawBytes(fixture.root),
-        contextBytes: byteLength(JSON.stringify(queryResult)) + byteLength(JSON.stringify(expandedResult ?? {})),
-        ...(evidence?.path !== undefined ? { actualPath: evidence.path } : {}),
-        ...(evidence?.lines !== undefined ? { actualLines: evidence.lines } : {}),
-        evidenceExcerpt,
-        answer: answerFromEvidence(evidence, evidenceExcerpt),
-        notes: [queryResult.routing.reason, ...(expandedResult ? [expandedResult.routing.reason] : ["expand skipped: no seed evidence"])],
-    };
+      })
+    : undefined;
+  const evidence = expandedResult?.evidence?.[0] ?? seedEvidence;
+  const evidenceExcerpt = evidence?.excerpt ?? "";
+  return {
+    toolPathUsed: "freeflowSearch repo query + expand",
+    proxyCalls: expandedResult ? 2 : 1,
+    rawBytes: await repoRawBytes(fixture.root),
+    contextBytes: byteLength(JSON.stringify(queryResult)) + byteLength(JSON.stringify(expandedResult ?? {})),
+    ...(evidence?.path !== undefined ? { actualPath: evidence.path } : {}),
+    ...(evidence?.lines !== undefined ? { actualLines: evidence.lines } : {}),
+    evidenceExcerpt,
+    answer: answerFromEvidence(evidence, evidenceExcerpt),
+    notes: [
+      queryResult.routing.reason,
+      ...(expandedResult ? [expandedResult.routing.reason] : ["expand skipped: no seed evidence"]),
+    ],
+  };
 }
 async function nativeBroadSearchObservation(fixture) {
-    const files = await readRepoTextFiles(fixture.root);
-    const tokens = tokenize(fixture.query);
-    const scored = files
-        .map((file) => ({ ...file, score: scoreText(file.text, tokens) }))
-        .filter((file) => file.score > 0)
-        .sort((a, b) => b.score - a.score || b.bytes - a.bytes || a.path.localeCompare(b.path));
-    const selected = scored[0];
-    const evidenceExcerpt = selected?.text ?? "";
-    return {
-        toolPathUsed: "native broad text-search proxy including generated artifacts",
-        proxyCalls: 1,
-        rawBytes: await repoRawBytes(fixture.root),
-        contextBytes: byteLength(evidenceExcerpt),
-        ...(selected?.path !== undefined ? { actualPath: selected.path } : {}),
-        ...(selected ? { actualLines: `1-${selected.lineCount}` } : {}),
-        evidenceExcerpt,
-        answer: answerFromEvidence(selected ? { path: selected.path, lines: `1-${selected.lineCount}` } : undefined, evidenceExcerpt),
-        notes: selected ? [`lexical score=${selected.score}`] : ["no native match"],
-    };
+  const files = await readRepoTextFiles(fixture.root);
+  const tokens = tokenize(fixture.query);
+  const scored = files
+    .map((file) => ({ ...file, score: scoreText(file.text, tokens) }))
+    .filter((file) => file.score > 0)
+    .sort((a, b) => b.score - a.score || b.bytes - a.bytes || a.path.localeCompare(b.path));
+  const selected = scored[0];
+  const evidenceExcerpt = selected?.text ?? "";
+  return {
+    toolPathUsed: "native broad text-search proxy including generated artifacts",
+    proxyCalls: 1,
+    rawBytes: await repoRawBytes(fixture.root),
+    contextBytes: byteLength(evidenceExcerpt),
+    ...(selected?.path !== undefined ? { actualPath: selected.path } : {}),
+    ...(selected ? { actualLines: `1-${selected.lineCount}` } : {}),
+    evidenceExcerpt,
+    answer: answerFromEvidence(
+      selected ? { path: selected.path, lines: `1-${selected.lineCount}` } : undefined,
+      evidenceExcerpt,
+    ),
+    notes: selected ? [`lexical score=${selected.score}`] : ["no native match"],
+  };
 }
 function gradeObservation(fixture, observation) {
-    const answerCorrect = fixture.expected.requiredAnswer.every((phrase) => includesNormalized(observation.answer, phrase));
-    const citationCorrect = observation.actualPath === fixture.expected.path && typeof observation.actualLines === "string";
-    const evidenceCorrect = fixture.expected.requiredEvidence.every((phrase) => includesNormalized(observation.evidenceExcerpt, phrase));
-    const generatedFalsePositive = observation.actualPath?.startsWith("graphify-out/") ?? false;
-    return {
-        passed: answerCorrect && citationCorrect && evidenceCorrect && !generatedFalsePositive,
-        answerCorrect,
-        citationCorrect,
-        evidenceCorrect,
-        generatedFalsePositive,
-    };
+  const answerCorrect = fixture.expected.requiredAnswer.every((phrase) =>
+    includesNormalized(observation.answer, phrase),
+  );
+  const citationCorrect =
+    observation.actualPath === fixture.expected.path && typeof observation.actualLines === "string";
+  const evidenceCorrect = fixture.expected.requiredEvidence.every((phrase) =>
+    includesNormalized(observation.evidenceExcerpt, phrase),
+  );
+  const generatedFalsePositive = observation.actualPath?.startsWith("graphify-out/") ?? false;
+  return {
+    passed: answerCorrect && citationCorrect && evidenceCorrect && !generatedFalsePositive,
+    answerCorrect,
+    citationCorrect,
+    evidenceCorrect,
+    generatedFalsePositive,
+  };
 }
 function summarizeCodexQaReport(fixtures) {
-    const improved = summarizeMode(fixtures, "improved-freeflow-router");
-    const nativeBaseline = summarizeMode(fixtures, "native-broad-search-proxy");
-    const modeResults = fixtures.reduce((count, fixture) => count + fixture.results.length, 0);
-    const generatedFalsePositiveCount = fixtures
-        .flatMap((fixture) => fixture.results)
-        .filter((result) => result.correctness.generatedFalsePositive).length;
-    return {
-        fixtures: fixtures.length,
-        modeResults,
-        improved,
-        nativeBaseline,
-        sandboxFailureFixed: improved.failed === 0 && nativeBaseline.generatedFalsePositiveCount > 0,
-        generatedFalsePositiveCount,
-    };
+  const improved = summarizeMode(fixtures, "improved-freeflow-router");
+  const nativeBaseline = summarizeMode(fixtures, "native-broad-search-proxy");
+  const modeResults = fixtures.reduce((count, fixture) => count + fixture.results.length, 0);
+  const generatedFalsePositiveCount = fixtures
+    .flatMap((fixture) => fixture.results)
+    .filter((result) => result.correctness.generatedFalsePositive).length;
+  return {
+    fixtures: fixtures.length,
+    modeResults,
+    improved,
+    nativeBaseline,
+    sandboxFailureFixed: improved.failed === 0 && nativeBaseline.generatedFalsePositiveCount > 0,
+    generatedFalsePositiveCount,
+  };
 }
 function summarizeMode(fixtures, mode) {
-    const results = fixtures
-        .map((fixture) => fixture.results.find((result) => result.mode === mode))
-        .filter((result) => Boolean(result));
-    const totalRawBytes = results.reduce((sum, result) => sum + result.rawBytes, 0);
-    const totalContextBytes = results.reduce((sum, result) => sum + result.contextBytes, 0);
-    const totalRawTokensApprox = results.reduce((sum, result) => sum + result.rawTokensApprox, 0);
-    const totalContextTokensApprox = results.reduce((sum, result) => sum + result.contextTokensApprox, 0);
-    return results.reduce((summary, result) => {
-        if (result.skipped) {
-            summary.skipped += 1;
-        }
-        else if (result.correctness.passed) {
-            summary.passed += 1;
-        }
-        else {
-            summary.failed += 1;
-        }
-        if (result.correctness.answerCorrect) {
-            summary.answerCorrect += 1;
-        }
-        if (result.correctness.citationCorrect) {
-            summary.citationCorrect += 1;
-        }
-        if (result.correctness.evidenceCorrect) {
-            summary.evidenceCorrect += 1;
-        }
-        if (result.correctness.generatedFalsePositive) {
-            summary.generatedFalsePositiveCount += 1;
-        }
-        return summary;
-    }, {
-        passed: 0,
-        failed: 0,
-        skipped: 0,
-        answerCorrect: 0,
-        citationCorrect: 0,
-        evidenceCorrect: 0,
-        generatedFalsePositiveCount: 0,
-        totalRawBytes,
-        totalContextBytes,
-        totalRawTokensApprox,
-        totalContextTokensApprox,
-        weightedByteReductionPercent: reductionPercent(totalRawBytes, totalContextBytes),
-        weightedTokenReductionPercent: reductionPercent(totalRawTokensApprox, totalContextTokensApprox),
-    });
+  const results = fixtures
+    .map((fixture) => fixture.results.find((result) => result.mode === mode))
+    .filter((result) => Boolean(result));
+  const totalRawBytes = results.reduce((sum, result) => sum + result.rawBytes, 0);
+  const totalContextBytes = results.reduce((sum, result) => sum + result.contextBytes, 0);
+  const totalRawTokensApprox = results.reduce((sum, result) => sum + result.rawTokensApprox, 0);
+  const totalContextTokensApprox = results.reduce((sum, result) => sum + result.contextTokensApprox, 0);
+  return results.reduce(
+    (summary, result) => {
+      if (result.skipped) {
+        summary.skipped += 1;
+      } else if (result.correctness.passed) {
+        summary.passed += 1;
+      } else {
+        summary.failed += 1;
+      }
+      if (result.correctness.answerCorrect) {
+        summary.answerCorrect += 1;
+      }
+      if (result.correctness.citationCorrect) {
+        summary.citationCorrect += 1;
+      }
+      if (result.correctness.evidenceCorrect) {
+        summary.evidenceCorrect += 1;
+      }
+      if (result.correctness.generatedFalsePositive) {
+        summary.generatedFalsePositiveCount += 1;
+      }
+      return summary;
+    },
+    {
+      passed: 0,
+      failed: 0,
+      skipped: 0,
+      answerCorrect: 0,
+      citationCorrect: 0,
+      evidenceCorrect: 0,
+      generatedFalsePositiveCount: 0,
+      totalRawBytes,
+      totalContextBytes,
+      totalRawTokensApprox,
+      totalContextTokensApprox,
+      weightedByteReductionPercent: reductionPercent(totalRawBytes, totalContextBytes),
+      weightedTokenReductionPercent: reductionPercent(totalRawTokensApprox, totalContextTokensApprox),
+    },
+  );
 }
 async function createSandboxPermissionsFixture() {
-    const root = await mkdtemp(join(tmpdir(), "freeflow-codex-qa-fixture-"));
-    await writeTextFile(join(root, SANDBOX_DOC_PATH), sandboxDocumentText());
-    await writeTextFile(join(root, GENERATED_GRAPH_PATH), [
-        "<html><body>",
-        `${"Sandbox Permissions SandboxPermissions UseDefault RequireEscalated WithAdditionalPermissions Plain-language meaning ".repeat(5_000)}GENERATED_GRAPH_DECOY_SENTINEL`,
-        "</body></html>",
-    ].join("\n"));
-    return {
-        id: "sandbox-permissions-structured-qa",
-        title: "Sandbox Permissions structured Q&A",
-        question: SANDBOX_QUESTION,
-        query: SANDBOX_QUERY,
-        expected: {
-            path: SANDBOX_DOC_PATH,
-            requiredEvidence: SANDBOX_REQUIRED_EVIDENCE,
-            requiredAnswer: SANDBOX_REQUIRED_ANSWER,
-            sourceCitationStatus: "skipped-unavailable",
-        },
-        root,
-        cleanup: () => rm(root, { recursive: true, force: true }),
-    };
+  const root = await mkdtemp(join(tmpdir(), "freeflow-codex-qa-fixture-"));
+  await writeTextFile(join(root, SANDBOX_DOC_PATH), sandboxDocumentText());
+  await writeTextFile(
+    join(root, GENERATED_GRAPH_PATH),
+    [
+      "<html><body>",
+      `${"Sandbox Permissions SandboxPermissions UseDefault RequireEscalated WithAdditionalPermissions Plain-language meaning ".repeat(5_000)}GENERATED_GRAPH_DECOY_SENTINEL`,
+      "</body></html>",
+    ].join("\n"),
+  );
+  return {
+    id: "sandbox-permissions-structured-qa",
+    title: "Sandbox Permissions structured Q&A",
+    question: SANDBOX_QUESTION,
+    query: SANDBOX_QUERY,
+    expected: {
+      path: SANDBOX_DOC_PATH,
+      requiredEvidence: SANDBOX_REQUIRED_EVIDENCE,
+      requiredAnswer: SANDBOX_REQUIRED_ANSWER,
+      sourceCitationStatus: "skipped-unavailable",
+    },
+    root,
+    cleanup: () => rm(root, { recursive: true, force: true }),
+  };
 }
 function sandboxDocumentText() {
-    return [
-        "# Pass 3",
-        "",
-        "### Sandbox Permissions",
-        "",
-        "SandboxPermissions is a per-command request shape.",
-        "",
-        "Codex defines:",
-        "",
-        "```text",
-        "UseDefault",
-        "RequireEscalated",
-        "WithAdditionalPermissions",
-        "```",
-        "",
-        "Plain-language meaning:",
-        "",
-        "```text",
-        "UseDefault:",
-        "  Run with the turn's normal sandbox.",
-        "",
-        "RequireEscalated:",
-        "  Request unsandboxed execution.",
-        "",
-        "WithAdditionalPermissions:",
-        "  Stay sandboxed but widen permissions for this one command.",
-        "```",
-        "",
-        "This is a strong pattern for local agents.",
-    ].join("\n");
+  return [
+    "# Pass 3",
+    "",
+    "### Sandbox Permissions",
+    "",
+    "SandboxPermissions is a per-command request shape.",
+    "",
+    "Codex defines:",
+    "",
+    "```text",
+    "UseDefault",
+    "RequireEscalated",
+    "WithAdditionalPermissions",
+    "```",
+    "",
+    "Plain-language meaning:",
+    "",
+    "```text",
+    "UseDefault:",
+    "  Run with the turn's normal sandbox.",
+    "",
+    "RequireEscalated:",
+    "  Request unsandboxed execution.",
+    "",
+    "WithAdditionalPermissions:",
+    "  Stay sandboxed but widen permissions for this one command.",
+    "```",
+    "",
+    "This is a strong pattern for local agents.",
+  ].join("\n");
 }
 function answerFromEvidence(evidence, text) {
-    const citation = evidence?.path ? `${evidence.path}${evidence.lines ? `:${evidence.lines}` : ""}` : "unknown source";
-    const facts = [];
-    if (includesNormalized(text, "Run with the turn's normal sandbox")) {
-        facts.push("UseDefault: run with the turn's normal sandbox");
-    }
-    if (includesNormalized(text, "Request unsandboxed execution")) {
-        facts.push("RequireEscalated: request unsandboxed execution");
-    }
-    if (includesNormalized(text, "Stay sandboxed but widen permissions for this one command")) {
-        facts.push("WithAdditionalPermissions: stay sandboxed but widen permissions for this one command");
-    }
-    if (facts.length === 0) {
-        return `Sandbox Permissions evidence from ${citation} did not include enough plain-language definitions to answer.`;
-    }
-    return `Sandbox Permissions are documented at ${citation}. ${facts.join(". ")}.`;
+  const citation = evidence?.path ? `${evidence.path}${evidence.lines ? `:${evidence.lines}` : ""}` : "unknown source";
+  const facts = [];
+  if (includesNormalized(text, "Run with the turn's normal sandbox")) {
+    facts.push("UseDefault: run with the turn's normal sandbox");
+  }
+  if (includesNormalized(text, "Request unsandboxed execution")) {
+    facts.push("RequireEscalated: request unsandboxed execution");
+  }
+  if (includesNormalized(text, "Stay sandboxed but widen permissions for this one command")) {
+    facts.push("WithAdditionalPermissions: stay sandboxed but widen permissions for this one command");
+  }
+  if (facts.length === 0) {
+    return `Sandbox Permissions evidence from ${citation} did not include enough plain-language definitions to answer.`;
+  }
+  return `Sandbox Permissions are documented at ${citation}. ${facts.join(". ")}.`;
 }
 async function readRepoTextFiles(root) {
-    const files = [];
-    await walk(root, async (absolutePath) => {
-        const text = await readFile(absolutePath, "utf8");
-        files.push({
-            path: relative(root, absolutePath),
-            text,
-            bytes: byteLength(text),
-            lineCount: text.length === 0 ? 1 : text.split("\n").length,
-        });
+  const files = [];
+  await walk(root, async (absolutePath) => {
+    const text = await readFile(absolutePath, "utf8");
+    files.push({
+      path: relative(root, absolutePath),
+      text,
+      bytes: byteLength(text),
+      lineCount: text.length === 0 ? 1 : text.split("\n").length,
     });
-    return files;
+  });
+  return files;
 }
 async function repoRawBytes(root) {
-    let total = 0;
-    await walk(root, async (absolutePath) => {
-        total += (await stat(absolutePath)).size;
-    });
-    return total;
+  let total = 0;
+  await walk(root, async (absolutePath) => {
+    total += (await stat(absolutePath)).size;
+  });
+  return total;
 }
 async function walk(root, visit) {
-    const entries = await readdir(root, { withFileTypes: true });
-    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
-        const absolutePath = join(root, entry.name);
-        if (entry.isDirectory()) {
-            await walk(absolutePath, visit);
-        }
-        else if (entry.isFile()) {
-            await visit(absolutePath);
-        }
+  const entries = await readdir(root, { withFileTypes: true });
+  for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+    const absolutePath = join(root, entry.name);
+    if (entry.isDirectory()) {
+      await walk(absolutePath, visit);
+    } else if (entry.isFile()) {
+      await visit(absolutePath);
     }
+  }
 }
 async function writeTextFile(path, text) {
-    await mkdir(dirname(path), { recursive: true });
-    await writeFile(path, text, "utf8");
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, text, "utf8");
 }
 function scoreText(text, queryTokens) {
-    const counts = tokenCounts(text);
-    return queryTokens.reduce((score, token) => score + (counts.get(token) ?? 0), 0);
+  const counts = tokenCounts(text);
+  return queryTokens.reduce((score, token) => score + (counts.get(token) ?? 0), 0);
 }
 function tokenCounts(text) {
-    const counts = new Map();
-    for (const token of tokenize(text)) {
-        counts.set(token, (counts.get(token) ?? 0) + 1);
-    }
-    return counts;
+  const counts = new Map();
+  for (const token of tokenize(text)) {
+    counts.set(token, (counts.get(token) ?? 0) + 1);
+  }
+  return counts;
 }
 function tokenize(text) {
-    return text.toLowerCase().match(/[a-z0-9_./-]+/g) ?? [];
+  return text.toLowerCase().match(/[a-z0-9_./-]+/g) ?? [];
 }
 function includesNormalized(text, phrase) {
-    return normalizeText(text).includes(normalizeText(phrase));
+  return normalizeText(text).includes(normalizeText(phrase));
 }
 function normalizeText(text) {
-    return text.toLowerCase().replace(/[`*_]/g, "").replace(/\s+/g, " ").trim();
+  return text.toLowerCase().replace(/[`*_]/g, "").replace(/\s+/g, " ").trim();
 }
 function mark(value) {
-    return value ? "✓" : "✗";
+  return value ? "✓" : "✗";
 }
 function byteLength(text) {
-    return Buffer.byteLength(text, "utf8");
+  return Buffer.byteLength(text, "utf8");
 }
 function skippedExternalTools() {
-    return [
-        {
-            name: "Graphify",
-            reason: "Skipped unless a fresh graph for the benchmark fixture is explicitly supplied.",
-        },
-        {
-            name: "Claude Context",
-            reason: "Skipped unless an index is configured and fresh for the benchmark fixture.",
-        },
-    ];
+  return [
+    {
+      name: "Graphify",
+      reason: "Skipped unless a fresh graph for the benchmark fixture is explicitly supplied.",
+    },
+    {
+      name: "Claude Context",
+      reason: "Skipped unless an index is configured and fresh for the benchmark fixture.",
+    },
+  ];
 }
 function defaultReportPath() {
-    return resolve(process.cwd(), "evals/reports/runtime/output-router-codex-qa-benchmark-1-report.md");
+  return resolve(process.cwd(), "router/evals/reports/output-router-codex-qa-benchmark-1-report.md");
 }
 async function main() {
-    const { iterations, reportPath, jsonReportPath } = parseBenchmarkCliArgs(process.argv.slice(2), { reportPath: defaultReportPath() });
-    const options = {};
-    if (iterations !== undefined) {
-        options.iterations = iterations;
-    }
-    const report = await runCodexQaBenchmarks(options);
-    const reports = await writeCodexQaBenchmarkReports(report, reportPath, { jsonReportPath });
-    console.log(`Freeflow Codex Q&A benchmark: improved ${report.summary.improved.passed}/${report.summary.fixtures} pass; native ${report.summary.nativeBaseline.passed}/${report.summary.fixtures} pass`);
-    console.log(`Markdown report: ${reports.markdown}`);
-    if (reports.json) {
-        console.log(`JSON run data: ${reports.json}`);
-    }
+  const { iterations, reportPath, jsonReportPath } = parseBenchmarkCliArgs(process.argv.slice(2), {
+    reportPath: defaultReportPath(),
+  });
+  const options = {};
+  if (iterations !== undefined) {
+    options.iterations = iterations;
+  }
+  const report = await runCodexQaBenchmarks(options);
+  const reports = await writeCodexQaBenchmarkReports(report, reportPath, { jsonReportPath });
+  console.log(
+    `Freeflow Codex Q&A benchmark: improved ${report.summary.improved.passed}/${report.summary.fixtures} pass; native ${report.summary.nativeBaseline.passed}/${report.summary.fixtures} pass`,
+  );
+  console.log(`Markdown report: ${reports.markdown}`);
+  if (reports.json) {
+    console.log(`JSON run data: ${reports.json}`);
+  }
 }
 if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
-    main().catch((error) => {
-        console.error(error);
-        process.exitCode = 1;
-    });
+  main().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+  });
 }