@hassangameryt/freeflow 0.5.0 → 0.7.0

package/router/dist/benchmarks/command-benchmarks.js

@@ -1,698 +0,0 @@
-import { createHash } from "node:crypto";
-import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { dirname, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { performance } from "node:perf_hooks";
-import {
-  approximateTokens,
-  averagePercent as average,
-  defaultJsonRunReportPath,
-  escapeMarkdownTableCell as escapeTable,
-  formatPercent,
-  latencySummary,
-  medianPercent as median,
-  normalizeIterations,
-  parseBenchmarkCliArgs,
-  reductionPercent,
-  writeBenchmarkReportPair,
-} from "./benchmark-harness.js";
-import { freeflowRun } from "../tools/run.js";
-import { createVault, readOutputText } from "../vault/vault.js";
-const DEFAULT_ITERATIONS = 3;
-const DEFAULT_SKIPPED_EXTERNAL_TOOLS = [
-  {
-    name: "RTK",
-    reason: "Optional command compressor comparator is skipped unless a caller supplies a configured comparator hook.",
-  },
-  {
-    name: "Squeez",
-    reason:
-      "Optional session/output compressor comparator is skipped unless a caller supplies a configured comparator hook.",
-  },
-];
-export async function runCommandBenchmarks(options = {}) {
-  const iterations = normalizeIterations(options.iterations, DEFAULT_ITERATIONS);
-  const vaultRoot = await createTempDir("freeflow-router-command-benchmark-vault-");
-  const externalComparators = options.externalComparators ?? [];
-  const fixtures = createCommandBenchmarkFixtures(vaultRoot.path, externalComparators);
-  try {
-    const fixtureResults = [];
-    for (const fixture of fixtures) {
-      const results = [];
-      for (const mode of commandBenchmarkModes(externalComparators)) {
-        results.push(await runCommandFixtureMode(fixture, mode, iterations));
-      }
-      fixtureResults.push({
-        id: fixture.id,
-        title: fixture.title,
-        kind: fixture.kind,
-        expected: fixture.expected,
-        results,
-      });
-    }
-    return {
-      generatedAt: options.generatedAt ?? new Date().toISOString(),
-      iterations,
-      summary: summarizeCommandReport(fixtureResults),
-      fixtures: fixtureResults,
-      skippedExternalTools: skippedExternalTools(externalComparators),
-    };
-  } finally {
-    await vaultRoot.cleanup();
-  }
-}
-export function renderCommandBenchmarkReport(report) {
-  const date = report.generatedAt.slice(0, 10);
-  const lines = [
-    "# Output Router Command Benchmark Report - Iteration 1",
-    "",
-    `Date: ${date}`,
-    "",
-    "## Scope",
-    "",
-    "Deterministic, CI-friendly command-output benchmark for `freeflow_run`. The runner compares native direct output against the improved Freeflow command router and records optional RTK/Squeez comparators as skipped unless a caller supplies configured comparator hooks.",
-    "",
-    "Reduction percentages compare routed/context bytes and approximate tokens against raw command output bytes for each fixture. Correctness is gated on execution status, exit code, exact key fact preservation, and Freeflow raw vault recovery.",
-    "",
-    "## Command",
-    "",
-    "```sh",
-    "npm run bench:router:commands",
-    "```",
-    "",
-    "The CLI writes machine-readable JSON under `router/evals/runs/` by default. That JSON is generated run data; this Markdown file is the durable runtime report.",
-    "",
-    "## Summary",
-    "",
-    `- Iterations per mode: ${report.iterations}`,
-    `- Fixtures: ${report.summary.fixtures}`,
-    `- Improved Freeflow run gated pass: ${report.summary.improved.passed}/${report.summary.fixtures}`,
-    `- Native baseline proxy pass: ${report.summary.nativeBaseline.passed}/${report.summary.fixtures}`,
-    `- Improved exact fact preservation: ${report.summary.improved.exactFactsPreserved}/${report.summary.fixtures}`,
-    `- Improved recovery pass: ${report.summary.improved.recoveryPassed}/${report.summary.fixtures}`,
-    `- Improved weighted byte/token reduction: ${formatPercent(report.summary.improved.weightedByteReductionPercent)} / ${formatPercent(report.summary.improved.weightedTokenReductionPercent)} (${report.summary.improved.totalRawBytes}/${report.summary.improved.totalRawTokensApprox} raw to ${report.summary.improved.totalRoutedBytes}/${report.summary.improved.totalRoutedTokensApprox} routed)`,
-    `- Improved average byte/token reduction: ${formatPercent(report.summary.improved.averageByteReductionPercent)} / ${formatPercent(report.summary.improved.averageTokenReductionPercent)}`,
-    `- Improved median byte/token reduction: ${formatPercent(report.summary.improved.medianByteReductionPercent)} / ${formatPercent(report.summary.improved.medianTokenReductionPercent)}`,
-    `- Failed command facts preserved: ${report.summary.failedCommandFactsPreserved ? "yes" : "no"}`,
-    "",
-    "## Command Output Fixtures",
-    "",
-    "| fixture | mode | correctness | checks | status/code | parser | raw bytes/tokens | routed bytes/tokens | byte/token reduction | latency p50/p95 ms | recovery | outputId | notes |",
-    "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- |",
-  ];
-  for (const fixture of report.fixtures) {
-    for (const result of fixture.results) {
-      const correctness = result.skipped ? "skipped" : result.correctness.passed ? "pass" : "fail";
-      const checks = formatCommandCorrectnessChecks(result.correctness);
-      const status = result.executionStatus ? `${result.executionStatus}/${result.exitCode ?? "null"}` : "-";
-      const parser = result.parser
-        ? `${result.parser.name} ${result.parser.confidence.toFixed(2)} ${result.parser.fidelity}`
-        : "-";
-      const notes = result.notes.length ? result.notes.join("; ") : (result.skipReason ?? "");
-      lines.push(
-        `| ${escapeTable(fixture.id)} | ${escapeTable(result.mode)} | ${correctness} | ${escapeTable(checks)} | ${escapeTable(status)} | ${escapeTable(parser)} | ${result.rawBytes}/${result.rawTokensApprox} | ${result.routedBytes}/${result.routedTokensApprox} | ${formatPercent(result.byteReductionPercent)}/${formatPercent(result.tokenReductionPercent)} | ${result.latencyMs.p50.toFixed(2)}/${result.latencyMs.p95.toFixed(2)} | ${escapeTable(result.recovery.status)} | ${escapeTable(result.outputId ?? "-")} | ${escapeTable(notes)} |`,
-      );
-    }
-  }
-  lines.push("", "## Skipped Optional Command Compressors", "");
-  if (report.skippedExternalTools.length === 0) {
-    lines.push("- None. All supplied optional comparator hooks ran.");
-  } else {
-    for (const tool of report.skippedExternalTools) {
-      lines.push(`- ${tool.name}: ${tool.reason}`);
-    }
-  }
-  lines.push(
-    "",
-    "## Regression Status",
-    "",
-    report.summary.improved.failed === 0
-      ? "Improved Freeflow command routing passed all gated command-output benchmark fixtures."
-      : `Improved Freeflow command routing failed ${report.summary.improved.failed} gated command-output benchmark fixture(s).`,
-    "",
-    `Failed command facts preserved: ${report.summary.failedCommandFactsPreserved ? "yes" : "no"}.`,
-    "",
-  );
-  return lines.join("\n");
-}
-export async function writeCommandBenchmarkReport(report, reportPath) {
-  await mkdir(dirname(reportPath), { recursive: true });
-  await writeFile(reportPath, renderCommandBenchmarkReport(report), "utf8");
-}
-export async function writeCommandBenchmarkJsonReport(report, reportPath) {
-  await mkdir(dirname(reportPath), { recursive: true });
-  await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
-}
-export async function writeCommandBenchmarkReports(report, markdownReportPath, options = {}) {
-  return writeBenchmarkReportPair({
-    report,
-    markdownReportPath,
-    jsonReportPath: options.jsonReportPath,
-    renderMarkdown: renderCommandBenchmarkReport,
-  });
-}
-async function runCommandFixtureMode(fixture, mode, iterations) {
-  const latencies = [];
-  const observations = [];
-  const run = fixture.modes[mode];
-  if (!run) {
-    observations.push(
-      skippedObservation(`optional comparator ${mode}`, `No command benchmark mode registered for ${mode}.`),
-    );
-  } else {
-    for (let index = 0; index < iterations; index += 1) {
-      const startedAt = performance.now();
-      observations.push(await run());
-      latencies.push(performance.now() - startedAt);
-    }
-  }
-  const observation = selectCommandBenchmarkObservation(fixture, observations);
-  const correctness = scoreCommandCorrectness(fixture.expected, observation);
-  const rawTokensApprox = approximateTokens(observation.rawBytes);
-  const routedTokensApprox = approximateTokens(observation.routedBytes);
-  const result = {
-    mode,
-    toolPathUsed: observation.toolPathUsed,
-    skipped: observation.skipped ?? false,
-    rawBytes: observation.rawBytes,
-    rawTokensApprox,
-    routedBytes: observation.routedBytes,
-    routedTokensApprox,
-    byteReductionPercent: reductionPercent(observation.rawBytes, observation.routedBytes),
-    tokenReductionPercent: reductionPercent(rawTokensApprox, routedTokensApprox),
-    latencyMs: latencySummary(latencies),
-    routedExcerpt: observation.routedExcerpt,
-    correctness,
-    recovery: observation.recovery,
-    notes: observation.notes ?? [],
-  };
-  if (observation.skipReason) {
-    result.skipReason = observation.skipReason;
-  }
-  if (observation.executionStatus !== undefined) {
-    result.executionStatus = observation.executionStatus;
-  }
-  if (Object.prototype.hasOwnProperty.call(observation, "exitCode")) {
-    result.exitCode = observation.exitCode ?? null;
-  }
-  if (observation.outputId) {
-    result.outputId = observation.outputId;
-  }
-  if (observation.parser) {
-    result.parser = observation.parser;
-  }
-  return result;
-}
-function selectCommandBenchmarkObservation(fixture, observations) {
-  const firstObservation = observations[0];
-  if (firstObservation === undefined) {
-    return skippedObservation("command benchmark runner", "No observation was produced.");
-  }
-  if (fixture.kind === "repeated-output") {
-    return observations[observations.length - 1] ?? firstObservation;
-  }
-  return observations.find((observation) => observation.parser?.name !== "duplicate-output") ?? firstObservation;
-}
-function summarizeCommandReport(fixtures) {
-  const improved = summarizeCommandMode(fixtures, "improved-freeflow-run");
-  const nativeBaseline = summarizeCommandMode(fixtures, "native-baseline-proxy");
-  const failedFixtures = fixtures.filter((fixture) => fixture.expected.executionStatus !== "success");
-  const failedCommandFactsPreserved = failedFixtures.every((fixture) => {
-    const improvedResult = fixture.results.find((result) => result.mode === "improved-freeflow-run");
-    return Boolean(improvedResult?.correctness.failedFactsExact);
-  });
-  return {
-    fixtures: fixtures.length,
-    modeResults: fixtures.reduce((count, fixture) => count + fixture.results.length, 0),
-    improved,
-    nativeBaseline,
-    failedCommandFactsPreserved,
-  };
-}
-function summarizeCommandMode(fixtures, mode) {
-  const modeResults = fixtures
-    .map((fixture) => fixture.results.find((candidate) => candidate.mode === mode))
-    .filter((result) => Boolean(result));
-  const measuredResults = modeResults.filter((result) => !result.skipped);
-  const totalRawBytes = measuredResults.reduce((sum, result) => sum + result.rawBytes, 0);
-  const totalRoutedBytes = measuredResults.reduce((sum, result) => sum + result.routedBytes, 0);
-  const totalRawTokensApprox = measuredResults.reduce((sum, result) => sum + result.rawTokensApprox, 0);
-  const totalRoutedTokensApprox = measuredResults.reduce((sum, result) => sum + result.routedTokensApprox, 0);
-  return modeResults.reduce(
-    (summary, result) => {
-      if (result.skipped) {
-        summary.skipped += 1;
-      } else if (isCommandGatedPass(result, mode)) {
-        summary.passed += 1;
-      } else {
-        summary.failed += 1;
-      }
-      if (result.correctness.exactFactsPreserved) {
-        summary.exactFactsPreserved += 1;
-      }
-      if (result.recovery.status === "passed") {
-        summary.recoveryPassed += 1;
-      }
-      return summary;
-    },
-    {
-      passed: 0,
-      failed: 0,
-      skipped: 0,
-      exactFactsPreserved: 0,
-      recoveryPassed: 0,
-      totalRawBytes,
-      totalRoutedBytes,
-      totalRawTokensApprox,
-      totalRoutedTokensApprox,
-      weightedByteReductionPercent: reductionPercent(totalRawBytes, totalRoutedBytes),
-      weightedTokenReductionPercent: reductionPercent(totalRawTokensApprox, totalRoutedTokensApprox),
-      averageByteReductionPercent: average(measuredResults.map((result) => result.byteReductionPercent)),
-      medianByteReductionPercent: median(measuredResults.map((result) => result.byteReductionPercent)),
-      averageTokenReductionPercent: average(measuredResults.map((result) => result.tokenReductionPercent)),
-      medianTokenReductionPercent: median(measuredResults.map((result) => result.tokenReductionPercent)),
-    },
-  );
-}
-function isCommandGatedPass(result, mode) {
-  if (!result.correctness.passed) {
-    return false;
-  }
-  if (mode !== "improved-freeflow-run") {
-    return true;
-  }
-  return result.recovery.status === "passed";
-}
-function createCommandBenchmarkFixtures(vaultRoot, externalComparators) {
-  return commandFixtureDefinitions().map((definition) => commandFixture(definition, vaultRoot, externalComparators));
-}
-function commandFixtureDefinitions() {
-  const noisySuccess = [
-    "NOISY_SUCCESS completed 1200 files with 0 errors",
-    ...Array.from({ length: 80 }, (_, index) => `cache warm line ${index + 1}: unchanged`),
-  ].join("\n");
-  const failedStack = [
-    "Error: ROUTER_BENCH_FAILED exact stack fact",
-    "    at main (/repo/src/main.ts:7:11)",
-    "    at async run (/repo/src/run.ts:2:3)",
-  ].join("\n");
-  const testSummary = [
-    "FAIL tests/router.test.ts",
-    "  ● preserves command output facts",
-    "    Expected true to be false",
-    "Tests: 1 failed, 3 passed, 4 total",
-  ].join("\n");
-  const diagnostics = [
-    "src/router.ts(12,7): error TS2322: Type 'string' is not assignable to type 'number'.",
-    "src/other.ts(4,1): warning TS6133: 'unused' is declared but its value is never read.",
-  ].join("\n");
-  const gitOutput = [
-    " M router/src/tools/run.ts",
-    "?? router/src/benchmarks/command-benchmarks.ts",
-    " router/src/tools/run.ts | 12 +++++++++---",
-    " 1 file changed, 9 insertions(+), 3 deletions(-)",
-  ].join("\n");
-  const repetitiveLog = [
-    "REPETITIVE_LOG completed after 3 retry cycles",
-    ...Array.from({ length: 120 }, () => "retry heartbeat: waiting for worker"),
-  ].join("\n");
-  const hugeJson = `${JSON.stringify({ status: "ok", marker: "HUGE_JSON_TABLE preserved header fact", rows: 20_000 }).slice(0, -1)},"payload":"${"x".repeat(60_000)}"}`;
-  const repeatedOutput = [
-    "REPEATED_OUTPUT identical command output stays recoverable",
-    ...Array.from({ length: 120 }, (_, index) => `repeated payload line ${index + 1}: unchanged`),
-  ].join("\n");
-  return [
-    {
-      id: "noisy-success",
-      title: "Noisy successful command output",
-      kind: "noisy-success",
-      command: "npm run build",
-      executionStatus: "success",
-      exitCode: 0,
-      stdout: noisySuccess,
-      stderr: "",
-      combined: noisySuccess,
-      requiredFacts: ["NOISY_SUCCESS completed 1200 files with 0 errors"],
-      thresholds: { largeOutputLines: 10, largeOutputBytes: 2_000 },
-    },
-    {
-      id: "failed-stack-trace",
-      title: "Failed command with stack trace",
-      kind: "failed-stack",
-      command: "node fail.js",
-      executionStatus: "failed",
-      exitCode: 1,
-      stdout: "",
-      stderr: failedStack,
-      combined: failedStack,
-      requiredFacts: ["Error: ROUTER_BENCH_FAILED exact stack fact", "at main"],
-      failedFactsMustBeExact: true,
-    },
-    {
-      id: "test-summary",
-      title: "Test runner summary and failure facts",
-      kind: "test-summary",
-      command: "npm test",
-      executionStatus: "failed",
-      exitCode: 1,
-      stdout: testSummary,
-      stderr: "",
-      combined: testSummary,
-      requiredFacts: ["FAIL tests/router.test.ts", "Tests: 1 failed, 3 passed, 4 total"],
-      failedFactsMustBeExact: true,
-      goal: "verification",
-    },
-    {
-      id: "diagnostics",
-      title: "TypeScript/lint diagnostics",
-      kind: "diagnostics",
-      command: "npm run typecheck",
-      executionStatus: "failed",
-      exitCode: 2,
-      stdout: diagnostics,
-      stderr: "",
-      combined: diagnostics,
-      requiredFacts: ["src/router.ts(12,7): error TS2322", "src/other.ts(4,1): warning TS6133"],
-      failedFactsMustBeExact: true,
-    },
-    {
-      id: "git-output",
-      title: "Git status and diffstat output",
-      kind: "git-output",
-      command: "git status --short && git diff --stat",
-      executionStatus: "success",
-      exitCode: 0,
-      stdout: gitOutput,
-      stderr: "",
-      combined: gitOutput,
-      requiredFacts: ["M router/src/tools/run.ts", "1 file changed"],
-      thresholds: { largeOutputLines: 2, largeOutputBytes: 10_000 },
-    },
-    {
-      id: "repetitive-log",
-      title: "Repetitive log output",
-      kind: "repetitive-log",
-      command: "tail -f worker.log --snapshot",
-      executionStatus: "success",
-      exitCode: 0,
-      stdout: repetitiveLog,
-      stderr: "",
-      combined: repetitiveLog,
-      requiredFacts: ["REPETITIVE_LOG completed after 3 retry cycles"],
-      thresholds: { largeOutputLines: 10, largeOutputBytes: 2_000 },
-    },
-    {
-      id: "huge-json-table",
-      title: "Huge JSON/table-like output",
-      kind: "huge-json-table",
-      command: "node scripts/dump-json.js",
-      executionStatus: "success",
-      exitCode: 0,
-      stdout: hugeJson,
-      stderr: "",
-      combined: hugeJson,
-      requiredFacts: ["HUGE_JSON_TABLE preserved header fact"],
-      thresholds: { largeOutputLines: 1_000, largeOutputBytes: 10_000 },
-    },
-    {
-      id: "repeated-command-output",
-      title: "Repeated command output",
-      kind: "repeated-output",
-      command: "printf repeated-output",
-      executionStatus: "success",
-      exitCode: 0,
-      stdout: repeatedOutput,
-      stderr: "",
-      combined: repeatedOutput,
-      requiredFacts: ["REPEATED_OUTPUT identical command output stays recoverable"],
-      repeatRuns: 2,
-      thresholds: { largeOutputLines: 10, largeOutputBytes: 10_000 },
-      notes: [
-        "Exact duplicate detection returns a compact note; current metadata points to the prior exact raw output.",
-      ],
-    },
-  ];
-}
-function commandFixture(definition, vaultRoot, externalComparators) {
-  const expected = {
-    executionStatus: definition.executionStatus,
-    exitCode: definition.exitCode,
-    requiredFacts: definition.requiredFacts,
-  };
-  if (definition.failedFactsMustBeExact !== undefined) {
-    expected.failedFactsMustBeExact = definition.failedFactsMustBeExact;
-  }
-  const modes = {
-    "native-baseline-proxy": async () => nativeCommandObservation(definition),
-    "improved-freeflow-run": async () => improvedCommandObservation(definition, vaultRoot),
-  };
-  for (const comparator of externalComparators) {
-    modes[comparator.mode] = async () => externalComparatorObservation(definition, comparator);
-  }
-  return {
-    id: definition.id,
-    title: definition.title,
-    kind: definition.kind,
-    expected,
-    modes,
-  };
-}
-async function nativeCommandObservation(fixture) {
-  return {
-    toolPathUsed: "native-baseline-proxy: direct command output",
-    rawBytes: byteLength(fixture.combined),
-    routedBytes: byteLength(fixture.combined),
-    executionStatus: fixture.executionStatus,
-    exitCode: fixture.exitCode,
-    routedExcerpt: fixture.combined,
-    recovery: { status: "not-applicable", detail: "Native direct output has no Freeflow vault recovery contract." },
-    notes: fixture.notes ?? [],
-  };
-}
-async function improvedCommandObservation(fixture, vaultRoot) {
-  const vault = createVault({ root: vaultRoot });
-  const sessionId = `command-benchmark-${fixture.id}`;
-  const outputIds = [];
-  let result = null;
-  const repeatRuns = Math.max(1, fixture.repeatRuns ?? 1);
-  for (let runIndex = 0; runIndex < repeatRuns; runIndex += 1) {
-    const runner = fixedCommandRunner(fixture);
-    const runOptions = {
-      command: fixture.command,
-      sessionId,
-      vaultRoot: vault.root,
-      preserve: fixture.preserve ?? "important",
-    };
-    if (fixture.goal !== undefined) {
-      runOptions.goal = fixture.goal;
-    }
-    if (fixture.thresholds !== undefined) {
-      runOptions.thresholds = fixture.thresholds;
-    }
-    result = await freeflowRun(runOptions, runner);
-    outputIds.push(result.outputId);
-  }
-  if (!result) {
-    return skippedObservation("improved-freeflow-run", "freeflowRun did not produce a result.");
-  }
-  const recovery = await verifyCommandRecovery(
-    vault.root,
-    sessionId,
-    result.recovery?.outputId ?? result.outputId,
-    fixture.combined,
-    fixture.requiredFacts,
-  );
-  const notes = [result.routing.reason, ...(fixture.notes ?? [])];
-  if (outputIds.length > 1) {
-    notes.push(`repeatedRuns=${outputIds.length}`);
-    notes.push(`outputIds=${outputIds.join(",")}`);
-  }
-  notes.push(`recovery=${recovery.status}: ${recovery.detail}`);
-  const observation = {
-    toolPathUsed: "improved-freeflow-run: freeflowRun synthetic fixture runner",
-    rawBytes: byteLength(fixture.combined),
-    routedBytes: byteLength(JSON.stringify(result)),
-    executionStatus: result.execution.status,
-    exitCode: result.execution.exitCode,
-    outputId: result.outputId,
-    routedExcerpt: routedText(result),
-    recovery,
-    notes,
-  };
-  if (result.parser) {
-    observation.parser = result.parser;
-  }
-  return observation;
-}
-async function externalComparatorObservation(fixture, comparator) {
-  const startedAt = performance.now();
-  const observation = await comparator.run({
-    id: fixture.id,
-    title: fixture.title,
-    command: fixture.command,
-    executionStatus: fixture.executionStatus,
-    exitCode: fixture.exitCode,
-    stdout: fixture.stdout,
-    stderr: fixture.stderr,
-    combined: fixture.combined,
-    requiredFacts: fixture.requiredFacts,
-  });
-  const latencyMs = observation.latencyMs ?? performance.now() - startedAt;
-  const notes = [
-    `externalComparator=${comparator.name}`,
-    `latencyMs=${latencyMs.toFixed(2)}`,
-    ...(observation.notes ?? []),
-  ];
-  const benchmarkObservation = {
-    toolPathUsed: `${comparator.name}: configured external command benchmark hook`,
-    rawBytes: observation.rawBytes ?? byteLength(fixture.combined),
-    routedBytes: observation.routedBytes ?? byteLength(observation.routedText),
-    executionStatus: fixture.executionStatus,
-    exitCode: fixture.exitCode,
-    routedExcerpt: observation.routedText,
-    recovery: { status: "not-applicable", detail: `${comparator.name} hook does not use Freeflow vault recovery.` },
-    notes,
-  };
-  if (observation.parser) {
-    benchmarkObservation.parser = observation.parser;
-  }
-  return benchmarkObservation;
-}
-function fixedCommandRunner(fixture) {
-  return {
-    async run() {
-      const result = {
-        stdout: fixture.stdout,
-        stderr: fixture.stderr,
-        combined: fixture.combined,
-        executionStatus: fixture.executionStatus,
-        exitCode: fixture.exitCode,
-      };
-      if (fixture.durationMs !== undefined) {
-        result.durationMs = fixture.durationMs;
-      }
-      return result;
-    },
-  };
-}
-async function verifyCommandRecovery(vaultRoot, sessionId, outputId, expectedCombined, requiredFacts) {
-  if (!outputId) {
-    return { status: "failed", detail: "No Freeflow outputId was returned." };
-  }
-  try {
-    const vault = createVault({ root: vaultRoot });
-    const recovered = await readOutputText(vault, sessionId, outputId, "combined");
-    if (recovered !== expectedCombined) {
-      return { status: "failed", detail: "Recovered combined output did not match exact fixture output." };
-    }
-    const missingFact = requiredFacts.find((fact) => !recovered.includes(fact));
-    if (missingFact) {
-      return { status: "failed", detail: `Recovered output missed required fact: ${missingFact}` };
-    }
-    return { status: "passed", detail: `Verified exact combined output recovery for ${outputId}.` };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    return { status: "failed", detail: `Recovery failed: ${message}` };
-  }
-}
-function scoreCommandCorrectness(expected, observation) {
-  if (observation.skipped) {
-    return {
-      passed: false,
-      statusCorrect: false,
-      exitCodeCorrect: false,
-      exactFactsPreserved: false,
-      failedFactsExact: false,
-    };
-  }
-  const statusCorrect = observation.executionStatus === expected.executionStatus;
-  const exitCodeCorrect = observation.exitCode === expected.exitCode;
-  const factsInRoutedExcerpt = expected.requiredFacts.every((fact) => observation.routedExcerpt.includes(fact));
-  const duplicateFactsRecoverable =
-    observation.parser?.name === "duplicate-output" && observation.recovery.status === "passed";
-  const exactFactsPreserved = factsInRoutedExcerpt || duplicateFactsRecoverable;
-  const failedFactsExact = expected.failedFactsMustBeExact ? exactFactsPreserved : true;
-  return {
-    passed: statusCorrect && exitCodeCorrect && exactFactsPreserved && failedFactsExact,
-    statusCorrect,
-    exitCodeCorrect,
-    exactFactsPreserved,
-    failedFactsExact,
-  };
-}
-function routedText(result) {
-  return result.importantLines?.map((line) => line.excerpt).join("\n") ?? "";
-}
-function commandBenchmarkModes(externalComparators) {
-  return [
-    "native-baseline-proxy",
-    "improved-freeflow-run",
-    ...externalComparators.map((comparator) => comparator.mode),
-  ];
-}
-function skippedExternalTools(externalComparators) {
-  const configuredNames = new Set(externalComparators.map((comparator) => comparator.name.toLowerCase()));
-  return DEFAULT_SKIPPED_EXTERNAL_TOOLS.filter((tool) => !configuredNames.has(tool.name.toLowerCase()));
-}
-async function createTempDir(prefix) {
-  const path = await mkdtemp(resolve(tmpdir(), prefix));
-  return {
-    path,
-    cleanup: async () => {
-      await rm(path, { recursive: true, force: true });
-    },
-  };
-}
-function skippedObservation(toolPathUsed, reason) {
-  return {
-    toolPathUsed,
-    rawBytes: 0,
-    routedBytes: 0,
-    routedExcerpt: "",
-    recovery: { status: "skipped", detail: reason },
-    skipped: true,
-    skipReason: reason,
-  };
-}
-function byteLength(value) {
-  return Buffer.byteLength(value, "utf8");
-}
-function formatCommandCorrectnessChecks(correctness) {
-  return [
-    `status ${correctness.statusCorrect ? "✓" : "✗"}`,
-    `exit ${correctness.exitCodeCorrect ? "✓" : "✗"}`,
-    `facts ${correctness.exactFactsPreserved ? "✓" : "✗"}`,
-    `failed-exact ${correctness.failedFactsExact ? "✓" : "✗"}`,
-  ].join(" ");
-}
-function defaultReportPath() {
-  return resolve(process.cwd(), "router/evals/reports/output-router-command-benchmark-1-report.md");
-}
-async function runCli() {
-  const { iterations, reportPath, jsonReportPath } = parseBenchmarkCliArgs(process.argv.slice(2), {
-    reportPath: defaultReportPath(),
-  });
-  const options = {};
-  if (iterations !== undefined) {
-    options.iterations = iterations;
-  }
-  const report = await runCommandBenchmarks(options);
-  const reports = await writeCommandBenchmarkReports(report, reportPath, {
-    jsonReportPath: jsonReportPath === undefined ? defaultJsonRunReportPath(reportPath) : jsonReportPath,
-  });
-  const shortId = createHash("sha256").update(JSON.stringify(report.summary)).digest("hex").slice(0, 8);
-  console.log(
-    `Freeflow command benchmark ${shortId}: improved ${report.summary.improved.passed}/${report.summary.fixtures} pass`,
-  );
-  console.log(`Markdown report: ${reports.markdown}`);
-  if (reports.json) {
-    console.log(`JSON run data: ${reports.json}`);
-  }
-  if (report.summary.improved.failed > 0) {
-    process.exitCode = 1;
-  }
-}
-if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
-  await runCli();
-}