@piagent/platform 1.2.17 → 1.3.0-ie.8

package/packages/piagent-core/benchmark/benchmark-preflight.js

@@ -0,0 +1,105 @@
+import {
+  codexProcessEnvironment,
+  controlledCodexFeatures
+} from "./benchmark-runtime.js";
+
+function fail(message) {
+  const error = new Error(message);
+  error.exitCode = 1;
+  throw error;
+}
+
+async function checkedVersion(runCommand, packageRoot, command, args, label, env = process.env) {
+  let result;
+  try { result = await runCommand(command, args, { cwd: packageRoot, timeoutMs: 15_000, env }); }
+  catch (error) { fail(`Required command is unavailable: ${label} (${error.message})`); }
+  if (result.code !== 0) fail(`Required command failed preflight: ${label} (${result.stderr.trim() || result.signal || `exit ${result.code}`})`);
+  return result.stdout.trim();
+}
+
+export async function benchmarkPreflight({ runCommand, packageRoot, piCommand, piEnvironment, codexCommand, gitCommand, surfaces, codexMode, codexRuntime }) {
+  const gitVersion = await checkedVersion(runCommand, packageRoot, gitCommand, ["--version"], "git");
+  const piVersion = await checkedVersion(runCommand, packageRoot, piCommand, ["--version"], "pi", piEnvironment);
+  let codexVersion;
+  let codexAuth;
+  let codexDisabledFeatures = [];
+  if (surfaces.includes("codex-cli")) {
+    const codexEnv = codexProcessEnvironment(codexRuntime);
+    codexVersion = await checkedVersion(runCommand, packageRoot, codexCommand, ["--version"], "codex", codexEnv);
+    let result;
+    try { result = await runCommand(codexCommand, ["login", "status"], { cwd: packageRoot, timeoutMs: 15_000, env: codexEnv }); }
+    catch (error) { fail(`Codex authentication preflight failed: ${error.message}`); }
+    if (result.code === 0) codexAuth = "login-status";
+    else if (process.env.OPENAI_API_KEY || process.env.CODEX_ACCESS_TOKEN) codexAuth = "environment-credential";
+    else fail("Codex CLI is not authenticated; run codex login before this benchmark");
+    if (codexMode === "controlled") {
+      const features = await runCommand(codexCommand, ["features", "list"], { cwd: packageRoot, timeoutMs: 15_000, env: codexEnv });
+      if (features.code !== 0) fail("Codex CLI cannot list features required by controlled benchmark mode; update Codex CLI or use --codex-mode native");
+      const available = new Set(features.stdout.split("\n").map((line) => line.trim().split(/\s+/)[0]).filter(Boolean));
+      codexDisabledFeatures = controlledCodexFeatures.filter((feature) => available.has(feature));
+    }
+  }
+  return { gitVersion, piVersion, codexVersion, codexAuth, codexDisabledFeatures };
+}
+
+function publicCommandIdentity(value) {
+  if (!value) return null;
+  return {
+    contentDigest: value.contentDigest,
+    size: value.size,
+    executable: value.executable,
+    package: value.packageClosure ? {
+      name: value.packageClosure.name,
+      version: value.packageClosure.version,
+      contentDigest: value.packageClosure.tree?.contentDigest ?? null
+    } : null
+  };
+}
+
+export function benchmarkPreflightReceipt({
+  packageVersion, source, candidateProvenance, suite, suiteDigest,
+  runtimeDependencies, runtimeCommands, environmentPolicy, configurationDigest,
+  rootSeedDigest, options, runtime
+}) {
+  return {
+    schemaVersion: 1,
+    kind: "benchmark-provider-free-preflight",
+    status: "ready",
+    providerSessionsStarted: 0,
+    packageVersion,
+    source,
+    candidateProvenance,
+    suite: { id: suite.id, contentDigest: suiteDigest, scenarioCount: suite.scenarios.length },
+    configuration: {
+      contentDigest: configurationDigest,
+      runtimeDependencyDigest: runtimeDependencies?.digest ?? null,
+      environmentPolicyDigest: environmentPolicy.digest,
+      rootSeedDigest,
+      surfaces: options.surfaces,
+      model: options.model ?? null,
+      thinking: options.thinking ?? null,
+      codexMode: options.codexMode,
+      piagentTreatment: options.piagentTreatment,
+      repeats: options.repeats,
+      timeoutSeconds: options.timeoutSeconds,
+      infrastructureRetries: options.infrastructureRetries,
+      retryDelaySeconds: options.retryDelaySeconds,
+      maxSessions: options.maxSessions ?? null,
+      stopAfterFailedPair: options.stopAfterFailedPair
+    },
+    runtime: {
+      gitVersion: runtime.gitVersion,
+      piVersion: runtime.piVersion,
+      codexVersion: runtime.codexVersion ?? null,
+      codexAuth: runtime.codexAuth ?? null,
+      codexDisabledFeatures: runtime.codexDisabledFeatures,
+      commands: Object.fromEntries(Object.entries(runtimeCommands).map(([name, value]) => [name, publicCommandIdentity(value)]))
+    },
+    usageContract: {
+      piagent: "pi-session-jsonl-exact-or-unavailable",
+      codexCli: "turn.completed.usage-cache-exclusive-fresh-exact-or-unavailable",
+      failedAttempts: "failure-aware-known-or-unknown-paid-attempt"
+    },
+    claimBoundary: "provider-free-preflight-only; no quality, workflow, token, latency, generalization, or release claim"
+  };
+}

package/packages/piagent-core/benchmark/benchmark-process.js

@@ -0,0 +1,141 @@
+import crypto from "node:crypto";
+import { spawn } from "node:child_process";
+
+const outputLimit = 4 * 1024 * 1024;
+
+function terminateGroup(child, signal) {
+  try {
+    if (process.platform !== "win32" && child.pid) process.kill(-child.pid, signal);
+    else child.kill(signal);
+  } catch {
+    // The process group may already have exited.
+  }
+}
+
+function groupAlive(child) {
+  if (process.platform === "win32" || !child.pid) return false;
+  try {
+    process.kill(-child.pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function createBenchmarkProcessController(interrupted) {
+  const active = new Set();
+  const killTimers = new Map();
+  const scheduleKill = (child, delay) => {
+    const prior = killTimers.get(child);
+    if (prior) clearTimeout(prior);
+    const timer = setTimeout(() => { killTimers.delete(child); terminateGroup(child, "SIGKILL"); }, delay);
+    timer.unref();
+    killTimers.set(child, timer);
+  };
+  const terminateAll = (signal) => {
+    for (const child of active) {
+      terminateGroup(child, signal);
+      scheduleKill(child, 2_000);
+    }
+  };
+  const run = (command, args, options = {}) => new Promise((resolve, reject) => {
+    if (interrupted()) {
+      reject(new Error("Benchmark was interrupted before the child process started"));
+      return;
+    }
+    const started = Date.now();
+    const child = spawn(command, args, {
+      cwd: options.cwd,
+      env: options.env ?? process.env,
+      stdio: options.inherit ? "inherit" : [options.input === undefined ? "ignore" : "pipe", "pipe", "pipe"],
+      detached: process.platform !== "win32"
+    });
+    active.add(child);
+    const cleanup = () => {
+      active.delete(child);
+      const killTimer = killTimers.get(child);
+      if (killTimer) clearTimeout(killTimer);
+      killTimers.delete(child);
+    };
+    if (!options.inherit && options.input !== undefined) {
+      child.stdin.on("error", () => { /* Child exit is reported by the process result. */ });
+      child.stdin.end(String(options.input));
+    }
+    let stdout = "";
+    let stderr = "";
+    const stdoutDigest = crypto.createHash("sha256");
+    const forbidden = [...new Set(options.forbiddenSubstrings ?? [])].filter((value) => typeof value === "string" && value);
+    const required = [...new Set(options.requiredSubstrings ?? [])].filter((value) => typeof value === "string" && value);
+    const forbiddenHits = new Set();
+    const requiredHits = new Set();
+    const scanTails = { stdout: "", stderr: "" };
+    const scanWindow = Math.max(0, ...[...forbidden, ...required].map((value) => value.length - 1));
+    const append = (current, chunk) => `${current}${chunk}`.slice(-outputLimit);
+    const inspect = (stream, chunk) => {
+      const text = `${scanTails[stream]}${chunk}`;
+      for (const value of forbidden) if (text.includes(value)) forbiddenHits.add(value);
+      if (stream === "stdout") for (const value of required) if (text.includes(value)) requiredHits.add(value);
+      scanTails[stream] = scanWindow > 0 ? text.slice(-scanWindow) : "";
+    };
+    if (!options.inherit) {
+      child.stdout.on("data", (chunk) => {
+        stdoutDigest.update(chunk);
+        options.onStdoutChunk?.(chunk);
+        inspect("stdout", chunk);
+        stdout = append(stdout, chunk);
+      });
+      child.stderr.on("data", (chunk) => {
+        inspect("stderr", chunk);
+        stderr = append(stderr, chunk);
+      });
+    }
+    let timedOut = false;
+    const timer = options.timeoutMs ? setTimeout(() => {
+      timedOut = true;
+      terminateGroup(child, "SIGTERM");
+      scheduleKill(child, 2_000);
+    }, options.timeoutMs) : undefined;
+    let settled = false;
+    child.once("error", (error) => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
+      cleanup();
+      terminateGroup(child, "SIGTERM");
+      reject(error);
+    });
+    child.once("exit", (code, signal) => {
+      if (timer) clearTimeout(timer);
+      terminateGroup(child, "SIGTERM");
+      scheduleKill(child, 500);
+    });
+    child.once("close", (code, signal) => {
+      if (settled) return;
+      if (timer) clearTimeout(timer);
+      const finish = () => {
+        if (settled) return;
+        settled = true;
+        cleanup();
+        resolve({ code: code ?? 1, signal, timedOut, stdout, stdoutHash: options.inherit ? undefined : stdoutDigest.digest("hex"), stderr, forbiddenHits: [...forbiddenHits], requiredHits: [...requiredHits], durationSeconds: (Date.now() - started) / 1000 });
+      };
+      if (!groupAlive(child)) {
+        finish();
+        return;
+      }
+      // A detached descendant can close its inherited stdio before exiting. Keep
+      // the process group owned until TERM has had time to work and the bounded
+      // KILL escalation has run, rather than treating stream close as cleanup.
+      const deadline = Date.now() + 1_000;
+      const drain = () => {
+        if (!groupAlive(child)) finish();
+        else if (Date.now() >= deadline) {
+          settled = true;
+          cleanup();
+          reject(new Error(`Benchmark child process group ${child.pid} survived bounded SIGKILL cleanup`));
+        } else setTimeout(drain, 20);
+      };
+      setTimeout(drain, 20);
+    });
+  });
+  return { run, terminateAll };
+}

package/packages/piagent-core/benchmark/benchmark-record-validation.js

@@ -0,0 +1,72 @@
+const benchmarkSurfaces = new Set(["raw-pi", "piagent", "codex-cli"]);
+
+export function completedBenchmarkRecord(record) {
+  const nonnegative = (value) => Number.isFinite(value) && value >= 0;
+  const hash = (value) => /^[a-f0-9]{64}$/.test(String(value ?? ""));
+  const usage = record?.usage;
+  return record?.schemaVersion === 1
+    && typeof record.runId === "string" && record.runId.length > 0
+    && typeof record.attemptId === "string" && record.attemptId.length > 0
+    && hash(record.configurationDigest)
+    && Number.isInteger(record.orderIndex) && record.orderIndex > 0
+    && typeof record.scenarioId === "string" && record.scenarioId.length > 0
+    && typeof record.scenarioTitle === "string"
+    && typeof record.scenarioKind === "string"
+    && typeof record.category === "string"
+    && typeof record.difficulty === "string"
+    && typeof record.profile === "string"
+    && typeof record.lifecycle === "string"
+    && benchmarkSurfaces.has(record.surface)
+    && Number.isInteger(record.repeat) && record.repeat > 0
+    && Number.isInteger(record.infrastructureAttempt) && record.infrastructureAttempt > 0
+    && Number.isInteger(record.infrastructureAttempts) && record.infrastructureAttempts > 0
+    && typeof record.sessionId === "string" && record.sessionId.length > 0
+    && record.abortSuite !== true
+    && typeof record.resolved === "boolean"
+    && Number.isInteger(record.agent?.exitCode)
+    && typeof record.agent?.timedOut === "boolean"
+    && hash(record.agent?.stdoutHash) && hash(record.agent?.stderrHash)
+    && typeof record.grade?.passed === "boolean" && nonnegative(record.grade?.score) && record.grade.score <= 10 && Array.isArray(record.grade?.checks)
+    && typeof record.graderIntegrity?.passed === "boolean"
+    && typeof record.scope?.passed === "boolean" && Array.isArray(record.scope?.changedFiles) && Array.isArray(record.scope?.outsideScope)
+    && typeof record.outputSafety?.passed === "boolean" && Array.isArray(record.outputSafety?.forbiddenHits)
+    && typeof record.outputEvidence?.passed === "boolean" && Number.isInteger(record.outputEvidence?.requiredCount)
+    && nonnegative(record.durationSeconds)
+    && hash(record.promptHash)
+    && typeof record.variant?.generated === "boolean"
+    && hash(record.variant?.fixtureDigest)
+    && (!record.variant.generated || (hash(record.variant.seedDigest) && hash(record.variant.oracleDigest)))
+    && Number.isInteger(usage?.sessions) && usage.sessions > 0
+    && nonnegative(usage?.fresh) && usage.fresh > 0
+    && ["input", "output", "cacheRead", "cacheWrite", "reasoning", "total"].every((field) => nonnegative(usage?.[field]))
+    && usage.total === usage.input + usage.output + usage.cacheRead + usage.cacheWrite
+    && usage.fresh === usage.input + usage.output
+    && (nonnegative(usage?.cost) || (usage?.cost === null && usage?.costSource === "unavailable"));
+}
+
+export function pairedBenchmarkVariantMatched(record, runs) {
+  const pair = runs.find((value) => value.scenarioId === record.scenarioId && value.repeat === record.repeat && value.surface !== record.surface);
+  if (!pair) return true;
+  return pair.promptHash === record.promptHash
+    && pair.variant?.generated === record.variant?.generated
+    && pair.variant?.fixtureDigest === record.variant?.fixtureDigest
+    && pair.variant?.seedDigest === record.variant?.seedDigest
+    && pair.variant?.oracleDigest === record.variant?.oracleDigest;
+}
+
+export function expectedBenchmarkRecord(record, index, expected, runId, suite, configurationDigest) {
+  const scenario = expected?.scenario;
+  return completedBenchmarkRecord(record)
+    && record.runId === runId
+    && record.configurationDigest === configurationDigest
+    && record.orderIndex === index + 1
+    && record.scenarioId === scenario?.id
+    && record.surface === expected?.surface
+    && record.repeat === expected?.repeat
+    && record.scenarioTitle === scenario?.title
+    && record.scenarioKind === scenario?.kind
+    && record.category === (scenario?.category ?? "unspecified")
+    && record.difficulty === (scenario?.difficulty ?? "unspecified")
+    && record.profile === (scenario?.profile ?? suite.profile)
+    && record.lifecycle === (scenario?.lifecycle ?? "steady-state");
+}

package/packages/piagent-core/benchmark/benchmark-report.js

@@ -0,0 +1,183 @@
+const SURFACE_LABELS = Object.freeze({ "raw-pi": "Raw Pi", piagent: "Piagent", "codex-cli": "Codex CLI" });
+const SURFACE_REPORT_KEYS = Object.freeze({ "raw-pi": "rawPi", piagent: "piagent", "codex-cli": "codexCli" });
+
+function surfaceLabel(surface) {
+  return SURFACE_LABELS[surface] ?? surface;
+}
+
+function surfaceReportKey(surface) {
+  return SURFACE_REPORT_KEYS[surface] ?? surface;
+}
+
+function display(value, digits = 2) {
+  return Number.isFinite(value) ? Number(value).toFixed(digits) : "n/a";
+}
+
+function displayPercent(value) {
+  return Number.isFinite(value) ? `${display(value)}%` : "n/a";
+}
+
+function topToolSummary(toolNames, limit = 8) {
+  const entries = Object.entries(toolNames ?? {}).slice(0, limit);
+  return entries.length ? entries.map(([name, count]) => `${name}:${count}`).join(", ") : "none";
+}
+
+function workflowGapSummary(runs) {
+  const gaps = {};
+  for (const run of runs ?? []) {
+    for (const check of run.workflow?.checks ?? []) {
+      if (check.passed === true) continue;
+      gaps[check.id] = (gaps[check.id] ?? 0) + 1;
+    }
+  }
+  return topToolSummary(gaps);
+}
+
+function pairedOutcomeSummary(value, candidateLabel, baselineLabel) {
+  if (!value) return "n/a";
+  return `both ${value.bothPass} | ${candidateLabel}-only ${value.candidateOnlyPass} | ${baselineLabel}-only ${value.baselineOnlyPass} | neither ${value.bothFail}`;
+}
+
+function pairedUsageBandLines(bands) {
+  return Object.entries(bands ?? {}).map(([name, band]) => (
+    `  ${name}`.padEnd(30)
+      + `${band.pairs}`.padEnd(8)
+      + `${band.scenarioFamilies}`.padEnd(10)
+      + `${display(band.freshTokenRatio, 4)}`.padEnd(12)
+      + display(band.medianFreshTokenDelta, 0)
+  ));
+}
+
+export function renderBenchmarkText(report) {
+  const baselineSurface = report.comparison.baselineSurface ?? "raw-pi";
+  const candidateSurface = report.comparison.candidateSurface ?? "piagent";
+  const baselineKey = surfaceReportKey(baselineSurface);
+  const candidateKey = surfaceReportKey(candidateSurface);
+  const baseline = report.surfaces[baselineKey];
+  const candidate = report.surfaces[candidateKey];
+  const baselineLabel = surfaceLabel(baselineSurface);
+  const candidateLabel = surfaceLabel(candidateSurface);
+  const runtimeParts = [
+    `Platform: v${report.environment.platformVersion ?? "unknown"}`,
+    `Pi: ${report.environment.piVersion ?? "unknown"}`
+  ];
+  if (report.environment.codexVersion) runtimeParts.push(`Codex: ${report.environment.codexVersion}`);
+  runtimeParts.push(`Node: ${report.environment.nodeVersion ?? "unknown"}`);
+  const scoreLine = (label, value) => `${label}`.padEnd(12)
+    + `${value.resolved}/${value.runs}`.padEnd(11)
+    + `${value.qualityCorrect}/${value.qualityRuns}`.padEnd(14)
+    + `${value.scopePassed}/${value.runs}`.padEnd(10)
+    + `${display(value.scores.quality)}`.padEnd(9)
+    + `${display(value.scores.safety)}`.padEnd(8)
+    + `${display(value.scores.reliability)}`.padEnd(13)
+    + `${display(value.scores.workflow)}`.padEnd(10)
+    + `${display(value.scores.efficiency)}`.padEnd(12)
+    + display(value.scores.overall);
+  const usageLine = (label, value) => {
+    const usage = value.usage.allMeasuredRuns;
+    return `${label}`.padEnd(12)
+      + `${display(usage.medianInputTokens, 0)}`.padEnd(11)
+      + `${display(usage.medianOutputTokens, 0)}`.padEnd(11)
+      + `${display(usage.medianCacheReadTokens, 0)}`.padEnd(12)
+      + `${display(usage.medianReasoningTokens, 0)}`.padEnd(11)
+      + `${display(usage.medianFreshTokens, 0)}`.padEnd(11)
+      + `${display(usage.medianToolCalls, 0)}`.padEnd(7)
+      + display(usage.medianCost, 6);
+  };
+  const categoryLines = Object.entries(candidate.bands?.categories ?? {}).map(([name, band]) => (
+    `  ${name}`.padEnd(30) + `${band.resolved}/${band.runs}`.padEnd(11) + display(band.score)
+  ));
+  const usageBandLines = pairedUsageBandLines(report.comparison.pairedUsageBands?.categories);
+  const confidence = report.comparison.freshTokenRatioConfidence95;
+  const lines = [
+    `Piagent Benchmark — ${report.suite.title}`,
+    `Run: ${report.runId} | ${report.runCount} sessions | ${report.repeats} repeat(s)`,
+    runtimeParts.join(" | "),
+    `Treatment baseline: ${report.environment.treatmentBaseline ?? "unknown"}`,
+    `Piagent treatment: ${report.environment.piagentTreatment?.id ?? "unrecorded"}`,
+    `Comparison: ${candidateLabel} vs ${baselineLabel}`,
+    `Comparison purpose: ${report.comparison.purpose ?? "unspecified"}`,
+    `Claim tier: ${report.comparison.claimEligibility?.achievedTier ?? "unavailable"} (declared ${report.comparison.claimEligibility?.declaredTier ?? "unavailable"})`,
+    `Suite digest: ${report.environment.suiteDigest ?? "unknown"} | Source: ${report.environment.source?.kind ?? "unknown"}${report.environment.source?.dirty === true ? " (dirty)" : ""}`,
+    `Infrastructure: ${report.infrastructure?.attempts ?? report.runCount} attempts | ${report.infrastructure?.retries ?? 0} retries across ${report.infrastructure?.retriedRuns ?? 0} measured runs`,
+    "",
+    "Surface     Resolved   Task grade    Scope     Quality  Safety  Reliability  Workflow  Efficiency  Overall",
+    scoreLine(baselineLabel, baseline),
+    scoreLine(candidateLabel, candidate),
+    "",
+    "Median usage across all measured runs",
+    "Surface     Input      Output     Cache read  Reasoning  Fresh      Tools  Cost",
+    usageLine(baselineLabel, baseline),
+    usageLine(candidateLabel, candidate),
+    `${baselineLabel} tools: ${topToolSummary(baseline.usage.toolNames)}`,
+    `${candidateLabel} tools: ${topToolSummary(candidate.usage.toolNames)}`,
+    `Piagent workflow gaps: ${workflowGapSummary(report.runs.filter((run) => run.surface === "piagent"))}`,
+    "",
+    `Paired successful runs: ${report.comparison.pairedSuccessfulRuns}`,
+    `Paired runs with comparable usage: ${report.comparison.pairedUsageRuns}`,
+    `Independent paired scenario families: ${report.comparison.pairedUsageScenarios ?? 0}`,
+    `Complete paired scenario families: ${report.comparison.pairedCompleteScenarios ?? 0}`,
+    `Complete paired outcome families: ${report.comparison.pairedOutcomeScenarios ?? 0}`,
+    `Usage estimator: ${report.comparison.usageEstimator}`,
+    `Fresh-token pair wins: ${candidateLabel} ${report.comparison.pairedFreshTokenWins[candidateKey]} | ${baselineLabel} ${report.comparison.pairedFreshTokenWins[baselineKey]} | ties ${report.comparison.pairedFreshTokenWins.ties}`,
+    `Median paired fresh-token delta: ${display(report.comparison.medianPairedFreshTokenDelta, 0)} tok (negative favors ${candidateLabel})`,
+    `Fresh-token ratio 95% CI: ${confidence ? `${display(confidence.lower, 4)}..${display(confidence.upper, 4)}` : "n/a"}`,
+    `Failure-aware fresh tokens/resolved outcome: ${candidateLabel} ${display(report.comparison.freshTokensPerResolvedOutcome?.[candidateKey], 2)} | ${baselineLabel} ${display(report.comparison.freshTokensPerResolvedOutcome?.[baselineKey], 2)} | ratio ${display(report.comparison.failureAwareFreshTokenRatio, 4)}`,
+    `Paired resolved outcomes: ${pairedOutcomeSummary(report.comparison.pairedOutcomes?.resolved, candidateLabel, baselineLabel)}`,
+    `Paired quality outcomes: ${pairedOutcomeSummary(report.comparison.pairedOutcomes?.quality, candidateLabel, baselineLabel)}`,
+    `Paired safety outcomes: ${pairedOutcomeSummary(report.comparison.pairedOutcomes?.safety, candidateLabel, baselineLabel)}`,
+    `Comparison protocol gate: ${report.comparison.comparisonProtocolGate?.passed ? "pass" : `fail (${(report.comparison.comparisonProtocolGate?.failedChecks ?? []).join(", ") || "missing evidence"})`}`,
+    `Paired outcome evidence gate: ${report.comparison.outcomeEvidenceGate ? "pass" : "fail"}`,
+    `Efficiency evidence gate: ${report.comparison.efficiencyEvidenceGate ? "pass" : "fail"}`,
+    `Efficiency category coverage: ${report.comparison.efficiencyBandCoverageGate ? "pass" : `fail (${(report.comparison.efficiencyCategoryCoverage?.missing ?? []).join(", ") || "missing evidence"})`}`,
+    `Failure-aware efficiency gate: ${report.comparison.failureAwareEfficiencyGate ? "pass" : "fail"}`,
+    `Paired candidate-regression gate: ${report.comparison.pairedRegressionGate ? "pass" : "fail"}`,
+    `Repeat-count gate: ${report.comparison.repeatGate === null ? "n/a" : report.comparison.repeatGate ? "pass" : "fail"}`,
+    `Efficiency confidence gate: ${report.comparison.efficiencyConfidenceGate === null ? "n/a" : report.comparison.efficiencyConfidenceGate ? "pass" : "fail"}`,
+    `Quality gate: ${report.comparison.qualityGate ? "pass" : "fail"}`,
+    `Reliability gate: ${report.comparison.reliabilityGate ? "pass" : "fail"}`,
+    `Fresh-token delta: ${displayPercent(report.comparison.freshTokenDeltaPercent)} (negative favors ${candidateLabel})`,
+    `Cost delta: ${displayPercent(report.comparison.costDeltaPercent)} (negative favors ${candidateLabel})`,
+    `Workflow gate: ${report.comparison.workflowGate === null ? "n/a" : report.comparison.workflowGate ? "pass" : "fail"}`,
+    `Category gate: ${report.comparison.categoryGate === null ? "n/a" : report.comparison.categoryGate ? "pass" : "fail"}`,
+    `Every outcome score > floor: ${report.comparison.outcomeScoreGate === null ? "n/a" : report.comparison.outcomeScoreGate ? "pass" : `fail (${(report.comparison.outcomeScoreFailures ?? []).map((item) => `${item.id}=${item.score ?? "n/a"}`).join(", ")})`}`,
+    ...(usageBandLines.length ? ["", "Paired fresh-token ratio by category", "Category                      Pairs   Families  Ratio       Median delta", ...usageBandLines] : []),
+    ...(categoryLines.length ? ["", `${candidateLabel} category bands`, "Category                      Resolved   Score", ...categoryLines] : []),
+    `Verdict: ${report.verdict.status}`,
+    `Token-saving claim allowed: ${report.comparison.tokenClaimAllowed ? "yes" : "no"}`,
+    `Generalization claim allowed: ${report.comparison.claimEligibility?.generalizationClaimAllowed ? "yes" : "no"}`,
+    `Claim limitations: ${(report.comparison.claimEligibility?.limitations ?? []).join(", ") || "none"}`
+  ];
+  return `${lines.join("\n")}\n`;
+}
+
+function htmlEscape(value) {
+  return String(value ?? "").replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
+}
+
+export function renderBenchmarkHtml(report) {
+  const baselineSurface = report.comparison.baselineSurface ?? "raw-pi";
+  const candidateSurface = report.comparison.candidateSurface ?? "piagent";
+  const baselineKey = surfaceReportKey(baselineSurface);
+  const candidateKey = surfaceReportKey(candidateSurface);
+  const surfaceEntries = [[baselineSurface, report.surfaces[baselineKey]], [candidateSurface, report.surfaces[candidateKey]]];
+  const baselineLabel = surfaceLabel(baselineSurface);
+  const candidateLabel = surfaceLabel(candidateSurface);
+  const rows = report.runs.map((run) => `<tr><td>${htmlEscape(run.scenarioId)}</td><td>${htmlEscape(run.category ?? "unspecified")}</td><td>${htmlEscape(run.difficulty ?? "unspecified")}</td><td>${htmlEscape(run.profile ?? "unspecified")}</td><td>${htmlEscape(run.lifecycle ?? "unspecified")}</td><td>${htmlEscape(run.surface)}</td><td>${run.repeat}</td><td>${run.infrastructureRetries ?? 0}</td><td>${run.resolved ? "PASS" : "FAIL"}</td><td>${run.grade?.passed ? "PASS" : "FAIL"}</td><td>${run.scope?.passed ? "PASS" : "FAIL"}</td><td>${display(run.workflow?.score)}</td><td>${htmlEscape((run.workflow?.checks ?? []).filter((check) => !check.passed).map((check) => check.id).join(", ") || "none")}</td><td>${htmlEscape(run.usage?.model ?? "unknown")}</td><td>${htmlEscape(run.usage?.thinkingLevel ?? "unknown")}</td><td>${display(run.usage?.input, 0)}</td><td>${display(run.usage?.output, 0)}</td><td>${display(run.usage?.cacheRead, 0)}</td><td>${display(run.usage?.reasoning, 0)}</td><td>${display(run.usage?.fresh, 0)}</td><td>${display(run.usage?.toolCalls, 0)}</td><td>${htmlEscape(topToolSummary(run.usage?.toolNames, 5))}</td><td>${display(run.usage?.cost, 6)}</td><td>${display(run.durationSeconds, 1)}</td><td>${htmlEscape(run.failure ?? "")}</td></tr>`).join("");
+  const scoreRows = surfaceEntries.map(([id, surface]) => `<tr><th>${htmlEscape(surfaceLabel(id))}</th><td>${surface.resolved}/${surface.runs}</td><td>${surface.qualityCorrect}/${surface.qualityRuns}</td><td>${surface.scopePassed}/${surface.runs}</td><td>${display(surface.scores.quality)}</td><td>${display(surface.scores.safety)}</td><td>${display(surface.scores.reliability)}</td><td>${display(surface.scores.workflow)}</td><td>${display(surface.scores.efficiency)}</td><td>${display(surface.scores.overall)}</td></tr>`).join("");
+  const usageRows = surfaceEntries.map(([id, surface]) => { const usage = surface.usage.allMeasuredRuns; return `<tr><th>${htmlEscape(surfaceLabel(id))}</th><td>${display(usage.medianInputTokens, 0)}</td><td>${display(usage.medianOutputTokens, 0)}</td><td>${display(usage.medianCacheReadTokens, 0)}</td><td>${display(usage.medianReasoningTokens, 0)}</td><td>${display(usage.medianFreshTokens, 0)}</td><td>${display(usage.medianToolCalls, 0)}</td><td>${display(usage.medianCost, 6)}</td><td>${display(usage.medianDurationSeconds, 1)}</td><td>${htmlEscape(topToolSummary(surface.usage.toolNames))}</td></tr>`; }).join("");
+  const categoryRows = Object.entries(report.surfaces[candidateKey].bands?.categories ?? {}).map(([name, band]) => `<tr><th>${htmlEscape(name)}</th><td>${band.resolved}/${band.runs}</td><td>${display(band.score)}</td><td>${display(band.correctness)}</td></tr>`).join("");
+  const tokenBandRows = Object.entries(report.comparison.pairedUsageBands?.categories ?? {}).map(([name, band]) => `<tr><th>${htmlEscape(name)}</th><td>${band.pairs}</td><td>${band.scenarioFamilies}</td><td>${display(band.freshTokenRatio, 4)}</td><td>${display(band.medianFreshTokenDelta, 0)}</td><td>${band.candidateWins}/${band.baselineWins}/${band.ties}</td></tr>`).join("");
+  const confidence = report.comparison.freshTokenRatioConfidence95;
+  return `<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Piagent Benchmark ${htmlEscape(report.runId)}</title>
+<style>body{font:14px system-ui,sans-serif;color:#202124;max-width:1180px;margin:32px auto;padding:0 20px}h1{font-size:24px}h2{font-size:17px;margin-top:28px}.table-wrap{overflow-x:auto}table{border-collapse:collapse;width:100%;white-space:nowrap}th,td{border:1px solid #d8dadd;padding:8px;text-align:left}thead th{background:#f4f5f6}.metric{display:inline-block;margin:0 24px 8px 0}.note{color:#5f6368}</style></head><body>
+<h1>Piagent Benchmark</h1><p>${htmlEscape(report.suite.title)} · ${htmlEscape(report.runId)}</p><p class="note">Platform v${htmlEscape(report.environment.platformVersion ?? "unknown")} · Pi ${htmlEscape(report.environment.piVersion ?? "unknown")}${report.environment.codexVersion ? ` · Codex ${htmlEscape(report.environment.codexVersion)}` : ""} · Node ${htmlEscape(report.environment.nodeVersion ?? "unknown")} · Baseline ${htmlEscape(report.environment.treatmentBaseline ?? "unknown")} · Piagent treatment ${htmlEscape(report.environment.piagentTreatment?.id ?? "unrecorded")} · Suite ${htmlEscape(report.environment.suiteDigest ?? "unknown")}</p>
+<p class="metric"><strong>Comparison:</strong> ${htmlEscape(candidateLabel)} vs ${htmlEscape(baselineLabel)}</p><p class="metric"><strong>Purpose:</strong> ${htmlEscape(report.comparison.purpose ?? "unspecified")}</p><p class="metric"><strong>Claim tier:</strong> ${htmlEscape(report.comparison.claimEligibility?.achievedTier ?? "unavailable")}</p><p class="metric"><strong>Verdict:</strong> ${htmlEscape(report.verdict.status)}</p><p class="metric"><strong>Protocol:</strong> ${report.comparison.comparisonProtocolGate?.passed ? "PASS" : `FAIL (${htmlEscape((report.comparison.comparisonProtocolGate?.failedChecks ?? []).join(", ") || "missing evidence")})`}</p><p class="metric"><strong>Infrastructure retries:</strong> ${report.infrastructure?.retries ?? 0}</p><p class="metric"><strong>Paired token-ratio delta:</strong> ${displayPercent(report.comparison.freshTokenDeltaPercent)}</p><p class="metric"><strong>Token ratio 95% CI:</strong> ${confidence ? `${display(confidence.lower, 4)}–${display(confidence.upper, 4)}` : "n/a"}</p><p class="metric"><strong>Failure-aware ratio:</strong> ${display(report.comparison.failureAwareFreshTokenRatio, 4)}</p><p class="metric"><strong>Paired cost-ratio delta:</strong> ${displayPercent(report.comparison.costDeltaPercent)}</p><p class="metric"><strong>Comparable pairs:</strong> ${report.comparison.pairedUsageRuns}</p><p class="metric"><strong>Scenario families:</strong> ${report.comparison.pairedUsageScenarios ?? 0}</p><p class="metric"><strong>Complete families:</strong> ${report.comparison.pairedCompleteScenarios ?? 0}</p><p class="metric"><strong>Outcome families:</strong> ${report.comparison.pairedOutcomeScenarios ?? 0}</p><p class="metric"><strong>Pair wins:</strong> ${htmlEscape(candidateLabel)} ${report.comparison.pairedFreshTokenWins[candidateKey]} · ${htmlEscape(baselineLabel)} ${report.comparison.pairedFreshTokenWins[baselineKey]} · ties ${report.comparison.pairedFreshTokenWins.ties}</p>
+<h2>Score bands</h2><div class="table-wrap"><table><thead><tr><th>Surface</th><th>Resolved</th><th>Task grader</th><th>Scope</th><th>Quality</th><th>Safety</th><th>Reliability</th><th>Workflow</th><th>Efficiency</th><th>Overall</th></tr></thead><tbody>${scoreRows}</tbody></table></div>
+${categoryRows ? `<h2>${htmlEscape(candidateLabel)} category bands</h2><div class="table-wrap"><table><thead><tr><th>Category</th><th>Resolved</th><th>Score</th><th>Correctness</th></tr></thead><tbody>${categoryRows}</tbody></table></div>` : ""}
+${tokenBandRows ? `<h2>Paired fresh-token ratio by category</h2><div class="table-wrap"><table><thead><tr><th>Category</th><th>Pairs</th><th>Families</th><th>Ratio</th><th>Median delta</th><th>${htmlEscape(candidateLabel)}/${htmlEscape(baselineLabel)}/ties</th></tr></thead><tbody>${tokenBandRows}</tbody></table></div>` : ""}
+<h2>Median usage across all measured runs</h2><div class="table-wrap"><table><thead><tr><th>Surface</th><th>Input</th><th>Output</th><th>Cache read</th><th>Reasoning</th><th>Fresh</th><th>Tools</th><th>Cost</th><th>Seconds</th><th>Top tools</th></tr></thead><tbody>${usageRows}</tbody></table></div>
+<h2>Runs</h2><div class="table-wrap"><table><thead><tr><th>Scenario</th><th>Category</th><th>Difficulty</th><th>Profile</th><th>Lifecycle</th><th>Surface</th><th>Repeat</th><th>Infra retries</th><th>Resolved</th><th>Grader</th><th>Scope</th><th>Workflow</th><th>Workflow gaps</th><th>Model</th><th>Thinking</th><th>Input</th><th>Output</th><th>Cache read</th><th>Reasoning</th><th>Fresh</th><th>Tools</th><th>Top tools</th><th>Cost</th><th>Seconds</th><th>Failure</th></tr></thead><tbody>${rows}</tbody></table></div>
+<p class="note">${htmlEscape(report.verdict.note)}</p></body></html>\n`;
+}