@piagent/platform 1.5.4 → 1.6.0

package/packages/piagent-core/benchmark/benchmark-record-validation.js

@@ -1,4 +1,186 @@
 const benchmarkSurfaces = new Set(["raw-pi", "piagent", "codex-cli"]);
+const causalCoverageLanesV1 = new Set([
+  "telemetry-window", "session-lifecycle", "criterion-initial-pack", "pack-lifecycle", "direct-fallback-rereads", "managed-prefix"
+]);
+const causalCoverageLanesV2 = new Set([...causalCoverageLanesV1, "edit-recovery-context"]);
+const causalMaximumAggregate = 1_000_000_000;
+
+function exactKeys(value, keys) {
+  return value && typeof value === "object" && !Array.isArray(value)
+    && Object.keys(value).length === keys.length
+    && keys.every((key) => Object.hasOwn(value, key));
+}
+
+function boundedInteger(value) {
+  return Number.isSafeInteger(value) && value >= 0 && value <= causalMaximumAggregate;
+}
+
+function validCausalTriplet(value) {
+  return exactKeys(value, ["offered", "delivered", "injected"])
+    && [value.offered, value.delivered, value.injected].every(boundedInteger)
+    && value.delivered <= value.offered && value.injected === value.delivered;
+}
+
+function validCriterionAggregate(value, aggregate) {
+  if (!exactKeys(value, ["attempts", "selectedAttempts", "candidates", "selectedItems", "estimatedTokens", "zeroSelectionReasonCounts", "offered", "delivered", "injected"])
+    || ![value.attempts, value.selectedAttempts, value.candidates, value.selectedItems, value.estimatedTokens,
+      value.offered, value.delivered, value.injected].every(boundedInteger)
+    || !exactKeys(value.zeroSelectionReasonCounts, ["autoContextDisabled", "criterionGraphUnavailable", "noCandidates", "noReadableSelection"])
+    || !Object.values(value.zeroSelectionReasonCounts).every(boundedInteger)) return false;
+  const zeroAttempts = Object.values(value.zeroSelectionReasonCounts).reduce((sum, count) => sum + count, 0);
+  return value.selectedAttempts <= value.attempts
+    && value.selectedAttempts === value.offered
+    && value.offered === value.delivered && value.delivered === value.injected
+    && value.selectedItems >= value.selectedAttempts && value.candidates >= value.selectedItems
+    && zeroAttempts === value.attempts - value.selectedAttempts
+    && value.offered <= aggregate.packCounts.offered
+    && value.selectedItems <= aggregate.selectedItemCounts.offered
+    && value.estimatedTokens <= aggregate.estimatedTokens.offered;
+}
+
+function validEditRecoveryAggregate(value) {
+  if (!exactKeys(value, ["count", "failuresObserved", "suppressedFailures", "injectedChars", "injectedEstimatedTokens", "evidenceCoverage", "definition"])
+    || ![value.count, value.failuresObserved, value.suppressedFailures, value.injectedChars, value.injectedEstimatedTokens].every(boundedInteger)
+    || value.failuresObserved !== value.count + value.suppressedFailures
+    || !exactKeys(value.evidenceCoverage, ["status", "observed", "comparable", "rate"])
+    || value.evidenceCoverage.status !== "complete"
+    || ![value.evidenceCoverage.observed, value.evidenceCoverage.comparable].every(boundedInteger)
+    || value.evidenceCoverage.observed !== value.count || value.evidenceCoverage.comparable !== value.count
+    || value.evidenceCoverage.rate !== 1
+    || value.definition !== "matched-edit-recovery-context-receipt-v1") return false;
+  return value.count === 0
+    ? value.injectedChars === 0 && value.injectedEstimatedTokens === 0
+    : value.injectedChars > 0 && value.injectedEstimatedTokens > 0;
+}
+
+function validCausalAggregates(value, criterionExpected, schemaVersion) {
+  const expectedKeys = ["packCounts", "estimatedTokens", "selectedItemEstimatedTokens", "selectedItemCounts", "criterionInitialPack", "directFallbackRereads", "compaction", "managedPrefix"];
+  if (schemaVersion === 2) expectedKeys.splice(6, 0, "editRecoveryContext");
+  if (!exactKeys(value, expectedKeys)
+    || !validCausalTriplet(value.packCounts)
+    || !validCausalTriplet(value.estimatedTokens)
+    || !validCausalTriplet(value.selectedItemEstimatedTokens)
+    || !validCausalTriplet(value.selectedItemCounts)
+    || !exactKeys(value.directFallbackRereads, ["successfulCalls", "shellToolCallsObserved", "definition"])
+    || !boundedInteger(value.directFallbackRereads.successfulCalls)
+    || !boundedInteger(value.directFallbackRereads.shellToolCallsObserved)
+    || value.directFallbackRereads.definition !== "successful-direct-path-tool-call-v1"
+    || !exactKeys(value.compaction, ["eventsObserved", "state"])
+    || !boundedInteger(value.compaction.eventsObserved)
+    || !["observed", "not-observed"].includes(value.compaction.state)
+    || (value.compaction.eventsObserved > 0) !== (value.compaction.state === "observed")
+    || !exactKeys(value.managedPrefix, ["promptsObserved", "compactedPrompts", "state"])
+    || !boundedInteger(value.managedPrefix.promptsObserved) || value.managedPrefix.promptsObserved === 0
+    || !boundedInteger(value.managedPrefix.compactedPrompts)
+    || value.managedPrefix.compactedPrompts > value.managedPrefix.promptsObserved
+    || !["compacted", "uncompacted", "mixed"].includes(value.managedPrefix.state)
+    || (schemaVersion === 2 && !validEditRecoveryAggregate(value.editRecoveryContext))) return false;
+  const compacted = value.managedPrefix.compactedPrompts;
+  const prompts = value.managedPrefix.promptsObserved;
+  const expectedState = compacted === 0 ? "uncompacted" : compacted === prompts ? "compacted" : "mixed";
+  return value.managedPrefix.state === expectedState
+    && validCriterionAggregate(value.criterionInitialPack, value)
+    && (criterionExpected !== true || value.criterionInitialPack.attempts > 0);
+}
+
+export function validBenchmarkCausalContextReceipt(receipt, surface) {
+  if (!exactKeys(receipt, ["schemaVersion", "evidenceSource", "applicability", "available", "coverage", "aggregates"])
+    || ![1, 2].includes(receipt.schemaVersion) || typeof receipt.available !== "boolean"
+    || !exactKeys(receipt.coverage, ["status", "telemetryTruncated", "telemetryIntegrityFailures", "recoverableTailBytes", "criterionExpected", "sessionEventsObserved", "observedLanes", "requiredLanes", "missingLanes"])) return false;
+  const coverage = receipt.coverage;
+  if (surface !== "piagent") return receipt.schemaVersion === 1 && receipt.evidenceSource === "not-applicable"
+    && receipt.applicability === "not-applicable" && receipt.available === false && receipt.aggregates === null
+    && coverage.status === "not-applicable" && coverage.telemetryTruncated === false
+    && coverage.telemetryIntegrityFailures === 0 && coverage.recoverableTailBytes === 0
+    && coverage.criterionExpected === false && coverage.sessionEventsObserved === 0
+    && coverage.observedLanes === 0 && coverage.requiredLanes === 0
+    && Array.isArray(coverage.missingLanes) && coverage.missingLanes.length === 0;
+  const coverageLanes = receipt.schemaVersion === 2 ? causalCoverageLanesV2 : causalCoverageLanesV1;
+  const evidenceSource = `context-telemetry-closed-aggregate-v${receipt.schemaVersion}`;
+  if (receipt.evidenceSource !== evidenceSource || receipt.applicability !== "piagent"
+    || typeof coverage.telemetryTruncated !== "boolean" || typeof coverage.criterionExpected !== "boolean"
+    || ![coverage.telemetryIntegrityFailures, coverage.recoverableTailBytes, coverage.sessionEventsObserved,
+      coverage.observedLanes, coverage.requiredLanes].every(boundedInteger)
+    || coverage.requiredLanes !== coverageLanes.size || !Array.isArray(coverage.missingLanes)
+    || new Set(coverage.missingLanes).size !== coverage.missingLanes.length
+    || coverage.missingLanes.some((lane) => !coverageLanes.has(lane))
+    || coverage.observedLanes + coverage.missingLanes.length !== coverage.requiredLanes) return false;
+  if (receipt.available) return coverage.status === "complete" && coverage.missingLanes.length === 0
+    && coverage.telemetryTruncated === false && coverage.telemetryIntegrityFailures === 0
+    && coverage.recoverableTailBytes === 0 && coverage.sessionEventsObserved > 0
+    && validCausalAggregates(receipt.aggregates, coverage.criterionExpected, receipt.schemaVersion);
+  return ["partial", "unavailable"].includes(coverage.status)
+    && coverage.missingLanes.length > 0 && receipt.aggregates === null;
+}
+
+export function summarizeBenchmarkCausalContextEvidence(runs, { required = false } = {}) {
+  const piagentRuns = Array.isArray(runs) ? runs.filter((run) => run?.surface === "piagent") : [];
+  const complete = piagentRuns.filter((run) => validBenchmarkCausalContextReceipt(run.causalContextReceipt, run.surface)
+    && run.causalContextReceipt.available === true
+    && run.causalContextReceipt.coverage.status === "complete"
+    && run.causalContextReceipt.aggregates);
+  const fullyCovered = piagentRuns.length > 0 && complete.length === piagentRuns.length;
+  const recoveryComplete = complete.filter((run) => run.causalContextReceipt.schemaVersion === 2);
+  const recoveryFullyCovered = fullyCovered && recoveryComplete.length === piagentRuns.length;
+  const sum = (select) => complete.reduce((total, run) => total + select(run.causalContextReceipt.aggregates), 0);
+  const sumRecovery = (select) => recoveryComplete.reduce((total, run) => total + select(run.causalContextReceipt.aggregates.editRecoveryContext), 0);
+  const triplet = (field) => Object.fromEntries(["offered", "delivered", "injected"]
+    .map((lane) => [lane, sum((value) => value[field][lane])]));
+  const criterion = fullyCovered ? complete.reduce((total, run) => {
+    const value = run.causalContextReceipt.aggregates.criterionInitialPack;
+    for (const field of ["attempts", "selectedAttempts", "candidates", "selectedItems", "estimatedTokens", "offered", "delivered", "injected"]) {
+      total[field] += value[field];
+    }
+    for (const field of Object.keys(total.zeroSelectionReasonCounts)) {
+      total.zeroSelectionReasonCounts[field] += value.zeroSelectionReasonCounts[field];
+    }
+    return total;
+  }, {
+    attempts: 0, selectedAttempts: 0, candidates: 0, selectedItems: 0, estimatedTokens: 0,
+    zeroSelectionReasonCounts: { autoContextDisabled: 0, criterionGraphUnavailable: 0, noCandidates: 0, noReadableSelection: 0 },
+    offered: 0, delivered: 0, injected: 0
+  }) : null;
+  return {
+    required,
+    runs: piagentRuns.length,
+    availableRuns: complete.length,
+    unavailableRuns: piagentRuns.length - complete.length,
+    coverageStatus: fullyCovered ? "complete" : complete.length > 0 ? "partial" : "unavailable",
+    requiredSchemaVersion: 2,
+    currentAvailableRuns: recoveryComplete.length,
+    currentUnavailableRuns: piagentRuns.length - recoveryComplete.length,
+    currentCoverageStatus: recoveryFullyCovered ? "complete" : recoveryComplete.length > 0 ? "partial" : "unavailable",
+    aggregates: fullyCovered ? {
+      packCounts: triplet("packCounts"),
+      estimatedTokens: triplet("estimatedTokens"),
+      selectedItemEstimatedTokens: triplet("selectedItemEstimatedTokens"),
+      selectedItemCounts: triplet("selectedItemCounts"),
+      criterionInitialPack: criterion,
+      directFallbackRereads: {
+        successfulCalls: sum((value) => value.directFallbackRereads.successfulCalls),
+        shellToolCallsObserved: sum((value) => value.directFallbackRereads.shellToolCallsObserved),
+        definition: "successful-direct-path-tool-call-v1"
+      },
+      editRecoveryContext: {
+        count: recoveryFullyCovered ? sumRecovery((value) => value.count) : null,
+        failuresObserved: recoveryFullyCovered ? sumRecovery((value) => value.failuresObserved) : null,
+        suppressedFailures: recoveryFullyCovered ? sumRecovery((value) => value.suppressedFailures) : null,
+        injectedChars: recoveryFullyCovered ? sumRecovery((value) => value.injectedChars) : null,
+        injectedEstimatedTokens: recoveryFullyCovered ? sumRecovery((value) => value.injectedEstimatedTokens) : null,
+        evidenceCoverage: {
+          status: recoveryFullyCovered ? "complete" : recoveryComplete.length > 0 ? "partial" : "unavailable",
+          runs: piagentRuns.length,
+          comparableRuns: recoveryComplete.length,
+          rate: piagentRuns.length > 0 ? recoveryComplete.length / piagentRuns.length : 0
+        },
+        definition: "matched-edit-recovery-context-receipt-v1"
+      },
+      compactionEventsObserved: sum((value) => value.compaction.eventsObserved),
+      managedPrefixPromptsObserved: sum((value) => value.managedPrefix.promptsObserved),
+      managedPrefixCompactedPrompts: sum((value) => value.managedPrefix.compactedPrompts)
+    } : null
+  };
+}
 
 export function completedBenchmarkRecord(record) {
   const nonnegative = (value) => Number.isFinite(value) && value >= 0;
@@ -17,6 +199,7 @@ export function completedBenchmarkRecord(record) {
     && typeof record.profile === "string"
     && typeof record.lifecycle === "string"
     && benchmarkSurfaces.has(record.surface)
+    && validBenchmarkCausalContextReceipt(record.causalContextReceipt, record.surface)
     && Number.isInteger(record.repeat) && record.repeat > 0
     && Number.isInteger(record.infrastructureAttempt) && record.infrastructureAttempt > 0
     && Number.isInteger(record.infrastructureAttempts) && record.infrastructureAttempts > 0

package/packages/piagent-core/benchmark/benchmark-report.js

@@ -38,6 +38,16 @@ function pairedOutcomeSummary(value, candidateLabel, baselineLabel) {
   return `both ${value.bothPass} | ${candidateLabel}-only ${value.candidateOnlyPass} | ${baselineLabel}-only ${value.baselineOnlyPass} | neither ${value.bothFail}`;
 }
 
+function timingSummary(value, surface) {
+  const summary = value?.surfaces?.[surface];
+  if (!summary) return "unavailable";
+  const phase = (name) => {
+    const item = summary.phases?.[name];
+    return `${display(item?.medianSeconds, 3)}s (${item?.availableRuns ?? 0}/${summary.runs})`;
+  };
+  return `startup ${phase("processStartup")} | model-turn wait ${phase("modelTurnWait")} | tool ${phase("toolExecution")} | other ${phase("other")} | complete ${summary.completeDiagnostics}/${summary.runs}`;
+}
+
 function pairedUsageBandLines(bands) {
   return Object.entries(bands ?? {}).map(([name, band]) => (
     `  ${name}`.padEnd(30)
@@ -79,8 +89,10 @@ export function renderBenchmarkText(report) {
       + `${display(usage.medianInputTokens, 0)}`.padEnd(11)
       + `${display(usage.medianOutputTokens, 0)}`.padEnd(11)
       + `${display(usage.medianCacheReadTokens, 0)}`.padEnd(12)
+      + `${display(usage.medianCacheWriteTokens, 0)}`.padEnd(13)
       + `${display(usage.medianReasoningTokens, 0)}`.padEnd(11)
       + `${display(usage.medianFreshTokens, 0)}`.padEnd(11)
+      + `${display(usage.medianTotalTokens, 0)}`.padEnd(11)
       + `${display(usage.medianToolCalls, 0)}`.padEnd(7)
       + display(usage.medianCost, 6);
   };
@@ -89,6 +101,18 @@ export function renderBenchmarkText(report) {
   ));
   const usageBandLines = pairedUsageBandLines(report.comparison.pairedUsageBands?.categories);
   const confidence = report.comparison.freshTokenRatioConfidence95;
+  const primaryConfidence = report.comparison.primaryEfficiencyRatioConfidence95;
+  const fixedWorkloadConfidence = report.comparison.fixedWorkloadFamilyFreshTokenRatioConfidence95;
+  const failureAwareFamilyConfidence = report.comparison.failureAwareFamilyFreshTokenRatioConfidence95;
+  const durationConfidence = report.comparison.durationRatioConfidence95;
+  const normalizedCost = report.comparison.normalizedCost;
+  const normalizedCostConfidence = normalizedCost?.ratioConfidence95;
+  const causalContext = report.comparison.causalContextEvidence;
+  const causalAggregates = causalContext?.aggregates;
+  const hostReadiness = report.environment.hostReadinessHistory;
+  const suiteFailureReasons = (report.comparison.suiteGate?.failureReasons ?? [])
+    .map((failure) => `${failure.id}: ${failure.message}`)
+    .join("; ");
   const lines = [
     `Piagent Benchmark — ${report.suite.title}`,
     `Run: ${report.runId} | ${report.runCount} sessions | ${report.repeats} repeat(s)`,
@@ -98,6 +122,10 @@ export function renderBenchmarkText(report) {
     `Comparison: ${candidateLabel} vs ${baselineLabel}`,
     `Comparison purpose: ${report.comparison.purpose ?? "unspecified"}`,
     `Claim tier: ${report.comparison.claimEligibility?.achievedTier ?? "unavailable"} (declared ${report.comparison.claimEligibility?.declaredTier ?? "unavailable"})`,
+    `Token claim scope: ${report.comparison.claimEligibility?.tokenClaimScope ?? "unavailable"}`,
+    ...(report.comparison.tokenClaimUnavailableReason
+      ? [`Token claim restriction: ${report.comparison.tokenClaimUnavailableReason}`]
+      : []),
     `Suite digest: ${report.environment.suiteDigest ?? "unknown"} | Source: ${report.environment.source?.kind ?? "unknown"}${report.environment.source?.dirty === true ? " (dirty)" : ""}`,
     `Infrastructure: ${report.infrastructure?.attempts ?? report.runCount} attempts | ${report.infrastructure?.retries ?? 0} retries across ${report.infrastructure?.retriedRuns ?? 0} measured runs`,
     "",
@@ -106,41 +134,99 @@ export function renderBenchmarkText(report) {
     scoreLine(candidateLabel, candidate),
     "",
     "Median usage across all measured runs",
-    "Surface     Input      Output     Cache read  Reasoning  Fresh      Tools  Cost",
+    "Surface     Input      Output     Cache read  Cache write  Reasoning  Fresh      Total      Tools  Reported cost",
     usageLine(baselineLabel, baseline),
     usageLine(candidateLabel, candidate),
     `${baselineLabel} tools: ${topToolSummary(baseline.usage.toolNames)}`,
     `${candidateLabel} tools: ${topToolSummary(candidate.usage.toolNames)}`,
+    `Observational timing (${baselineLabel}): ${timingSummary(report.timingDiagnostics, baselineSurface)}`,
+    `Observational timing (${candidateLabel}): ${timingSummary(report.timingDiagnostics, candidateSurface)}`,
+    "Timing authority: privacy-safe monotonic JSONL receipt boundaries only; unavailable boundaries are not estimated and timing never affects duration or a release/spend gate.",
     `Piagent workflow gaps: ${workflowGapSummary(report.runs.filter((run) => run.surface === "piagent"))}`,
+    `Token accounting: input=fresh input; reasoning is included in output; fresh=input+output; total=input+cache read+cache write+output`,
+    `Attempt completeness: accepted ${report.tokenAccounting?.acceptedAttempts.exactAttempts ?? 0}/${report.tokenAccounting?.acceptedAttempts.attempts ?? 0} exact | failed ${report.tokenAccounting?.failedAttempts.exactAttempts ?? 0}/${report.tokenAccounting?.failedAttempts.attempts ?? 0} exact | failed fresh ${display(report.tokenAccounting?.failedAttempts.tokens.fresh, 0)}`,
     "",
     `Paired successful runs: ${report.comparison.pairedSuccessfulRuns}`,
     `Paired runs with comparable usage: ${report.comparison.pairedUsageRuns}`,
     `Independent paired scenario families: ${report.comparison.pairedUsageScenarios ?? 0}`,
     `Complete paired scenario families: ${report.comparison.pairedCompleteScenarios ?? 0}`,
+    `Paired runs with comparable duration: ${report.comparison.pairedDurationRuns ?? 0}`,
+    `Complete paired duration families: ${report.comparison.pairedCompleteDurationScenarios ?? 0}`,
     `Complete paired outcome families: ${report.comparison.pairedOutcomeScenarios ?? 0}`,
     `Usage estimator: ${report.comparison.usageEstimator}`,
+    `Primary efficiency estimand: ${report.comparison.primaryEfficiencyEstimand ?? "unavailable"}`,
+    `Primary fresh-token ratio: ${display(report.comparison.primaryEfficiencyRatio, 4)} | 95% CI ${primaryConfidence ? `${display(primaryConfidence.lower, 4)}..${display(primaryConfidence.upper, 4)}` : "n/a"}`,
+    `Fixed-workload family ratio: ${display(report.comparison.fixedWorkloadFamilyFreshTokenRatio, 4)} | 95% CI ${fixedWorkloadConfidence ? `${display(fixedWorkloadConfidence.lower, 4)}..${display(fixedWorkloadConfidence.upper, 4)}` : "n/a"} | complete ${report.comparison.fixedWorkloadFamilyCoverage?.usableScenarioFamilies ?? 0}/${report.comparison.fixedWorkloadFamilyCoverage?.expectedScenarioFamilies ?? 0}`,
+    `Fixed-workload aggregate ratio (descriptive): ${display(report.comparison.fixedWorkloadAggregateFreshTokenRatio, 4)}`,
     `Fresh-token pair wins: ${candidateLabel} ${report.comparison.pairedFreshTokenWins[candidateKey]} | ${baselineLabel} ${report.comparison.pairedFreshTokenWins[baselineKey]} | ties ${report.comparison.pairedFreshTokenWins.ties}`,
     `Median paired fresh-token delta: ${display(report.comparison.medianPairedFreshTokenDelta, 0)} tok (negative favors ${candidateLabel})`,
-    `Fresh-token ratio 95% CI: ${confidence ? `${display(confidence.lower, 4)}..${display(confidence.upper, 4)}` : "n/a"}`,
+    `Successful-pair complete-family ratio: ${display(report.comparison.freshTokenRatio, 4)} | 95% CI ${confidence ? `${display(confidence.lower, 4)}..${display(confidence.upper, 4)}` : "n/a"}`,
+    `All-successful-pairs descriptive ratio: ${display(report.comparison.allSuccessfulPairsFreshTokenRatio, 4)}`,
+    `Normalized API-equivalent text-token cost: ${display(normalizedCost?.ratio, 4)} | 95% CI ${normalizedCostConfidence ? `${display(normalizedCostConfidence.lower, 4)}..${display(normalizedCostConfidence.upper, 4)}` : "n/a"} | delta ${displayPercent(normalizedCost?.deltaPercent)}`,
+    `Normalized-cost source: ${normalizedCost?.pricingSnapshot?.id ?? "n/a"} | billed/provider-reported cost: no | exact/applicable pairs ${normalizedCost?.comparablePairs ?? 0}/${normalizedCost?.expectedPairs ?? 0}`,
+    `Duration pair wins: ${candidateLabel} ${report.comparison.pairedDurationWins?.[candidateKey] ?? 0} | ${baselineLabel} ${report.comparison.pairedDurationWins?.[baselineKey] ?? 0} | ties ${report.comparison.pairedDurationWins?.ties ?? 0}`,
+    `Median paired duration delta: ${display(report.comparison.medianPairedDurationDeltaSeconds, 2)}s (negative favors ${candidateLabel})`,
+    `Duration ratio 95% CI: ${durationConfidence ? `${display(durationConfidence.lower, 4)}..${display(durationConfidence.upper, 4)}` : "n/a"}`,
     `Failure-aware fresh tokens/resolved outcome: ${candidateLabel} ${display(report.comparison.freshTokensPerResolvedOutcome?.[candidateKey], 2)} | ${baselineLabel} ${display(report.comparison.freshTokensPerResolvedOutcome?.[baselineKey], 2)} | ratio ${display(report.comparison.failureAwareFreshTokenRatio, 4)}`,
+    `Failure-aware family ratio: ${display(report.comparison.failureAwareFamilyFreshTokenRatio, 4)} | 95% CI ${failureAwareFamilyConfidence ? `${display(failureAwareFamilyConfidence.lower, 4)}..${display(failureAwareFamilyConfidence.upper, 4)}` : "n/a"} | complete ${report.comparison.failureAwareFamilyCoverage?.usableScenarioFamilies ?? 0}/${report.comparison.failureAwareFamilyCoverage?.expectedScenarioFamilies ?? 0}`,
     `Paired resolved outcomes: ${pairedOutcomeSummary(report.comparison.pairedOutcomes?.resolved, candidateLabel, baselineLabel)}`,
     `Paired quality outcomes: ${pairedOutcomeSummary(report.comparison.pairedOutcomes?.quality, candidateLabel, baselineLabel)}`,
     `Paired safety outcomes: ${pairedOutcomeSummary(report.comparison.pairedOutcomes?.safety, candidateLabel, baselineLabel)}`,
     `Comparison protocol gate: ${report.comparison.comparisonProtocolGate?.passed ? "pass" : `fail (${(report.comparison.comparisonProtocolGate?.failedChecks ?? []).join(", ") || "missing evidence"})`}`,
+    `Provider-wire surface gate: ${report.comparison.providerWireSurfaceGate === null ? "n/a" : report.comparison.providerWireSurfaceGate ? "pass" : "fail"}`,
+    `Provider-wire evidence: ${report.comparison.providerWireEvidence?.verifiedRuns ?? 0}/${report.comparison.providerWireEvidence?.runs ?? 0} runs | ${report.comparison.providerWireEvidence?.groups?.length ?? 0} repeat groups | ${(report.comparison.providerWireEvidence?.driftGroups ?? []).length} base-prefix drift groups | failures ${topToolSummary(report.comparison.providerWireEvidence?.failureCounts)}`,
+    `Deferred tool surfaces: ${(report.comparison.providerWireEvidence?.groups ?? []).filter((group) => group.deferredToolSurfaceHashCount > 1).length} varying groups (reported separately; not base-prefix drift)`,
+    `Causal context receipt gate: ${report.comparison.causalContextEvidenceGate === null ? "n/a" : report.comparison.causalContextEvidenceGate ? "pass" : "fail"} | current v${causalContext?.requiredSchemaVersion ?? "?"} complete ${causalContext?.currentAvailableRuns ?? 0}/${causalContext?.runs ?? 0} Piagent runs`,
+    `Causal pack totals: offered ${causalAggregates?.packCounts.offered ?? "n/a"} | delivered ${causalAggregates?.packCounts.delivered ?? "n/a"} | injected ${causalAggregates?.packCounts.injected ?? "n/a"} | injected estimated tokens ${causalAggregates?.estimatedTokens.injected ?? "n/a"} | selected items ${causalAggregates?.selectedItemCounts.injected ?? "n/a"}`,
+    `Causal criterion/continuity: attempts ${causalAggregates?.criterionInitialPack.attempts ?? "n/a"} | selected ${causalAggregates?.criterionInitialPack.selectedAttempts ?? "n/a"} | successful direct fallback rereads ${causalAggregates?.directFallbackRereads.successfulCalls ?? "n/a"} | shell calls after injection ${causalAggregates?.directFallbackRereads.shellToolCallsObserved ?? "n/a"} | compactions ${causalAggregates?.compactionEventsObserved ?? "n/a"} | managed-prefix compacted ${causalAggregates?.managedPrefixCompactedPrompts ?? "n/a"}/${causalAggregates?.managedPrefixPromptsObserved ?? "n/a"}`,
+    `Causal edit recovery: contexts ${causalAggregates?.editRecoveryContext?.count ?? "n/a"}/${causalAggregates?.editRecoveryContext?.failuresObserved ?? "n/a"} failures | suppressed ${causalAggregates?.editRecoveryContext?.suppressedFailures ?? "n/a"} | injected chars ${causalAggregates?.editRecoveryContext?.injectedChars ?? "n/a"} | injected estimated tokens ${causalAggregates?.editRecoveryContext?.injectedEstimatedTokens ?? "n/a"} | evidence ${causalAggregates?.editRecoveryContext?.evidenceCoverage?.status ?? "unavailable"} (${causalAggregates?.editRecoveryContext?.evidenceCoverage?.comparableRuns ?? 0}/${causalAggregates?.editRecoveryContext?.evidenceCoverage?.runs ?? 0} runs)`,
+    `Release-claim configuration gate: ${report.comparison.releaseClaimConfigurationGate === false ? "fail" : report.comparison.releaseClaimConfigurationGate === true ? "pass" : "n/a"}`,
+    `Codex baseline gate: ${report.comparison.codexBaselineGate === null ? "n/a" : report.comparison.codexBaselineGate ? "pass" : "fail"}`,
+    `Clean release source gate: ${report.comparison.cleanReleaseSourceGate === null ? "n/a" : report.comparison.cleanReleaseSourceGate ? "pass" : "fail"}`,
+    `Host-readiness history gate: ${report.comparison.hostReadinessGate === null ? "n/a" : report.comparison.hostReadinessGate ? "pass" : "fail"} | valid receipts ${hostReadiness?.validReceiptCount ?? 0}/${hostReadiness?.receiptCount ?? 0} | coverage ${hostReadiness?.windowCoverage ?? "n/a"}`,
+    `Full-suite gate: ${report.comparison.fullSuiteGate === null ? "n/a" : report.comparison.fullSuiteGate ? "pass" : "fail"}`,
     `Paired outcome evidence gate: ${report.comparison.outcomeEvidenceGate ? "pass" : "fail"}`,
-    `Efficiency evidence gate: ${report.comparison.efficiencyEvidenceGate ? "pass" : "fail"}`,
-    `Efficiency category coverage: ${report.comparison.efficiencyBandCoverageGate ? "pass" : `fail (${(report.comparison.efficiencyCategoryCoverage?.missing ?? []).join(", ") || "missing evidence"})`}`,
-    `Failure-aware efficiency gate: ${report.comparison.failureAwareEfficiencyGate ? "pass" : "fail"}`,
+    `Successful-pair evidence (${report.comparison.successfulPairEfficiencyRole ?? "blocking"}): ${report.comparison.efficiencyEvidenceGate ? "complete" : "incomplete"}`,
+    `Successful-pair category coverage (${report.comparison.successfulPairEfficiencyRole ?? "blocking"}): ${report.comparison.efficiencyBandCoverageGate ? "complete" : `incomplete (${(report.comparison.efficiencyCategoryCoverage?.missing ?? []).join(", ") || "missing evidence"})`}`,
+    `Failure-aware efficiency (${report.comparison.failureAwareEfficiencyRole ?? "blocking"}): ${report.comparison.failureAwareEfficiencyGate ? "within threshold" : "unavailable or above threshold"}`,
+    `Primary efficiency evidence gate: ${report.comparison.primaryEfficiencyEvidenceGate ? "pass" : "fail"}`,
+    `Primary efficiency category coverage: ${report.comparison.primaryEfficiencyBandCoverageGate ? "pass" : `fail (${(report.comparison.primaryEfficiencyCategoryCoverage?.missing ?? []).join(", ") || "missing evidence"})`}`,
+    `Primary efficiency confidence gate: ${report.comparison.primaryEfficiencyConfidenceGate === null ? "n/a" : report.comparison.primaryEfficiencyConfidenceGate ? "pass" : "fail"}`,
+    `Primary efficiency gate: ${report.comparison.primaryEfficiencyGate === null ? "n/a" : report.comparison.primaryEfficiencyGate ? "pass" : "fail"}`,
     `Paired candidate-regression gate: ${report.comparison.pairedRegressionGate ? "pass" : "fail"}`,
     `Repeat-count gate: ${report.comparison.repeatGate === null ? "n/a" : report.comparison.repeatGate ? "pass" : "fail"}`,
     `Efficiency confidence gate: ${report.comparison.efficiencyConfidenceGate === null ? "n/a" : report.comparison.efficiencyConfidenceGate ? "pass" : "fail"}`,
+    `Fresh-token band gate: ${report.comparison.freshTokenBandGate === null ? "n/a" : report.comparison.freshTokenBandGate ? "pass" : `fail (${(report.comparison.freshTokenBandFailures ?? []).map((item) => `${item.dimension}:${item.name}=${item.observedBound ?? "n/a"}`).join(", ")})`}`,
+    `Fresh-token family guardrail: ${report.comparison.freshTokenFamilyGate === null ? "n/a" : report.comparison.freshTokenFamilyGate ? "pass" : `fail (${(report.comparison.freshTokenFamilyFailures ?? []).map((item) => `${item.scenarioId}=${item.ratio}`).join(", ")})`}`,
+    `Performance evidence gate: ${report.comparison.performanceEvidenceGate === null ? "n/a" : report.comparison.performanceEvidenceGate ? "pass" : "fail"}`,
+    `Performance point-estimate gate (ratio <= 1.0): ${report.comparison.performancePointEstimateGate === null ? "n/a" : report.comparison.performancePointEstimateGate ? "pass" : "fail"}`,
+    `Performance confidence gate: ${report.comparison.performanceConfidenceGate === null ? "n/a" : report.comparison.performanceConfidenceGate ? "pass" : "fail"}`,
+    `Duration band gate: ${report.comparison.durationBandGate === null ? "n/a" : report.comparison.durationBandGate ? "pass" : `fail (${(report.comparison.durationBandFailures ?? []).map((item) => `${item.dimension}:${item.name}=${item.ratio ?? "n/a"}`).join(", ")})`}`,
+    `Duration family guardrail: ${report.comparison.durationFamilyGate === null ? "n/a" : report.comparison.durationFamilyGate ? "pass" : `fail (${(report.comparison.durationFamilyFailures ?? []).map((item) => `${item.scenarioId}=${item.ratio ?? "n/a"}`).join(", ")})`}`,
+    `Infrastructure retry gate: ${report.comparison.infrastructureRetryGate === null ? "n/a" : report.comparison.infrastructureRetryGate ? "pass" : "fail"}`,
+    `Infrastructure failure-ledger gate: ${report.comparison.infrastructureFailureLedgerGate === null ? "n/a" : report.comparison.infrastructureFailureLedgerGate ? "pass" : `fail (${report.comparison.infrastructureFailureLedgerIssues?.length ?? 0} runs)`}`,
+    `Unknown infrastructure usage gate: ${report.comparison.unknownInfrastructureUsageGate === null ? "n/a" : report.comparison.unknownInfrastructureUsageGate ? "pass" : "fail"}`,
+    `Accepted usage completeness gate: ${report.comparison.acceptedUsageCompletenessGate === null ? "n/a" : report.comparison.acceptedUsageCompletenessGate ? "pass" : "fail"}`,
+    `Failed-attempt usage completeness gate: ${report.comparison.failedUsageCompletenessGate === null ? "n/a" : report.comparison.failedUsageCompletenessGate ? "pass" : "fail"}`,
+    `All-attempt usage completeness gate: ${report.comparison.allAttemptUsageCompletenessGate === null ? "n/a" : report.comparison.allAttemptUsageCompletenessGate ? "pass" : "fail"}`,
+    `Normalized-cost configuration gate: ${report.comparison.normalizedCostClaimConfigurationGate === null ? "n/a" : report.comparison.normalizedCostClaimConfigurationGate ? "pass" : "fail"}`,
+    `Normalized-cost pricing applicability gate: ${report.comparison.normalizedCostPricingApplicabilityGate === null ? "n/a" : report.comparison.normalizedCostPricingApplicabilityGate ? "pass" : "fail"}`,
+    `Normalized-cost confidence gate: ${report.comparison.normalizedCostConfidenceGate === null ? "n/a" : report.comparison.normalizedCostConfidenceGate ? "pass" : "fail"}`,
+    `Normalized-cost band gate: ${report.comparison.normalizedCostBandGate === null ? "n/a" : report.comparison.normalizedCostBandGate ? "pass" : "fail"}`,
+    `Normalized-cost family gate: ${report.comparison.normalizedCostFamilyGate === null ? "n/a" : report.comparison.normalizedCostFamilyGate ? "pass" : "fail"}`,
     `Quality gate: ${report.comparison.qualityGate ? "pass" : "fail"}`,
     `Reliability gate: ${report.comparison.reliabilityGate ? "pass" : "fail"}`,
-    `Fresh-token delta: ${displayPercent(report.comparison.freshTokenDeltaPercent)} (negative favors ${candidateLabel})`,
-    `Cost delta: ${displayPercent(report.comparison.costDeltaPercent)} (negative favors ${candidateLabel})`,
+    `Successful-pair complete-family token delta: ${displayPercent(report.comparison.freshTokenDeltaPercent)} (negative favors ${candidateLabel})`,
+    `Primary efficiency token delta: ${displayPercent(report.comparison.primaryEfficiencyDeltaPercent)} (negative favors ${candidateLabel})`,
+    `Duration delta: ${displayPercent(report.comparison.durationDeltaPercent)} (negative favors ${candidateLabel})`,
+    `Provider-reported cost delta: ${displayPercent(report.comparison.costDeltaPercent)} (negative favors ${candidateLabel}; not normalized cost)`,
     `Workflow gate: ${report.comparison.workflowGate === null ? "n/a" : report.comparison.workflowGate ? "pass" : "fail"}`,
     `Category gate: ${report.comparison.categoryGate === null ? "n/a" : report.comparison.categoryGate ? "pass" : "fail"}`,
     `Every outcome score > floor: ${report.comparison.outcomeScoreGate === null ? "n/a" : report.comparison.outcomeScoreGate ? "pass" : `fail (${(report.comparison.outcomeScoreFailures ?? []).map((item) => `${item.id}=${item.score ?? "n/a"}`).join(", ")})`}`,
+    ...(report.comparison.suiteGate ? [
+      `Suite gate: ${report.comparison.suiteGate.passed ? "pass" : "fail"}`,
+      `Suite gate failures: ${suiteFailureReasons || "none"}`
+    ] : []),
     ...(usageBandLines.length ? ["", "Paired fresh-token ratio by category", "Category                      Pairs   Families  Ratio       Median delta", ...usageBandLines] : []),
     ...(categoryLines.length ? ["", `${candidateLabel} category bands`, "Category                      Resolved   Score", ...categoryLines] : []),
     `Verdict: ${report.verdict.status}`,
@@ -163,21 +249,39 @@ export function renderBenchmarkHtml(report) {
   const surfaceEntries = [[baselineSurface, report.surfaces[baselineKey]], [candidateSurface, report.surfaces[candidateKey]]];
   const baselineLabel = surfaceLabel(baselineSurface);
   const candidateLabel = surfaceLabel(candidateSurface);
-  const rows = report.runs.map((run) => `<tr><td>${htmlEscape(run.scenarioId)}</td><td>${htmlEscape(run.category ?? "unspecified")}</td><td>${htmlEscape(run.difficulty ?? "unspecified")}</td><td>${htmlEscape(run.profile ?? "unspecified")}</td><td>${htmlEscape(run.lifecycle ?? "unspecified")}</td><td>${htmlEscape(run.surface)}</td><td>${run.repeat}</td><td>${run.infrastructureRetries ?? 0}</td><td>${run.resolved ? "PASS" : "FAIL"}</td><td>${run.grade?.passed ? "PASS" : "FAIL"}</td><td>${run.scope?.passed ? "PASS" : "FAIL"}</td><td>${display(run.workflow?.score)}</td><td>${htmlEscape((run.workflow?.checks ?? []).filter((check) => !check.passed).map((check) => check.id).join(", ") || "none")}</td><td>${htmlEscape(run.usage?.model ?? "unknown")}</td><td>${htmlEscape(run.usage?.thinkingLevel ?? "unknown")}</td><td>${display(run.usage?.input, 0)}</td><td>${display(run.usage?.output, 0)}</td><td>${display(run.usage?.cacheRead, 0)}</td><td>${display(run.usage?.reasoning, 0)}</td><td>${display(run.usage?.fresh, 0)}</td><td>${display(run.usage?.toolCalls, 0)}</td><td>${htmlEscape(topToolSummary(run.usage?.toolNames, 5))}</td><td>${display(run.usage?.cost, 6)}</td><td>${display(run.durationSeconds, 1)}</td><td>${htmlEscape(run.failure ?? "")}</td></tr>`).join("");
+  const rows = report.runs.map((run) => `<tr><td>${htmlEscape(run.scenarioId)}</td><td>${htmlEscape(run.category ?? "unspecified")}</td><td>${htmlEscape(run.difficulty ?? "unspecified")}</td><td>${htmlEscape(run.profile ?? "unspecified")}</td><td>${htmlEscape(run.lifecycle ?? "unspecified")}</td><td>${htmlEscape(run.surface)}</td><td>${run.repeat}</td><td>${run.infrastructureRetries ?? 0}</td><td>${run.resolved ? "PASS" : "FAIL"}</td><td>${run.grade?.passed ? "PASS" : "FAIL"}</td><td>${run.scope?.passed ? "PASS" : "FAIL"}</td><td>${display(run.workflow?.score)}</td><td>${htmlEscape((run.workflow?.checks ?? []).filter((check) => !check.passed).map((check) => check.id).join(", ") || "none")}</td><td>${htmlEscape(run.usage?.model ?? "unknown")}</td><td>${htmlEscape(run.usage?.thinkingLevel ?? "unknown")}</td><td>${htmlEscape(run.usage?.usageSource ?? "unknown")}</td><td>${display(run.usage?.input, 0)}</td><td>${display(run.usage?.output, 0)}</td><td>${display(run.usage?.cacheRead, 0)}</td><td>${display(run.usage?.cacheWrite, 0)}</td><td>${display(run.usage?.reasoning, 0)}</td><td>${display(run.usage?.fresh, 0)}</td><td>${display(run.usage?.total, 0)}</td><td>${display(run.usage?.toolCalls, 0)}</td><td>${htmlEscape(topToolSummary(run.usage?.toolNames, 5))}</td><td>${display(run.usage?.cost, 6)}</td><td>${display(run.durationSeconds, 1)}</td><td>${htmlEscape(run.failure ?? "")}</td></tr>`).join("");
   const scoreRows = surfaceEntries.map(([id, surface]) => `<tr><th>${htmlEscape(surfaceLabel(id))}</th><td>${surface.resolved}/${surface.runs}</td><td>${surface.qualityCorrect}/${surface.qualityRuns}</td><td>${surface.scopePassed}/${surface.runs}</td><td>${display(surface.scores.quality)}</td><td>${display(surface.scores.safety)}</td><td>${display(surface.scores.reliability)}</td><td>${display(surface.scores.workflow)}</td><td>${display(surface.scores.efficiency)}</td><td>${display(surface.scores.overall)}</td></tr>`).join("");
-  const usageRows = surfaceEntries.map(([id, surface]) => { const usage = surface.usage.allMeasuredRuns; return `<tr><th>${htmlEscape(surfaceLabel(id))}</th><td>${display(usage.medianInputTokens, 0)}</td><td>${display(usage.medianOutputTokens, 0)}</td><td>${display(usage.medianCacheReadTokens, 0)}</td><td>${display(usage.medianReasoningTokens, 0)}</td><td>${display(usage.medianFreshTokens, 0)}</td><td>${display(usage.medianToolCalls, 0)}</td><td>${display(usage.medianCost, 6)}</td><td>${display(usage.medianDurationSeconds, 1)}</td><td>${htmlEscape(topToolSummary(surface.usage.toolNames))}</td></tr>`; }).join("");
+  const usageRows = surfaceEntries.map(([id, surface]) => { const usage = surface.usage.allMeasuredRuns; return `<tr><th>${htmlEscape(surfaceLabel(id))}</th><td>${display(usage.medianInputTokens, 0)}</td><td>${display(usage.medianOutputTokens, 0)}</td><td>${display(usage.medianCacheReadTokens, 0)}</td><td>${display(usage.medianCacheWriteTokens, 0)}</td><td>${display(usage.medianReasoningTokens, 0)}</td><td>${display(usage.medianFreshTokens, 0)}</td><td>${display(usage.medianTotalTokens, 0)}</td><td>${display(usage.medianToolCalls, 0)}</td><td>${display(usage.medianCost, 6)}</td><td>${display(usage.medianDurationSeconds, 1)}</td><td>${htmlEscape(topToolSummary(surface.usage.toolNames))}</td></tr>`; }).join("");
   const categoryRows = Object.entries(report.surfaces[candidateKey].bands?.categories ?? {}).map(([name, band]) => `<tr><th>${htmlEscape(name)}</th><td>${band.resolved}/${band.runs}</td><td>${display(band.score)}</td><td>${display(band.correctness)}</td></tr>`).join("");
   const tokenBandRows = Object.entries(report.comparison.pairedUsageBands?.categories ?? {}).map(([name, band]) => `<tr><th>${htmlEscape(name)}</th><td>${band.pairs}</td><td>${band.scenarioFamilies}</td><td>${display(band.freshTokenRatio, 4)}</td><td>${display(band.medianFreshTokenDelta, 0)}</td><td>${band.candidateWins}/${band.baselineWins}/${band.ties}</td></tr>`).join("");
   const confidence = report.comparison.freshTokenRatioConfidence95;
+  const primaryConfidence = report.comparison.primaryEfficiencyRatioConfidence95;
+  const fixedWorkloadConfidence = report.comparison.fixedWorkloadFamilyFreshTokenRatioConfidence95;
+  const failureAwareFamilyConfidence = report.comparison.failureAwareFamilyFreshTokenRatioConfidence95;
+  const durationConfidence = report.comparison.durationRatioConfidence95;
+  const normalizedCost = report.comparison.normalizedCost;
+  const normalizedCostConfidence = normalizedCost?.ratioConfidence95;
+  const causalContext = report.comparison.causalContextEvidence;
+  const causalAggregates = causalContext?.aggregates;
+  const hostReadiness = report.environment.hostReadinessHistory;
+  const suiteFailureReasons = (report.comparison.suiteGate?.failureReasons ?? [])
+    .map((failure) => `${failure.id}: ${failure.message}`)
+    .join("; ");
   return `<!doctype html>
 <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Piagent Benchmark ${htmlEscape(report.runId)}</title>
 <style>body{font:14px system-ui,sans-serif;color:#202124;max-width:1180px;margin:32px auto;padding:0 20px}h1{font-size:24px}h2{font-size:17px;margin-top:28px}.table-wrap{overflow-x:auto}table{border-collapse:collapse;width:100%;white-space:nowrap}th,td{border:1px solid #d8dadd;padding:8px;text-align:left}thead th{background:#f4f5f6}.metric{display:inline-block;margin:0 24px 8px 0}.note{color:#5f6368}</style></head><body>
 <h1>Piagent Benchmark</h1><p>${htmlEscape(report.suite.title)} · ${htmlEscape(report.runId)}</p><p class="note">Platform v${htmlEscape(report.environment.platformVersion ?? "unknown")} · Pi ${htmlEscape(report.environment.piVersion ?? "unknown")}${report.environment.codexVersion ? ` · Codex ${htmlEscape(report.environment.codexVersion)}` : ""} · Node ${htmlEscape(report.environment.nodeVersion ?? "unknown")} · Baseline ${htmlEscape(report.environment.treatmentBaseline ?? "unknown")} · Piagent treatment ${htmlEscape(report.environment.piagentTreatment?.id ?? "unrecorded")} · Suite ${htmlEscape(report.environment.suiteDigest ?? "unknown")}</p>
-<p class="metric"><strong>Comparison:</strong> ${htmlEscape(candidateLabel)} vs ${htmlEscape(baselineLabel)}</p><p class="metric"><strong>Purpose:</strong> ${htmlEscape(report.comparison.purpose ?? "unspecified")}</p><p class="metric"><strong>Claim tier:</strong> ${htmlEscape(report.comparison.claimEligibility?.achievedTier ?? "unavailable")}</p><p class="metric"><strong>Verdict:</strong> ${htmlEscape(report.verdict.status)}</p><p class="metric"><strong>Protocol:</strong> ${report.comparison.comparisonProtocolGate?.passed ? "PASS" : `FAIL (${htmlEscape((report.comparison.comparisonProtocolGate?.failedChecks ?? []).join(", ") || "missing evidence")})`}</p><p class="metric"><strong>Infrastructure retries:</strong> ${report.infrastructure?.retries ?? 0}</p><p class="metric"><strong>Paired token-ratio delta:</strong> ${displayPercent(report.comparison.freshTokenDeltaPercent)}</p><p class="metric"><strong>Token ratio 95% CI:</strong> ${confidence ? `${display(confidence.lower, 4)}–${display(confidence.upper, 4)}` : "n/a"}</p><p class="metric"><strong>Failure-aware ratio:</strong> ${display(report.comparison.failureAwareFreshTokenRatio, 4)}</p><p class="metric"><strong>Paired cost-ratio delta:</strong> ${displayPercent(report.comparison.costDeltaPercent)}</p><p class="metric"><strong>Comparable pairs:</strong> ${report.comparison.pairedUsageRuns}</p><p class="metric"><strong>Scenario families:</strong> ${report.comparison.pairedUsageScenarios ?? 0}</p><p class="metric"><strong>Complete families:</strong> ${report.comparison.pairedCompleteScenarios ?? 0}</p><p class="metric"><strong>Outcome families:</strong> ${report.comparison.pairedOutcomeScenarios ?? 0}</p><p class="metric"><strong>Pair wins:</strong> ${htmlEscape(candidateLabel)} ${report.comparison.pairedFreshTokenWins[candidateKey]} · ${htmlEscape(baselineLabel)} ${report.comparison.pairedFreshTokenWins[baselineKey]} · ties ${report.comparison.pairedFreshTokenWins.ties}</p>
+<p class="metric"><strong>Comparison:</strong> ${htmlEscape(candidateLabel)} vs ${htmlEscape(baselineLabel)}</p><p class="metric"><strong>Purpose:</strong> ${htmlEscape(report.comparison.purpose ?? "unspecified")}</p><p class="metric"><strong>Claim tier:</strong> ${htmlEscape(report.comparison.claimEligibility?.achievedTier ?? "unavailable")}</p><p class="metric"><strong>Token claim scope:</strong> ${htmlEscape(report.comparison.claimEligibility?.tokenClaimScope ?? "unavailable")}</p>${report.comparison.tokenClaimUnavailableReason ? `<p class="metric"><strong>Token claim restriction:</strong> ${htmlEscape(report.comparison.tokenClaimUnavailableReason)}</p>` : ""}<p class="metric"><strong>Verdict:</strong> ${htmlEscape(report.verdict.status)}</p><p class="metric"><strong>Protocol:</strong> ${report.comparison.comparisonProtocolGate?.passed ? "PASS" : `FAIL (${htmlEscape((report.comparison.comparisonProtocolGate?.failedChecks ?? []).join(", ") || "missing evidence")})`}</p><p class="metric"><strong>Provider wire:</strong> ${report.comparison.providerWireSurfaceGate === null ? "n/a" : report.comparison.providerWireSurfaceGate ? "PASS" : "FAIL"} (${report.comparison.providerWireEvidence?.verifiedRuns ?? 0}/${report.comparison.providerWireEvidence?.runs ?? 0} runs, ${(report.comparison.providerWireEvidence?.driftGroups ?? []).length} drift groups)</p><p class="metric"><strong>Causal receipt:</strong> ${report.comparison.causalContextEvidenceGate === null ? "n/a" : report.comparison.causalContextEvidenceGate ? "PASS" : "FAIL"} (current v${causalContext?.requiredSchemaVersion ?? "?"} ${causalContext?.currentAvailableRuns ?? 0}/${causalContext?.runs ?? 0} runs)</p><p class="metric"><strong>Host readiness:</strong> ${report.comparison.hostReadinessGate === null ? "n/a" : report.comparison.hostReadinessGate ? "PASS" : "FAIL"} (${hostReadiness?.validReceiptCount ?? 0}/${hostReadiness?.receiptCount ?? 0} valid receipts · ${htmlEscape(hostReadiness?.windowCoverage ?? "n/a")} coverage)</p><p class="metric"><strong>Causal packs:</strong> ${causalAggregates ? `${causalAggregates.packCounts.offered}/${causalAggregates.packCounts.delivered}/${causalAggregates.packCounts.injected} offered/delivered/injected · ${causalAggregates.estimatedTokens.injected} injected estimated tokens · ${causalAggregates.selectedItemCounts.injected} selected items` : "unavailable"}</p><p class="metric"><strong>Causal direct rereads:</strong> ${causalAggregates ? `${causalAggregates.directFallbackRereads.successfulCalls} successful direct-path calls · ${causalAggregates.directFallbackRereads.shellToolCallsObserved} shell calls after injection` : "unavailable"}</p><p class="metric"><strong>Causal edit recovery:</strong> ${causalAggregates?.editRecoveryContext ? `${causalAggregates.editRecoveryContext.count ?? "n/a"} contexts · ${causalAggregates.editRecoveryContext.injectedChars ?? "n/a"} chars · ${causalAggregates.editRecoveryContext.injectedEstimatedTokens ?? "n/a"} estimated tokens · ${htmlEscape(causalAggregates.editRecoveryContext.evidenceCoverage?.status ?? "unavailable")} evidence` : "unavailable"}</p><p class="metric"><strong>Suite gate:</strong> ${report.comparison.suiteGate ? report.comparison.suiteGate.passed ? "PASS" : "FAIL" : "n/a"}</p><p class="metric"><strong>Infrastructure retries:</strong> ${report.infrastructure?.retries ?? 0}</p><p class="metric"><strong>Stability gate:</strong> ${report.comparison.stabilityGate === null ? "n/a" : report.comparison.stabilityGate ? "PASS" : "FAIL"}</p><p class="metric"><strong>Primary estimand:</strong> ${htmlEscape(report.comparison.primaryEfficiencyEstimand ?? "unavailable")}</p><p class="metric"><strong>Primary token delta:</strong> ${displayPercent(report.comparison.primaryEfficiencyDeltaPercent)}</p><p class="metric"><strong>Primary ratio 95% CI:</strong> ${primaryConfidence ? `${display(primaryConfidence.lower, 4)}–${display(primaryConfidence.upper, 4)}` : "n/a"}</p><p class="metric"><strong>Fixed-workload ratio:</strong> ${display(report.comparison.fixedWorkloadFamilyFreshTokenRatio, 4)}${fixedWorkloadConfidence ? ` (${display(fixedWorkloadConfidence.lower, 4)}–${display(fixedWorkloadConfidence.upper, 4)})` : ""}</p><p class="metric"><strong>Fixed-workload coverage:</strong> ${report.comparison.fixedWorkloadFamilyCoverage?.usableScenarioFamilies ?? 0}/${report.comparison.fixedWorkloadFamilyCoverage?.expectedScenarioFamilies ?? 0} families</p><p class="metric"><strong>Successful-pair diagnostic role:</strong> ${htmlEscape(report.comparison.successfulPairEfficiencyRole ?? "blocking")}</p><p class="metric"><strong>Successful-pair token delta:</strong> ${displayPercent(report.comparison.freshTokenDeltaPercent)}</p><p class="metric"><strong>Successful-pair ratio 95% CI:</strong> ${confidence ? `${display(confidence.lower, 4)}–${display(confidence.upper, 4)}` : "n/a"}</p><p class="metric"><strong>Normalized API-equivalent text-token cost ratio:</strong> ${display(normalizedCost?.ratio, 4)}${normalizedCostConfidence ? ` (${display(normalizedCostConfidence.lower, 4)}–${display(normalizedCostConfidence.upper, 4)})` : ""}</p><p class="metric"><strong>Normalized-cost gate:</strong> ${report.comparison.normalizedCostGate === null ? "n/a" : report.comparison.normalizedCostGate ? "PASS" : "FAIL"}</p><p class="metric"><strong>Failure-aware family ratio:</strong> ${display(report.comparison.failureAwareFamilyFreshTokenRatio, 4)}${failureAwareFamilyConfidence ? ` (${display(failureAwareFamilyConfidence.lower, 4)}–${display(failureAwareFamilyConfidence.upper, 4)})` : ""}</p><p class="metric"><strong>Failure-aware pooled ratio:</strong> ${display(report.comparison.failureAwareFreshTokenRatio, 4)}</p><p class="metric"><strong>Duration-ratio delta:</strong> ${displayPercent(report.comparison.durationDeltaPercent)}</p><p class="metric"><strong>Duration ratio 95% CI:</strong> ${durationConfidence ? `${display(durationConfidence.lower, 4)}–${display(durationConfidence.upper, 4)}` : "n/a"}</p><p class="metric"><strong>Performance gate:</strong> ${report.comparison.performanceGate === null ? "n/a" : report.comparison.performanceGate ? "PASS" : "FAIL"}</p><p class="metric"><strong>Provider-reported cost-ratio delta:</strong> ${displayPercent(report.comparison.costDeltaPercent)}</p><p class="metric"><strong>Comparable pairs:</strong> ${report.comparison.pairedUsageRuns}</p><p class="metric"><strong>Scenario families:</strong> ${report.comparison.pairedUsageScenarios ?? 0}</p><p class="metric"><strong>Complete families:</strong> ${report.comparison.pairedCompleteScenarios ?? 0}</p><p class="metric"><strong>Duration families:</strong> ${report.comparison.pairedCompleteDurationScenarios ?? 0}</p><p class="metric"><strong>Outcome families:</strong> ${report.comparison.pairedOutcomeScenarios ?? 0}</p><p class="metric"><strong>Pair wins:</strong> ${htmlEscape(candidateLabel)} ${report.comparison.pairedFreshTokenWins[candidateKey]} · ${htmlEscape(baselineLabel)} ${report.comparison.pairedFreshTokenWins[baselineKey]} · ties ${report.comparison.pairedFreshTokenWins.ties}</p>
+<p class="metric"><strong>Observational timing (${htmlEscape(baselineLabel)}):</strong> ${htmlEscape(timingSummary(report.timingDiagnostics, baselineSurface))}</p><p class="metric"><strong>Observational timing (${htmlEscape(candidateLabel)}):</strong> ${htmlEscape(timingSummary(report.timingDiagnostics, candidateSurface))}</p>
+<p class="note">Timing uses privacy-safe monotonic JSONL receipt boundaries only. Missing boundaries remain unavailable; these diagnostics do not alter authoritative duration or any release/spend gate.</p>
+<p class="metric"><strong>Edit-recovery failures:</strong> ${causalAggregates?.editRecoveryContext?.failuresObserved ?? "n/a"} observed · ${causalAggregates?.editRecoveryContext?.suppressedFailures ?? "n/a"} safely suppressed</p>
+<p class="metric"><strong>Exact attempts:</strong> accepted ${report.tokenAccounting?.acceptedAttempts.exactAttempts ?? 0}/${report.tokenAccounting?.acceptedAttempts.attempts ?? 0} · failed ${report.tokenAccounting?.failedAttempts.exactAttempts ?? 0}/${report.tokenAccounting?.failedAttempts.attempts ?? 0}</p><p class="metric"><strong>Failed-attempt fresh:</strong> ${display(report.tokenAccounting?.failedAttempts.tokens.fresh, 0)}</p>
+<p class="note"><strong>Token definitions:</strong> Input is fresh input only. Reasoning is already included in output. Fresh = input + output. Total = input + cache read + cache write + output. Normalized API-equivalent cost uses the versioned suite pricing snapshot and is not OAuth/provider-billed cost.</p>
+${suiteFailureReasons ? `<p><strong>Suite gate failures:</strong> ${htmlEscape(suiteFailureReasons)}</p>` : ""}
 <h2>Score bands</h2><div class="table-wrap"><table><thead><tr><th>Surface</th><th>Resolved</th><th>Task grader</th><th>Scope</th><th>Quality</th><th>Safety</th><th>Reliability</th><th>Workflow</th><th>Efficiency</th><th>Overall</th></tr></thead><tbody>${scoreRows}</tbody></table></div>
 ${categoryRows ? `<h2>${htmlEscape(candidateLabel)} category bands</h2><div class="table-wrap"><table><thead><tr><th>Category</th><th>Resolved</th><th>Score</th><th>Correctness</th></tr></thead><tbody>${categoryRows}</tbody></table></div>` : ""}
 ${tokenBandRows ? `<h2>Paired fresh-token ratio by category</h2><div class="table-wrap"><table><thead><tr><th>Category</th><th>Pairs</th><th>Families</th><th>Ratio</th><th>Median delta</th><th>${htmlEscape(candidateLabel)}/${htmlEscape(baselineLabel)}/ties</th></tr></thead><tbody>${tokenBandRows}</tbody></table></div>` : ""}
-<h2>Median usage across all measured runs</h2><div class="table-wrap"><table><thead><tr><th>Surface</th><th>Input</th><th>Output</th><th>Cache read</th><th>Reasoning</th><th>Fresh</th><th>Tools</th><th>Cost</th><th>Seconds</th><th>Top tools</th></tr></thead><tbody>${usageRows}</tbody></table></div>
-<h2>Runs</h2><div class="table-wrap"><table><thead><tr><th>Scenario</th><th>Category</th><th>Difficulty</th><th>Profile</th><th>Lifecycle</th><th>Surface</th><th>Repeat</th><th>Infra retries</th><th>Resolved</th><th>Grader</th><th>Scope</th><th>Workflow</th><th>Workflow gaps</th><th>Model</th><th>Thinking</th><th>Input</th><th>Output</th><th>Cache read</th><th>Reasoning</th><th>Fresh</th><th>Tools</th><th>Top tools</th><th>Cost</th><th>Seconds</th><th>Failure</th></tr></thead><tbody>${rows}</tbody></table></div>
+<h2>Median usage across all measured runs</h2><div class="table-wrap"><table><thead><tr><th>Surface</th><th>Fresh input</th><th>Output</th><th>Cache read</th><th>Cache write</th><th>Reasoning ⊂ output</th><th>Fresh</th><th>Total</th><th>Tools</th><th>Provider-reported cost</th><th>Seconds</th><th>Top tools</th></tr></thead><tbody>${usageRows}</tbody></table></div>
+<h2>Runs</h2><div class="table-wrap"><table><thead><tr><th>Scenario</th><th>Category</th><th>Difficulty</th><th>Profile</th><th>Lifecycle</th><th>Surface</th><th>Repeat</th><th>Infra retries</th><th>Resolved</th><th>Grader</th><th>Scope</th><th>Workflow</th><th>Workflow gaps</th><th>Model</th><th>Thinking</th><th>Usage source</th><th>Fresh input</th><th>Output</th><th>Cache read</th><th>Cache write</th><th>Reasoning ⊂ output</th><th>Fresh</th><th>Total</th><th>Tools</th><th>Top tools</th><th>Provider-reported cost</th><th>Seconds</th><th>Failure</th></tr></thead><tbody>${rows}</tbody></table></div>
 <p class="note">${htmlEscape(report.verdict.note)}</p></body></html>\n`;
 }

package/packages/piagent-core/benchmark/benchmark-resume-recovery.js

@@ -4,6 +4,7 @@ import path from "node:path";
 import { hardenPrivateRetentionRoot, retainedWorkspaceMarker, writeBenchmarkRunManifest, writePrivateAtomic } from "./benchmark-forensics.js";
 import { appendBenchmarkLedger, assertBenchmarkLedgerBinding, validateBenchmarkLedgerPrefix } from "./benchmark-ledger.js";
 import { expectedBenchmarkRecord, pairedBenchmarkVariantMatched } from "./benchmark-record-validation.js";
+import { canonicalBenchmarkTimingDiagnostics } from "./benchmark-timing-diagnostics.js";
 
 function fail(message) {
   const error = new Error(message);
@@ -29,6 +30,14 @@ function exactUsage(usage, usageStatus) {
     && usage.total === usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
 }
 
+function omitInvalidTimingDiagnostics(record) {
+  if (!record || typeof record !== "object") return record;
+  const timing = canonicalBenchmarkTimingDiagnostics(record.timingDiagnostics, record.surface, record.durationSeconds);
+  if (timing === undefined) delete record.timingDiagnostics;
+  else record.timingDiagnostics = timing;
+  return record;
+}
+
 export function persistUnacceptedBenchmarkAttempt({ runRoot, manifest, record, reason, forceTokenUnavailable = false }) {
   const attempt = {
     schemaVersion: 1,
@@ -63,6 +72,7 @@ export function clearRecoveredBenchmarkAttempts(manifest, record) {
 }
 
 export function stageMeasuredBenchmarkRecord({ runRoot, manifest, ledgerBinding, record, infrastructureFailures, index, expected, runId, suite, configurationDigest, runs }) {
+  omitInvalidTimingDiagnostics(record);
   record.infrastructureAttempts = record.infrastructureAttempt ?? 1;
   record.infrastructureRetries = Math.max(0, record.infrastructureAttempts - 1);
   record.infrastructureFailures = infrastructureFailures;
@@ -91,6 +101,7 @@ export function promoteMeasuredBenchmarkRecord({ runRoot, ledgerBinding, record,
 export function recoverPendingBenchmarkRecord({ runRoot, manifest, ledgerBinding, completedRuns, pending, measuredReady, fullOrder, suite }) {
   let binding = ledgerBinding;
   const runs = completedRuns;
+  let pendingRecordSanitized = false;
   if (pending) {
     if (pending.schemaVersion !== 2 || !pending.previousLedger || !pending.record || pending.postSessionGuard?.matched !== true
       || !String(pending.postSessionGuard.stage ?? "").startsWith("after-session:")) fail("Benchmark pending record is missing its post-session execution guard receipt");
@@ -100,6 +111,8 @@ export function recoverPendingBenchmarkRecord({ runRoot, manifest, ledgerBinding
     }
     if (binding.records === pending.previousLedger.records) {
       assertBenchmarkLedgerBinding(pending.previousLedger, binding, "pending-record previous ledger");
+      omitInvalidTimingDiagnostics(pending.record);
+      pendingRecordSanitized = true;
       binding = appendBenchmarkLedger(path.join(runRoot, "runs.jsonl"), pending.record, binding);
       runs.push(pending.record);
     } else if (binding.records === pending.previousLedger.records + 1) {
@@ -123,6 +136,7 @@ export function recoverPendingBenchmarkRecord({ runRoot, manifest, ledgerBinding
       fail("Measured benchmark record WAL does not match the frozen execution order");
     }
     if (pending) {
+      if (pendingRecordSanitized) omitInvalidTimingDiagnostics(measuredReady.record);
       assertBenchmarkLedgerBinding(measuredReady.previousLedger, pending.previousLedger, "measured/pending previous ledger");
       if (JSON.stringify(measuredReady.record) !== JSON.stringify(pending.record)) fail("Measured and post-guard pending records differ");
     } else {

package/packages/piagent-core/benchmark/benchmark-runner-policy.js

@@ -0,0 +1,26 @@
+export function applyBenchmarkExecutionDefaults(options, suite) {
+  options.repeats ??= suite.defaultRepeats;
+  options.infrastructureRetries ??= suite.releaseGate?.maximumInfrastructureRetries
+    ?? (suite.schemaVersion === 2 ? 2 : 0);
+  options.retryDelaySeconds ??= suite.schemaVersion === 2 && options.infrastructureRetries > 0 ? 60 : 0;
+  options.timeoutSeconds ??= suite.timeoutSeconds;
+  const contract = suite.executionContract;
+  if (!contract) return;
+  for (const [field, expected] of [
+    ["surfaces", contract.surfaces], ["model", contract.model], ["thinking", contract.thinking], ["codexMode", contract.codexMode]
+  ]) {
+    const actual = options[field];
+    if (actual === undefined) options[field] = Array.isArray(expected) ? [...expected] : expected;
+    else if (JSON.stringify(actual) !== JSON.stringify(expected)) {
+      throw new Error(`Suite ${suite.id} locks ${field} to ${Array.isArray(expected) ? expected.join(",") : expected}`);
+    }
+  }
+}
+
+export function benchmarkSuiteCoverage(declaredScenarios, selectedScenarios) {
+  return {
+    declaredScenarios,
+    selectedScenarios,
+    fullSuite: selectedScenarios === declaredScenarios
+  };
+}