@piagent/platform 1.6.1 → 1.7.1

package/packages/piagent-core/benchmark/benchmark-core.js

@@ -1,36 +1,19 @@
 import { benchmarkClaimEligibility } from "./benchmark-assurance.js";
 import { integrateCodexRelativeEfficiencyReport } from "./benchmark-codex-relative-efficiency.js";
-import {
-  comparableAttemptUsage,
-  comparableDuration,
-  comparisonProtocol,
-  completeCategoryCoverage,
-  completePairedScenarioCount,
-  familyClusteredFailureAwareUsage,
-  familyClusteredFixedWorkloadUsage,
-  pairedDurationBands,
-  pairedUsageBands,
-  selectPrimaryEfficiencyEstimate,
-  tokensPerResolvedOutcome
-} from "./benchmark-comparison.js";
+import { comparableAttemptUsage, comparableDuration, comparisonProtocol, completeCategoryCoverage, completePairedScenarioCount,
+  familyClusteredFailureAwareUsage, familyClusteredFixedWorkloadUsage, pairedDurationBands, pairedUsageBands,
+  selectPrimaryEfficiencyEstimate, tokensPerResolvedOutcome } from "./benchmark-comparison.js";
 import { benchmarkProviderWireEvidenceMatchesRequest } from "./benchmark-provider-wire.js";
+import { validateBenchmarkSuite } from "./benchmark-suite.js";
+import { canonicalProductionSuiteId, summarizeBenchmarkReleaseClaimControls } from "./benchmark-release-claim-controls.js";
 import { canonicalBenchmarkTimingDiagnostics, summarizeBenchmarkTimingDiagnostics } from "./benchmark-timing-diagnostics.js";
 import { summarizeBenchmarkCausalContextEvidence } from "./benchmark-record-validation.js";
-import {
-  benchmarkPricingSnapshotValidationErrors,
-  normalizedCostComparison,
-  normalizedCostGate
-} from "./benchmark-normalized-cost.js";
-import {
-  atMostWithinFloatingPrecision,
-  geometricMean,
-  geometricMeanConfidence95,
-  geometricMeanConfidence95Raw,
-  median,
-  rounded
-} from "./benchmark-statistics.js";
+import { benchmarkPricingSnapshotValidationErrors, normalizedCostComparison, normalizedCostGate } from "./benchmark-normalized-cost.js";
+import { atMostWithinFloatingPrecision, geometricMean, median, rounded } from "./benchmark-statistics.js";
+import { hierarchicalMatrixRatioSummary, hierarchicalMatrixReportSummary, primaryEfficiencyMatrixSummary } from "./benchmark-matrix-summary.js";
 import { benchmarkInfrastructureFailureLedgerIssues, benchmarkTokenAccounting as buildBenchmarkTokenAccounting } from "./benchmark-usage.js";
 import {
+  benchmarkOutcomePairs, conditionalBothPassFreshUsage,
   efficiencyScore,
   evaluateDurationBandGate,
   evaluateFreshTokenBandGate,
@@ -47,8 +30,12 @@ export { renderBenchmarkHtml, renderBenchmarkText } from "./benchmark-report.js"
 export { benchmarkAssuranceEvidenceValidationErrors, benchmarkClaimEligibility } from "./benchmark-assurance.js";
 export { median } from "./benchmark-statistics.js";
 export { benchmarkSuiteValidationErrors, validateBenchmarkSuite } from "./benchmark-suite.js";
-export { aggregateSessionUsage, benchmarkTokenAccounting, createCodexExecJsonlCollector, parseCodexExecJsonl } from "./benchmark-usage.js";
+export { aggregateCodexTurnUsage, aggregateSessionUsage, benchmarkTokenAccounting, createCodexExecJsonlCollector, parseCodexExecJsonl } from "./benchmark-usage.js";
 export { benchmarkPricingSnapshotValidationErrors, normalizeBenchmarkUsageCost } from "./benchmark-normalized-cost.js";
+export { evaluateAllAttemptPooledFreshEfficiency } from "./benchmark-all-attempt-efficiency.js";
+export { inspectCodexRolloutServiceTierEvidence } from "./benchmark-codex-rollout.js";
+export { summarizeBenchmarkServiceTierEvidence } from "./benchmark-service-tier.js";
+export { effectiveResourcesPerResolvedOutcome, normalizedApiCostPerResolvedOutcome, totalTokensPerResolvedOutcome } from "./benchmark-comparison.js";
 export { CODEX_RELATIVE_EFFICIENCY_POLICY, PRODUCTION_SUBAGENT_BUDGET_POLICY, evaluateCodexRelativeEfficiency, summarizeBenchmarkSubagentBudget } from "./benchmark-codex-relative-efficiency.js";
 export { evaluateWorkflowEvidence } from "./benchmark-workflow.js";
 const SURFACE_LABELS = Object.freeze({
@@ -70,13 +57,16 @@ export function summarizeBenchmark({
   completedAt,
   repeats,
   environment = {},
+  allAttemptTokenAccounting,
   runs,
   baselineSurface = "raw-pi",
   candidateSurface = "piagent"
 }) {
   if (baselineSurface === candidateSurface) throw new Error("Benchmark surfaces must be different");
-  if (canonicalProductionSuite && suite.id !== "production-v1") {
-    throw new Error("Canonical production gate requires the built-in production-v1 suite identity");
+  // Validate numerical opt-in only; it grants no claim and leaves unversioned behavior unchanged.
+  if (suite.releaseGate?.efficiencyProtocol !== undefined) validateBenchmarkSuite(suite);
+  if (canonicalProductionSuite && !canonicalProductionSuiteId(suite.id)) {
+    throw new Error("Canonical production gate requires a built-in production suite identity");
   }
   const baselineRuns = runs.filter((run) => run.surface === baselineSurface);
   const candidateRuns = runs.filter((run) => run.surface === candidateSurface);
@@ -85,7 +75,10 @@ export function summarizeBenchmark({
   }
   const baseline = surfaceSummary(baselineSurface, baselineRuns);
   const candidate = surfaceSummary(candidateSurface, candidateRuns);
-  const tokenAccounting = buildBenchmarkTokenAccounting(runs);
+  const measuredTokenAccounting = buildBenchmarkTokenAccounting(runs);
+  const tokenAccounting = allAttemptTokenAccounting
+    ? { ...measuredTokenAccounting, allAttempts: allAttemptTokenAccounting }
+    : measuredTokenAccounting;
   const timingDiagnostics = summarizeBenchmarkTimingDiagnostics(runs);
   const reportRuns = runs.map((run) => {
     const timing = canonicalBenchmarkTimingDiagnostics(run.timingDiagnostics, run.surface, run.durationSeconds);
@@ -94,10 +87,7 @@ export function summarizeBenchmark({
     delete sanitized.timingDiagnostics;
     return sanitized;
   });
-  const baselineByKey = new Map(baselineRuns.map((run) => [`${run.scenarioId}:${run.repeat}`, run]));
-  const allPairs = candidateRuns
-    .map((run) => ({ baseline: baselineByKey.get(`${run.scenarioId}:${run.repeat}`), candidate: run }))
-    .filter((pair) => pair.baseline);
+  const allPairs = benchmarkOutcomePairs(baselineRuns, candidateRuns);
   const pairs = allPairs.filter((pair) => pair.baseline.resolved && pair.candidate.resolved);
   const tokenPairs = pairs.filter(comparableAttemptUsage);
   const completeOutcomeScenarios = completePairedScenarioCount(allPairs, repeats);
@@ -152,16 +142,8 @@ export function summarizeBenchmark({
   ));
   const costRatios = costPairs.map((pair) => pair.candidate.usage.cost / pair.baseline.usage.cost);
   const allSuccessfulPairsFreshRatio = geometricMean(freshRatios);
-  const confidenceScenarioRatios = suite.schemaVersion === 2 ? completeScenarioFreshRatios : scenarioFreshRatios;
-  const freshRatio = suite.schemaVersion === 2
-    ? geometricMean(confidenceScenarioRatios.map((item) => item.ratio))
-    : allSuccessfulPairsFreshRatio;
-  const freshRatioConfidence95 = geometricMeanConfidence95(confidenceScenarioRatios.map((item) => item.ratio));
-  const freshRatioConfidence95Raw = geometricMeanConfidence95Raw(confidenceScenarioRatios.map((item) => item.ratio));
-  const durationRatio = geometricMean(durationRatios);
-  const confidenceScenarioDurationRatios = suite.schemaVersion === 2 ? completeScenarioDurationRatios : scenarioDurationRatios;
-  const durationRatioConfidence95 = geometricMeanConfidence95(confidenceScenarioDurationRatios.map((item) => item.ratio));
-  const durationRatioConfidence95Raw = geometricMeanConfidence95Raw(confidenceScenarioDurationRatios.map((item) => item.ratio));
+  const matrixRatios = hierarchicalMatrixRatioSummary({ suite, completeScenarioFreshRatios, scenarioFreshRatios, allSuccessfulPairsFreshRatio, completeScenarioDurationRatios, scenarioDurationRatios, durationRatios });
+  const { hierarchicalFreshRatioSample, confidenceScenarioRatios, freshRatio, freshRatioConfidence95, freshRatioConfidence95Raw, hierarchicalDurationRatioSample, confidenceScenarioDurationRatios, durationRatio, durationRatioConfidence95, durationRatioConfidence95Raw } = matrixRatios;
   const costRatio = geometricMean(costRatios);
   const freshDeltas = tokenPairs.map((pair) => pair.candidate.usage.fresh - pair.baseline.usage.fresh);
   const freshWins = freshDeltas.filter((delta) => delta < 0).length;
@@ -225,9 +207,12 @@ export function summarizeBenchmark({
     ratio: freshRatio,
     confidence95: freshRatioConfidence95,
     confidence95Raw: freshRatioConfidence95Raw,
-    scenarioRatios: completeScenarioFreshRatios
+    scenarioRatios: hierarchicalFreshRatioSample.matrix ? confidenceScenarioRatios : completeScenarioFreshRatios
   }, familyClusteredFailureAware, familyClusteredFixedWorkload);
-  const primaryEfficiencyCompleteScenarios = primaryEfficiencyScenarioRatios.length;
+  const matrixHierarchy = hierarchicalFreshRatioSample.matrix;
+  const primaryMatrix = primaryEfficiencyMatrixSummary(matrixHierarchy, primaryEfficiencyScenarioRatios);
+  const { scenarioIds: primaryEfficiencyScenarioIds, familyIds: primaryEfficiencyFamilyIds,
+    completeScenarios: primaryEfficiencyCompleteScenarios, completeSamples: primaryEfficiencyCompleteSamples } = primaryMatrix;
   const primaryEfficiencyCategoryCoverage = completeCategoryCoverage(suite, primaryEfficiencyScenarioRatios);
   const requiresConfidenceEfficiency = Number.isFinite(maximumFreshTokenRatioUpper95) || releaseGate.requireEfficiencyClaim === true;
   const requiresBandEfficiency = Number.isFinite(maximumBandFreshTokenRatio);
@@ -249,21 +234,21 @@ export function summarizeBenchmark({
   const requiresSuiteEfficiency = requiresConfidenceEfficiency;
   const requiresPerformance = Number.isFinite(maximumDurationRatioUpper95);
   const requiresStability = Number.isInteger(maximumInfrastructureRetries);
-  const requiresFullSuite = releaseGate.requireFullSuiteForClaim === true;
-  const requiresProviderWireSurface = releaseGate.requireStableProviderWireSurface === true;
-  const requiresCausalContextReceipt = releaseGate.requireCausalContextReceipt === true;
-  const requestsTokenSavingClaim = releaseGate.requireEfficiencyClaim === true;
-  const requestsNormalizedCostClaim = releaseGate.requireNormalizedCostClaim === true;
-  const requiresHostReadiness = releaseGate.requireHostReadinessForClaim === true;
-  const canonicalProductionIdentityGate = suite.id !== "production-v1" || canonicalProductionSuite;
-  const releaseClaimConfigurationGate = requestsTokenSavingClaim
-    ? suite.schemaVersion === 2 && Number.isFinite(maximumFreshTokenRatioUpper95)
-      && maximumFreshTokenRatioUpper95 <= 0.8 && requiresFullSuite && requiresProviderWireSurface
-      && requiresCausalContextReceipt
-      && ["successful-pair-family-ratio", "failure-aware-family-ratio", "fixed-workload-family-ratio"].includes(primaryEfficiencyEstimand)
-      && (!primaryUsesFixedWorkload
-        || (!Number.isFinite(maximumBandFreshTokenRatio) && !Number.isFinite(maximumFamilyFreshTokenRatio)))
-    : null;
+  const {
+    maximumAllAttemptPooledFreshTokenRatio,
+    requiresFullSuite, requiresProviderWireSurface, requiresCausalContextReceipt, requiresFastServiceTier, requiresCampaignAccounting,
+    requestsTokenSavingClaim, requestsNormalizedCostClaim, requiresHostReadiness,
+    canonicalProductionIdentityGate, releaseClaimConfigurationGate,
+    acceptedUsageCompletenessGate, failedUsageCompletenessGate, allAttemptUsageCompletenessGate,
+    infrastructureFailureLedgerGate, codexBaselineGate,
+    allAttemptPooledEfficiency, allAttemptPooledEfficiencyGate,
+    serviceTierEvidence, fastServiceTierGate,
+    cleanReleaseSourceGate, hostReadinessHistory, hostReadinessGate, campaignEvidence, campaignAccountingGate
+  } = summarizeBenchmarkReleaseClaimControls({
+    suite, canonicalProductionSuite, tokenAccounting, infrastructureFailureLedgerIssues, runs, environment,
+    baselineSurface, candidateSurface, maximumFreshTokenRatioUpper95, maximumBandFreshTokenRatio,
+    maximumFamilyFreshTokenRatio, primaryEfficiencyEstimand, primaryUsesFixedWorkload
+  });
   const normalizedCostClaimConfigurationGate = requestsNormalizedCostClaim
     ? requestsTokenSavingClaim
       && suite.schemaVersion === 2
@@ -291,30 +276,6 @@ export function summarizeBenchmark({
     familyFailures: [],
     passed: null
   };
-  const acceptedUsageCompletenessGate = requestsTokenSavingClaim
-    ? tokenAccounting.acceptedAttempts.complete === true
-    : null;
-  const failedUsageCompletenessGate = requestsTokenSavingClaim
-    ? tokenAccounting.failedAttempts.complete === true
-    : null;
-  const allAttemptUsageCompletenessGate = requestsTokenSavingClaim
-    ? tokenAccounting.allAttempts.complete === true
-    : null;
-  const infrastructureFailureLedgerGate = requestsTokenSavingClaim
-    ? infrastructureFailureLedgerIssues.length === 0
-    : null;
-  const codexBaselineGate = requestsTokenSavingClaim ? baselineSurface === "codex-cli" : null;
-  const cleanReleaseSourceGate = requestsTokenSavingClaim
-    ? environment.source?.kind === "git-working-tree"
-      && environment.source.dirty === false
-      && /^[a-f0-9]{40,64}$/.test(environment.source.commit ?? "")
-    : null;
-  const hostReadinessHistory = environment.hostReadinessHistory ?? null;
-  const hostReadinessGate = requiresHostReadiness
-    ? hostReadinessHistory?.valid === true
-      && hostReadinessHistory?.ready === true
-      && hostReadinessHistory?.windowCoverage === "complete"
-    : null;
   const qualityNonInferior = candidate.scores.quality >= baseline.scores.quality;
   const pairedQualityEvidence = pairedQualityNoninferiorityEvidence({
     suite,
@@ -359,8 +320,8 @@ export function summarizeBenchmark({
   const efficiencyCategoryCoverage = completeCategoryCoverage(suite, completeScenarioFreshRatios);
   const efficiencyBandCoverageGate = suite.schemaVersion === 2 ? efficiencyCategoryCoverage.passed : true;
   const primaryEfficiencyEvidenceGate = Number.isInteger(minimumComparableEfficiencyScenarios)
-    ? primaryEfficiencyCompleteScenarios >= minimumComparableEfficiencyScenarios
-    : primaryEfficiencyCompleteScenarios >= 3;
+    ? primaryEfficiencyCompleteSamples >= minimumComparableEfficiencyScenarios
+    : primaryEfficiencyCompleteSamples >= 3;
   const primaryEfficiencyBandCoverageGate = suite.schemaVersion === 2
     ? primaryEfficiencyCategoryCoverage.passed
     : true;
@@ -549,6 +510,9 @@ export function summarizeBenchmark({
     && stabilityGate !== false
     && acceptedUsageCompletenessGate !== false
     && allAttemptUsageCompletenessGate !== false
+    && allAttemptPooledEfficiencyGate !== false
+    && campaignAccountingGate !== false
+    && fastServiceTierGate !== false
     && normalizedCostClaimConfigurationGate !== false
     && normalizedCostGates.passed !== false
     && outcomeEvidenceGate
@@ -592,6 +556,9 @@ export function summarizeBenchmark({
     && stabilityGate !== false
     && acceptedUsageCompletenessGate !== false
     && allAttemptUsageCompletenessGate !== false
+    && allAttemptPooledEfficiencyGate !== false
+    && campaignAccountingGate !== false
+    && fastServiceTierGate !== false
     && normalizedCostClaimConfigurationGate !== false
     && normalizedCostGates.passed !== false
     && outcomeEvidenceGate
@@ -627,6 +594,9 @@ export function summarizeBenchmark({
     requiresStability && !unknownInfrastructureUsageGate ? "unknown-infrastructure-usage" : null,
     requestsTokenSavingClaim && !acceptedUsageCompletenessGate ? "accepted-usage-completeness" : null,
     requestsTokenSavingClaim && !allAttemptUsageCompletenessGate ? "all-attempt-usage-completeness" : null,
+    requiresCampaignAccounting && !campaignAccountingGate ? "campaign-all-attempt-accounting" : null,
+    Number.isFinite(maximumAllAttemptPooledFreshTokenRatio) && !allAttemptPooledEfficiencyGate ? "all-attempt-net-efficiency" : null,
+    requiresFastServiceTier && !fastServiceTierGate ? "fast-execution-configuration-parity" : null,
     requestsTokenSavingClaim && !infrastructureFailureLedgerGate ? "infrastructure-failure-ledger" : null,
     requestsNormalizedCostClaim && !normalizedCostGates.evidenceGate ? "normalized-cost-evidence" : null,
     requestsNormalizedCostClaim && !normalizedCostGates.applicabilityGate ? "normalized-cost-pricing-applicability" : null,
@@ -664,6 +634,7 @@ export function summarizeBenchmark({
       primaryEfficiencyEstimand,
       repeats: minimumRepeats ?? null,
       freshTokenRatioUpper95: maximumFreshTokenRatioUpper95 ?? null,
+      allAttemptPooledFreshTokenRatio: maximumAllAttemptPooledFreshTokenRatio ?? null,
       bandFreshTokenRatio: maximumBandFreshTokenRatio ?? null,
       familyFreshTokenRatio: maximumFamilyFreshTokenRatio ?? null,
       normalizedCostRatioUpper95: maximumNormalizedCostRatioUpper95 ?? null,
@@ -680,8 +651,10 @@ export function summarizeBenchmark({
       hostReadinessHistory: requiresHostReadiness,
       requireFullSuite: requiresFullSuite,
       stableProviderWireSurface: requiresProviderWireSurface,
+      fastServiceTier: requiresFastServiceTier,
+      campaignAccounting: requiresCampaignAccounting,
       causalContextReceipt: requiresCausalContextReceipt,
-      canonicalProductionIdentityRequired: suite.id === "production-v1"
+      canonicalProductionIdentityRequired: canonicalProductionSuiteId(suite.id)
     },
     observed: {
       completeOutcomeScenarios,
@@ -693,6 +666,8 @@ export function summarizeBenchmark({
       pairedQualityFailures: pairedQualityEvidence.failures,
       repeats,
       freshTokenRatioUpper95: freshRatioConfidence95Raw?.upper ?? null,
+      allAttemptPooledFreshTokenRatio: allAttemptPooledEfficiency.ratio,
+      allAttemptPooledFreshTokenReductionPercent: allAttemptPooledEfficiency.reductionPercent,
       bandFreshTokenRatioFailures: freshTokenBandGate.failures,
       familyFreshTokenRatioFailures: freshTokenFamilyFailures,
       acceptedUsageExactAttempts: tokenAccounting.acceptedAttempts.exactAttempts,
@@ -703,6 +678,8 @@ export function summarizeBenchmark({
       normalizedCostFamilyFailures: normalizedCostGates.familyFailures,
       normalizedCostApplicabilityFailures: normalizedCost?.applicabilityFailures ?? [],
       primaryEfficiencyCompleteScenarios,
+      primaryEfficiencyCompleteSamples,
+      primaryEfficiencySampleUnit: matrixHierarchy ? suite.matrixContract.confidenceSampleUnit : "scenario-family",
       primaryEfficiencyRatio: rounded(primaryEfficiencyRatio, 4),
       primaryEfficiencyRatioUpper95: primaryEfficiencyRatioConfidence95Raw?.upper ?? null,
       durationRatio: rounded(durationRatio, 4),
@@ -724,9 +701,13 @@ export function summarizeBenchmark({
       providerWireRuns: candidateRuns.length,
       providerWireGroups: providerWireGroups.length,
       providerWireDriftGroups: providerWireDriftGroups.length,
+      fastExecutionConfigurationParity: serviceTierEvidence.executionConfigurationParityGate,
+      providerResponseServiceTierEvidence: serviceTierEvidence.providerResponseEvidenceGate,
       causalContextAvailableRuns: causalContextEvidence.currentAvailableRuns,
       causalContextRuns: causalContextEvidence.runs,
-      causalContextRequiredSchemaVersion: causalContextEvidence.requiredSchemaVersion
+      causalContextRequiredSchemaVersion: causalContextEvidence.requiredSchemaVersion,
+      campaignId: campaignEvidence?.campaignId ?? null,
+      campaignAccountingPassed: campaignAccountingGate
     }
   } : null;
   const baselineKey = surfaceReportKey(baselineSurface);
@@ -738,11 +719,21 @@ export function summarizeBenchmark({
     protocolPassed: protocol.passed,
     tokenClaimAllowed
   });
+  const matrixReport = hierarchicalMatrixReportSummary({ suite, matrixHierarchy, hierarchicalFreshRatioSample, confidenceScenarioRatios,
+    hierarchicalDurationRatioSample, confidenceScenarioDurationRatios, primaryEfficiencyScenarioRatios, primaryEfficiencyScenarioIds,
+    primaryEfficiencyFamilyIds, primaryEfficiencyCompleteScenarios, primaryUsesFixedWorkload, familyClusteredFailureAware,
+    familyClusteredFixedWorkload });
   return integrateCodexRelativeEfficiencyReport({
     schemaVersion: 2,
     measurementSchemaVersion: BENCHMARK_MEASUREMENT_SCHEMA_VERSION,
     runId,
-    suite: { id: suite.id, title: suite.title, schemaVersion: suite.schemaVersion, assurance: suite.assurance ?? null },
+    suite: {
+      id: suite.id,
+      title: suite.title,
+      schemaVersion: suite.schemaVersion,
+      assurance: suite.assurance ?? null,
+      ...matrixReport.suiteFields
+    },
     startedAt,
     completedAt,
     repeats,
@@ -762,12 +753,10 @@ export function summarizeBenchmark({
       baselineSurface,
       candidateSurface,
       purpose: claimEligibility.comparisonPurpose,
-      usageEstimator: "paired-geometric-mean-ratio",
-      durationEstimator: "paired-geometric-mean-ratio-clustered-by-scenario-family",
+      ...matrixReport.estimatorFields,
       failureAwareUsageEstimator: "total-comparable-attempt-fresh-tokens-per-resolved-outcome",
       failureAwareFamilyUsageEstimator: "geometric-mean-of-family-total-comparable-attempt-fresh-tokens-per-resolved-outcome-ratios",
-      fixedWorkloadUsageEstimator: "geometric-mean-of-family-total-exact-scheduled-workload-ratios",
-      fixedWorkloadEstimatorVersion: 1,
+      fixedWorkloadUsageEstimator: "geometric-mean-of-family-total-exact-scheduled-workload-ratios", fixedWorkloadEstimatorVersion: matrixReport.fixedWorkloadEstimatorVersion,
       pairedOutcomeScenarios: completeOutcomeScenarios,
       pairedSuccessfulRuns: pairs.length,
       pairedUsageRuns: tokenPairs.length,
@@ -805,6 +794,15 @@ export function summarizeBenchmark({
         [candidateKey]: rounded(candidateFreshPerResolvedOutcome, 2)
       },
       failureAwareFreshTokenRatio: rounded(failureAwareFreshTokenRatio, 4),
+      ...(releaseGate.efficiencyProtocol === undefined ? {} : { efficiencyProtocol: {
+        version: releaseGate.efficiencyProtocol,
+        allAttemptPooledRatioRaw: allAttemptPooledEfficiency.ratioRaw, familyRatioUpper95Raw: primaryEfficiencyRatioConfidence95Raw?.upper ?? null,
+        numericGatesPassed: allAttemptPooledEfficiencyGate === true && primaryEfficiencyConfidenceGate === true
+      } }),
+      allAttemptPooledEfficiency,
+      allAttemptPooledEfficiencyGate,
+      campaignEvidence,
+      campaignAccountingGate,
       failureAwareFamilyFreshTokenRatio: rounded(familyClusteredFailureAware.ratio, 4),
       failureAwareFamilyFreshTokenRatioConfidence95: familyClusteredFailureAware.confidence95,
       failureAwareFamilyFreshTokenRatioConfidence95Raw: familyClusteredFailureAware.confidence95Raw,
@@ -813,7 +811,8 @@ export function summarizeBenchmark({
         expectedScenarioFamilies: familyClusteredFailureAware.expectedScenarioFamilies,
         usableScenarioFamilies: familyClusteredFailureAware.usableScenarioFamilies,
         sampleUnit: familyClusteredFailureAware.sampleUnit,
-        scenarioIds: familyClusteredFailureAware.scenarioIds
+        scenarioIds: familyClusteredFailureAware.scenarioIds,
+        ...matrixReport.failureAwareCoverageFields
       },
       failureAwareFamilyRatios: familyClusteredFailureAware.families,
       fixedWorkloadFamilyFreshTokenRatio: rounded(familyClusteredFixedWorkload.ratio, 4),
@@ -830,30 +829,24 @@ export function summarizeBenchmark({
         outcomeConditioning: familyClusteredFixedWorkload.outcomeConditioning,
         aggregation: familyClusteredFixedWorkload.aggregation,
         attemptPolicy: familyClusteredFixedWorkload.attemptPolicy,
-        scenarioIds: familyClusteredFixedWorkload.scenarioIds
+        scenarioIds: familyClusteredFixedWorkload.scenarioIds,
+        ...matrixReport.fixedWorkloadCoverageFields
       },
       fixedWorkloadFamilyRatios: familyClusteredFixedWorkload.families,
       allSuccessfulPairsFreshTokenRatio: rounded(allSuccessfulPairsFreshRatio, 4),
+      conditionalBothPassFreshUsage: conditionalBothPassFreshUsage({ suite, repeats, allPairs, pairs, tokenPairs,
+        baselineRuns, candidateRuns, allSuccessfulPairsFreshRatio }),
       freshTokenRatio: rounded(freshRatio, 4),
       freshTokenRatioRaw: freshRatio,
       freshTokenRatioConfidence95: freshRatioConfidence95,
       freshTokenRatioConfidence95Raw: freshRatioConfidence95Raw,
-      freshTokenRatioSample: {
-        sampleUnit: "scenario-family",
-        scenarioCount: confidenceScenarioRatios.length,
-        scenarioIds: confidenceScenarioRatios.map((item) => item.scenarioId)
-      },
+      freshTokenRatioSample: matrixReport.freshTokenRatioSample,
       primaryEfficiencyEstimand,
       primaryEfficiencyRatio: rounded(primaryEfficiencyRatio, 4),
       primaryEfficiencyRatioRaw: primaryEfficiencyRatio,
       primaryEfficiencyRatioConfidence95,
       primaryEfficiencyRatioConfidence95Raw,
-      primaryEfficiencySample: {
-        sampleUnit: "scenario-family",
-        scenarioCount: primaryEfficiencyCompleteScenarios,
-        scenarioIds: primaryEfficiencyScenarioRatios.map((item) => item.scenarioId),
-        outcomeConditioning: primaryUsesFixedWorkload ? "none" : "resolved-outcome-conditioned"
-      },
+      primaryEfficiencySample: matrixReport.primaryEfficiencySample,
       primaryEfficiencyDeltaPercent: Number.isFinite(primaryEfficiencyRatio)
         ? rounded((primaryEfficiencyRatio - 1) * 100, 2)
         : null,
@@ -862,6 +855,7 @@ export function summarizeBenchmark({
       durationRatioRaw: durationRatio,
       durationRatioConfidence95,
       durationRatioConfidence95Raw,
+      ...matrixReport.durationRatioSampleFields,
       durationDeltaPercent: Number.isFinite(durationRatio) ? rounded((durationRatio - 1) * 100, 2) : null,
       costRatio: rounded(costRatio, 4),
       costDeltaPercent: Number.isFinite(costRatio) ? rounded((costRatio - 1) * 100, 2) : null,
@@ -881,6 +875,8 @@ export function summarizeBenchmark({
       candidateContinuityFailures,
       comparisonProtocolGate: protocol,
       providerWireSurfaceGate,
+      fastServiceTierGate,
+      serviceTierEvidence,
       causalContextEvidenceGate,
       causalContextEvidence,
       providerWireEvidence: {
@@ -988,6 +984,10 @@ export function summarizeBenchmark({
                                         ? "accepted-usage-completeness-gate-failed"
                                         : requestsTokenSavingClaim && allAttemptUsageCompletenessGate === false
                                           ? "all-attempt-usage-completeness-gate-failed"
+                                        : Number.isFinite(maximumAllAttemptPooledFreshTokenRatio) && allAttemptPooledEfficiencyGate === false
+                                          ? "all-attempt-net-efficiency-gate-failed"
+                                        : requiresFastServiceTier && fastServiceTierGate === false
+                                          ? "fast-execution-configuration-parity-gate-failed"
                                         : requestsNormalizedCostClaim && normalizedCostClaimConfigurationGate === false
                                           ? "normalized-cost-configuration-gate-failed"
                                           : requestsNormalizedCostClaim && normalizedCostGates.applicabilityGate === false

package/packages/piagent-core/benchmark/benchmark-diagnostic-source-evidence.js

@@ -0,0 +1,40 @@
+import { candidateProvenance } from './benchmark-candidate.js';
+import { deriveDiagnosticCandidateEntries, DIAGNOSTIC_TREATMENT } from './benchmark-diagnostic-treatment.js';
+
+// Prove the source-to-treatment transformation from actual bytes.
+export function diagnosticS0SourceEvidence({ source, entries, derivation, candidate }) {
+  if (source?.kind !== 'git-working-tree' || typeof source.dirty !== 'boolean'
+    || !/^[a-f0-9]{40,64}$/.test(source.commit ?? '')) throw new Error('Invalid source identity');
+  const observedSource = candidateProvenance(entries);
+  const derived = deriveDiagnosticCandidateEntries(entries, { piagentTreatment: DIAGNOSTIC_TREATMENT, measurementOnly: true });
+  const observedCandidate = candidateProvenance(derived.entries);
+  const expected = { ...derived.derivation, sourceCandidateProvenance: observedSource,
+    derivedCandidateProvenance: observedCandidate };
+  if (JSON.stringify(derivation) !== JSON.stringify(expected)
+    || JSON.stringify(candidate) !== JSON.stringify(observedCandidate)) {
+    throw new Error('Diagnostic S0 source or fixed policy derivation changed');
+  }
+  return { kind: source.kind, commit: source.commit, clean: !source.dirty,
+    treeDigest: candidate.contentDigest, treeAlgorithm: candidate.algorithm,
+    conformanceSourceDigest: observedSource.contentDigest,
+    conformanceScope: 'source-runtime; diagnostic-policy-runtime-qualified-separately',
+    treatmentDerivation: expected };
+}
+
+export function validDiagnosticS0SourceEvidence(value) {
+  const hash = item => typeof item === 'string' && /^[a-f0-9]{64}$/.test(item);
+  const d = value?.treatmentDerivation, change = d?.changes?.[0];
+  return value?.kind === 'git-working-tree' && typeof value.clean === 'boolean'
+    && /^[a-f0-9]{40,64}$/.test(value.commit ?? '')
+    && value.conformanceScope === 'source-runtime; diagnostic-policy-runtime-qualified-separately'
+    && hash(value.conformanceSourceDigest) && hash(value.treeDigest)
+    && value.conformanceSourceDigest !== value.treeDigest
+    && d?.schemaVersion === 1 && d.treatment === DIAGNOSTIC_TREATMENT
+    && d.transform === 'installed-base-policy-acceptance-diagnostic-v1'
+    && d.changes?.length === 1
+    && change.path === 'packages/piagent-core/policies/base-policy.json'
+    && hash(change.beforeSha256) && hash(change.afterSha256) && change.beforeSha256 !== change.afterSha256
+    && d.sourceCandidateProvenance?.contentDigest === value.conformanceSourceDigest
+    && d.derivedCandidateProvenance?.contentDigest === value.treeDigest
+    && d.derivedCandidateProvenance?.algorithm === value.treeAlgorithm;
+}

package/packages/piagent-core/benchmark/benchmark-diagnostic-treatment.js

@@ -0,0 +1,104 @@
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+
+export const DIAGNOSTIC_TREATMENT = "acceptance-diagnostic";
+export const DIAGNOSTIC_POLICY_PATH = "packages/piagent-core/policies/base-policy.json";
+
+export function deriveDiagnosticCandidateEntries(entries, options = {}) {
+  if (options.piagentTreatment !== DIAGNOSTIC_TREATMENT) return { entries };
+  if (options.measurementOnly !== true) throw new Error("Diagnostic candidate derivation requires --measurement-only");
+  const policyEntries = entries.filter(entry => entry.path === DIAGNOSTIC_POLICY_PATH);
+  if (policyEntries.length !== 1 || policyEntries[0].kind !== "regular") {
+    throw new Error("Diagnostic candidate requires one regular base policy entry");
+  }
+  const entry = policyEntries[0], policy = JSON.parse(entry.payload.toString("utf8"));
+  if (!policy.finalGate || typeof policy.finalGate !== "object" || Array.isArray(policy.finalGate)) {
+    throw new Error("Diagnostic candidate requires a valid finalGate policy");
+  }
+  const prior = policy.finalGate.acceptanceProofMode;
+  if (prior !== undefined && prior !== "enforce") {
+    throw new Error("Diagnostic derivation requires an enforce source policy");
+  }
+  policy.finalGate.acceptanceProofMode = "diagnostic";
+  const payload = Buffer.from(`${JSON.stringify(policy, null, 2)}\n`);
+  const hash = value => crypto.createHash("sha256").update(value).digest("hex");
+  return {
+    entries: entries.map(value => value === entry ? { ...entry, payload } : value),
+    derivation: {
+      schemaVersion: 1, treatment: DIAGNOSTIC_TREATMENT,
+      transform: "installed-base-policy-acceptance-diagnostic-v1",
+      changes: [{ path: DIAGNOSTIC_POLICY_PATH, beforeSha256: hash(entry.payload), afterSha256: hash(payload) }]
+    }
+  };
+}
+
+export function assertDiagnosticBenchmarkMatrix(options, { builtInId, fullMatrix, expectedSessions }) {
+  if (options.piagentTreatment !== DIAGNOSTIC_TREATMENT) return;
+  const failedReplay = options.failedAttemptsOnly === true
+    && options.replaySource?.selection === "failed-attempts"
+    && options.replaySource?.evidenceComplete === true
+    && options.replaySource?.originalAttemptCount === 108
+    && options.replayRuns?.length > 0
+    && options.replayRuns.length === options.replaySource.selectedAttempts;
+  if (builtInId !== "production-v3" || (!fullMatrix && !failedReplay) || expectedSessions !== 108
+    || options.codexMode !== "controlled" || options.measurementOnly !== true) {
+    throw new Error("Diagnostic acceptance requires the complete production-v3 controlled 108-session measurement matrix or a bound failed-attempt-only diagnostic replay");
+  }
+}
+
+// Policy is part of the immutable candidate, never an inherited environment override.
+export function benchmarkAcceptancePolicyBinding(installedRoot, options) {
+  const file = path.join(installedRoot, DIAGNOSTIC_POLICY_PATH);
+  if (!fs.lstatSync(file).isFile() || fs.realpathSync(file) !== path.resolve(file)) {
+    throw new Error("Benchmark acceptance policy must be a regular file in the candidate");
+  }
+  const bytes = fs.readFileSync(file);
+  const policy = JSON.parse(bytes.toString("utf8"));
+  const diagnostic = policy.finalGate?.acceptanceProofMode === "diagnostic";
+  const selected = options.piagentTreatment === DIAGNOSTIC_TREATMENT;
+  const installedDefault = diagnostic && !selected;
+  if (diagnostic !== selected && !installedDefault) {
+    throw new Error("Benchmark treatment does not match the installed acceptance proof policy");
+  }
+  if (!diagnostic) return null;
+  if (!installedDefault && options.measurementOnly !== true) {
+    throw new Error("Diagnostic acceptance policy requires --measurement-only");
+  }
+  return {
+    mode: "diagnostic", ...(installedDefault ? { origin: "installed-release-policy" } : {}),
+    policyPath: DIAGNOSTIC_POLICY_PATH,
+    policySha256: crypto.createHash("sha256").update(bytes).digest("hex"),
+    pendingProof: "retained", qualityClaim: "withheld"
+  };
+}
+
+export function validDiagnosticPolicyBinding(binding) {
+  return binding?.mode === "diagnostic" && binding.policyPath === DIAGNOSTIC_POLICY_PATH
+    && (binding.origin === undefined || binding.origin === "installed-release-policy")
+    && /^[a-f0-9]{64}$/.test(binding.policySha256)
+    && binding.pendingProof === "retained" && binding.qualityClaim === "withheld";
+}
+
+export function assertDiagnosticCandidateDerivation(binding, derivation, candidate) {
+  if (!binding) return;
+  if (!validDiagnosticPolicyBinding(binding)) {
+    throw new Error("Diagnostic candidate derivation does not match frozen source, policy and candidate identities");
+  }
+  if (binding.origin === "installed-release-policy") {
+    if (derivation !== undefined) throw new Error("Installed diagnostic policy cannot claim a treatment derivation");
+    return;
+  }
+  const change = derivation?.changes?.[0];
+  if (derivation?.schemaVersion !== 1
+    || derivation.treatment !== DIAGNOSTIC_TREATMENT
+    || derivation.transform !== "installed-base-policy-acceptance-diagnostic-v1"
+    || derivation.changes.length !== 1 || change.path !== binding.policyPath
+    || change.afterSha256 !== binding.policySha256 || !/^[a-f0-9]{64}$/.test(change.beforeSha256)
+    || change.beforeSha256 === change.afterSha256
+    || !/^[a-f0-9]{64}$/.test(derivation.sourceCandidateProvenance?.contentDigest)
+    || derivation.sourceCandidateProvenance.contentDigest === candidate?.contentDigest
+    || JSON.stringify(derivation.derivedCandidateProvenance) !== JSON.stringify(candidate)) {
+    throw new Error("Diagnostic candidate derivation does not match frozen source, policy and candidate identities");
+  }
+}