@piagent/platform 1.6.1 → 1.7.1

package/scripts/benchmark-runner-finalization.mjs

@@ -7,8 +7,12 @@ import {
   summarizeBenchmark
 } from "../packages/piagent-core/benchmark/benchmark-core.js";
 import { codexModelName } from "../packages/piagent-core/benchmark/benchmark-codex.js";
+import { publicProductionBenchmarkCampaignEvidence } from "../packages/piagent-core/benchmark/benchmark-campaign.js";
 import { benchmarkTrustChecklist } from "../packages/piagent-core/benchmark/benchmark-matrix.js";
-import { applyBenchmarkClaimRestrictions } from "../packages/piagent-core/benchmark/benchmark-claim-restrictions.js";
+import { attachBenchmarkGoalAssessment } from "../packages/piagent-core/benchmark/benchmark-goal-report.js";
+import { renderBenchmarkMarkdown } from "../packages/piagent-core/benchmark/benchmark-report.js";
+import { adjudicateProductionV3Report, applyBenchmarkClaimRestrictions,
+  productionV3FatalMeasurementEvidence } from "../packages/piagent-core/benchmark/benchmark-claim-restrictions.js";
 import { piagentTreatment } from "../packages/piagent-core/benchmark/benchmark-runtime.js";
 import { cleanupBenchmarkPiRuntimeHome } from "../packages/piagent-core/benchmark/benchmark-pi-home.js";
 import { benchmarkSuiteCoverage } from "../packages/piagent-core/benchmark/benchmark-runner-policy.js";
@@ -38,6 +42,66 @@ import {
 } from "../packages/piagent-core/benchmark/benchmark-record-validation.js";
 import { recoverOrphanedBenchmarkAttempts } from "../packages/piagent-core/benchmark/benchmark-resume-recovery.js";
 import { benchmarkResumeCommand, benchmarkRunKey } from "./benchmark-runner-support.mjs";
+import { deferBenchmarkBudgetPublication } from "./benchmark-budget-publication.mjs";
+
+export function benchmarkReportExecutionMode({ options = {}, manifest = {} } = {}) {
+  const measurementOnly = options.measurementOnly === true || manifest.measurementOnly === true;
+  return { measurementOnly, executionMode: measurementOnly ? "measurement-only" : "release-gated" };
+}
+
+export function finalizeProductionCampaignClaimOutcome({ productionCampaign, manifest, report, runRoot,
+  verifiedLedgerRecords }) {
+  if (!productionCampaign) return null;
+  const measurementOnly = manifest.measurementOnly === true || report.environment?.measurementOnly === true || report.measurementOnly === true;
+  const productionV3Release = !measurementOnly && report.suite?.id === "production-v3";
+  if (measurementOnly) applyBenchmarkClaimRestrictions(report, { measurementOnly: true, surfaces: [] });
+  else if (productionV3Release) adjudicateProductionV3Report(report, {
+    expectedOrder: manifest.order,
+    expectedLedger: manifest.ledger,
+    verifiedLedgerRecords
+  });
+  if (productionV3Release && report.verdict?.status !== "PASS_VALID") {
+    report.comparison.tokenClaimAllowed = false;
+    report.comparison.tokenClaimUnavailableReason = `production-v3-verdict:${report.verdict?.status ?? "unavailable"}`;
+  }
+  const allowed = report.comparison?.tokenClaimAllowed === true
+    && (!productionV3Release || report.verdict?.status === "PASS_VALID");
+  const verdict = /^[a-z0-9._-]{1,120}$/i.test(String(report.verdict?.status ?? ""))
+    ? report.verdict.status
+    : "unavailable";
+  // The budgeted parent, not the core, owns the final campaign transition.
+  if (!manifest.budgetControl) manifest.campaignEvidence = productionCampaign.finalizeClaim({
+    allowed,
+    reason: measurementOnly ? "measurement-only-completed-no-release-claim"
+      : allowed ? "release-token-claim-allowed" : `release-token-claim-not-allowed:${verdict}`
+  });
+  writeBenchmarkRunManifest(runRoot, manifest);
+  const publicEvidence = publicProductionBenchmarkCampaignEvidence(manifest.campaignEvidence);
+  report.environment.campaignEvidence = publicEvidence;
+  report.comparison.campaignEvidence = publicEvidence;
+  return publicEvidence;
+}
+
+export function finalizeProductionCampaignTerminalNoClaim({ productionCampaign, manifest, runRoot, terminalStop, runs }) {
+  if (!productionCampaign) return null;
+  manifest.campaignEvidence = productionCampaign.finalizeTerminalNoClaim({
+    reason: `terminal-stop:${terminalStop.reason}`,
+    runs
+  });
+  writeBenchmarkRunManifest(runRoot, manifest);
+  return manifest.campaignEvidence;
+}
+
+export function rollbackProductionCampaignPublication({ productionCampaign, manifest, runRoot, error }) {
+  for (const name of ["report.html", "report.md", "summary.txt", "report.json"]) {
+    try { fs.rmSync(path.join(runRoot, name), { force: true }); } catch { /* Best effort; campaign authority still fails closed. */ }
+  }
+  if (!productionCampaign) return null;
+  const code = /^[A-Z0-9_:-]{1,120}$/.test(String(error?.code ?? "")) ? error.code : "UNCLASSIFIED";
+  manifest.campaignEvidence = productionCampaign.invalidateClaimPublication({ reason: `publication-failed:${code}` });
+  writeBenchmarkRunManifest(runRoot, manifest);
+  return manifest.campaignEvidence;
+}
 
 export function finalizeBenchmarkRun(context) {
   const {
@@ -68,6 +132,7 @@ export function finalizeBenchmarkRun(context) {
     preservePiRuntime,
     productionAllStageBoundaries,
     productionHostReadinessPolicy,
+    productionCampaign,
     productionSpendControlled,
     rootSeed,
     rootSeedDigest,
@@ -94,6 +159,7 @@ export function finalizeBenchmarkRun(context) {
       runRoot,
       productionSpendControlled,
       completedRuns: interruptedCompletedRuns,
+      verificationRequired: Boolean(manifest.verificationPlan),
       stageControl: manifest.stageControl,
       stageBoundaries: productionAllStageBoundaries
     });
@@ -104,21 +170,34 @@ export function finalizeBenchmarkRun(context) {
   }
   if (fatalRunError) {
     const provenanceStamp = finalizationReceipt?.stamp ?? fatalExecutionReceipt?.stamp ?? executionGuard.stamp("fatal", piRuntimeHome ? [piRuntimeHome] : []);
+    const invalidMeasurement = fatalRunError.code === "BENCHMARK_INVALID_MEASUREMENT"
+      || (productionSpendControlled && suite.id === "production-v3" && options.measurementOnly !== true);
+    const invalidMeasurementEvidence = invalidMeasurement ? productionV3FatalMeasurementEvidence(fatalRunError) : {};
     if (!preservePiRuntime) cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome);
     detachPiRuntimeHome();
     recoverOrphanedBenchmarkAttempts({ runRoot, manifest, fullOrder, completedKeys: new Set(runs.map(benchmarkRunKey)) });
     cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
     const provenanceFailure = writeBenchmarkAbort(runRoot, { runId, completedRuns: runs.filter(completedBenchmarkRecord).length, expectedRuns: fullOrder.length }, fatalRunError, {
       ledger: ledgerBinding,
-      provenanceStamp
+      provenanceStamp,
+      ...invalidMeasurementEvidence
     });
-    process.stderr.write(provenanceFailure
+    process.stderr.write(invalidMeasurement
+      ? `Benchmark stopped after a measurement-invalidating outcome. Partial ledger: ${ledgerPath}\n`
+      : provenanceFailure
       ? `Benchmark aborted because candidate provenance changed. Partial ledger: ${ledgerPath}\n`
       : `Benchmark aborted after an infrastructure error. Partial ledger: ${ledgerPath}\n`);
     process.exitCode = 1;
     return;
   }
   if (terminalStop) {
+    finalizeProductionCampaignTerminalNoClaim({
+      productionCampaign,
+      manifest,
+      runRoot,
+      terminalStop,
+      runs: runs.filter(completedBenchmarkRecord)
+    });
     cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome);
     detachPiRuntimeHome();
     cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
@@ -159,6 +238,7 @@ export function finalizeBenchmarkRun(context) {
       productionSpendControlled,
       completedRuns: completedRuns.length,
       stageControl: manifest.stageControl,
+      verificationRequired: Boolean(manifest.verificationPlan),
       stageBoundaries: productionAllStageBoundaries
     });
     const paused = {
@@ -182,6 +262,7 @@ export function finalizeBenchmarkRun(context) {
       baselineSurface: comparison.baselineSurface,
       requestedModel: options.model,
       requestedThinking: options.thinking,
+      requestedServiceTier: options.serviceTier,
       suite,
       manifest,
       hostReadinessPolicy: productionHostReadinessPolicy,
@@ -211,6 +292,11 @@ export function finalizeBenchmarkRun(context) {
 
   cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome);
   detachPiRuntimeHome();
+  if (productionCampaign) {
+    manifest.campaignEvidence = productionCampaign.sealForClaim(completedRuns);
+    writeBenchmarkRunManifest(runRoot, manifest);
+  }
+  const publicCampaignEvidence = publicProductionBenchmarkCampaignEvidence(manifest.campaignEvidence ?? null);
   const report = summarizeBenchmark({
     suite,
     canonicalProductionSuite,
@@ -219,6 +305,8 @@ export function finalizeBenchmarkRun(context) {
     completedAt: new Date().toISOString(),
     repeats: options.repeats,
     environment: {
+      runId,
+      ...benchmarkReportExecutionMode({ options, manifest }),
       platformVersion: packageVersion,
       suiteDigest,
       variantRootSeed: suite.scenarios.some((scenario) => scenario.variantGenerator) ? rootSeed : null,
@@ -228,7 +316,11 @@ export function finalizeBenchmarkRun(context) {
       profile: suite.profile,
       requestedModel: options.model ?? null,
       requestedThinking: options.thinking ?? null,
+      requestedServiceTier: options.serviceTier ?? null,
+      ...(manifest.verificationPlan ? { independentVerification: manifest.verificationPlan.identity } : {}),
       piagentTreatment: piagentTreatment(options.piagentTreatment),
+      ...(manifest.acceptancePolicyBinding ? { acceptancePolicyBinding: manifest.acceptancePolicyBinding } : {}),
+      ...(manifest.treatmentDerivation ? { treatmentDerivation: manifest.treatmentDerivation } : {}),
       treatmentBaseline: lifecycles.length === 1 && lifecycles[0] === "steady-state"
         ? options.surfaces.includes("codex-cli")
           ? "piagent-initialized-and-onboarded; codex-clean-fixture"
@@ -239,6 +331,7 @@ export function finalizeBenchmarkRun(context) {
       piVersion: runtime.piVersion,
       codexVersion: runtime.codexVersion ?? null,
       codexMode: options.surfaces.includes("codex-cli") ? options.codexMode : null,
+      codexBaseline: options.surfaces.includes("codex-cli") ? options.codexBaseline : null,
       codexAuth: runtime.codexAuth ?? null,
       codexIsolation: options.surfaces.includes("codex-cli")
         ? options.codexMode === "controlled" ? "per-session-temporary-home" : "operator-home"
@@ -268,46 +361,66 @@ export function finalizeBenchmarkRun(context) {
       suiteIdentity,
       runtimeCommands,
       configurationDigest,
+      providerFreeConfigurationDigest: manifest.providerFreeConfigurationDigest ?? null,
       hostReadinessHistory,
       environmentPolicy,
       runtimeDependencies: bootstrapMetadata?.runtimeDependencies ?? null,
+      webUiAssets: manifest.webUiAssets ?? null,
       providerFreeEvidence: manifest.providerFreeEvidence ?? null,
+      campaignEvidence: publicCampaignEvidence,
       assuranceEvidence
     },
+    allAttemptTokenAccounting: manifest.campaignEvidence?.allAttempts,
     runs: completedRuns,
     ...comparison
   });
   report.ledger = ledgerBinding;
-  applyBenchmarkClaimRestrictions(report, { tokenReason: manifest.tokenClaimsUnavailableReason, replaySource: options.replaySource, codexMode: options.codexMode, surfaces: options.surfaces });
-  report.trustChecklist = benchmarkTrustChecklist(report);
-  const text = renderBenchmarkText(report);
+  applyBenchmarkClaimRestrictions(report, { tokenReason: manifest.tokenClaimsUnavailableReason, replaySource: options.replaySource, codexMode: options.codexMode, codexBaseline: options.codexBaseline, surfaces: options.surfaces, measurementOnly: report.environment.measurementOnly });
   const reportLedger = inspectBenchmarkLedger(ledgerPath);
   assertBenchmarkLedgerBinding(ledgerBinding, reportLedger.binding, "benchmark report ledger");
-  validateBenchmarkLedgerPrefix(reportLedger.records, fullOrder, (record, index, expected) => expectedBenchmarkRecord(record, index, expected, runId, suite, configurationDigest));
-  writePrivate(path.join(runRoot, "report.html"), renderBenchmarkHtml(report));
-  writePrivate(path.join(runRoot, "summary.txt"), text);
+  validateBenchmarkLedgerPrefix(reportLedger.records, fullOrder, (record, index, expected) => expectedBenchmarkRecord(record, index, expected, runId, suite, configurationDigest, manifest.verificationPlan?.identity));
   cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
   const prepublishReceipt = executionGuard.receipt("prepublish");
   const prepublishError = prepublishReceipt.error;
   if (prepublishError) {
-    writeBenchmarkAbort(runRoot, { runId, completedRuns: completedRuns.length, expectedRuns: fullOrder.length }, prepublishError, { ledger: ledgerBinding, provenanceStamp: prepublishReceipt.stamp });
+    const invalidMeasurementEvidence = productionSpendControlled && suite.id === "production-v3"
+      && options.measurementOnly !== true ? productionV3FatalMeasurementEvidence(prepublishError) : {};
+    writeBenchmarkAbort(runRoot, { runId, completedRuns: completedRuns.length, expectedRuns: fullOrder.length }, prepublishError, {
+      ledger: ledgerBinding,
+      provenanceStamp: prepublishReceipt.stamp,
+      ...invalidMeasurementEvidence
+    });
     throw prepublishError;
   }
-  for (const marker of ["paused.json", "stage-diagnostic.json", "interrupted.json", "aborted.json", "stopped.json"]) fs.rmSync(path.join(runRoot, marker), { force: true });
-  writePrivateAtomic(path.join(runRoot, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
-  process.stdout.write(options.json ? `${JSON.stringify(report, null, 2)}\n` : text);
-  process.stdout.write(`Reports: ${runRoot}\n`);
-  const productionTokenClaimFailed = canonicalProductionSuite
-    && suite.releaseGate?.requireEfficiencyClaim === true
-    && report.comparison.tokenClaimAllowed !== true;
-  if (
-    report.comparison.qualityGate === false
-    || report.comparison.safetyGate === false
-    || report.comparison.reliabilityGate === false
-    || report.comparison.qualityNonInferior === false
-    || report.comparison.workflowGate === false
-    || report.comparison.categoryGate === false
-    || report.comparison.suiteGate?.passed === false
-    || productionTokenClaimFailed
-  ) process.exitCode = 1;
+  let text;
+  try {
+    finalizeProductionCampaignClaimOutcome({ productionCampaign, manifest, report, runRoot,
+      verifiedLedgerRecords: reportLedger.records });
+    attachBenchmarkGoalAssessment({ report, suite, manifest, verifiedLedgerRecords: reportLedger.records });
+    report.trustChecklist = benchmarkTrustChecklist(report);
+    const productionTokenClaimFailed = canonicalProductionSuite
+      && suite.releaseGate?.requireEfficiencyClaim === true
+      && report.comparison.tokenClaimAllowed !== true;
+    const productionV3VerdictFailed = suite.id === "production-v3"
+      && report.environment.measurementOnly !== true && report.verdict?.status !== "PASS_VALID";
+    if (report.comparison.qualityGate === false || report.comparison.safetyGate === false
+      || report.comparison.reliabilityGate === false || report.comparison.qualityNonInferior === false
+      || report.comparison.workflowGate === false || report.comparison.categoryGate === false
+      || report.comparison.suiteGate?.passed === false || productionTokenClaimFailed
+      || productionV3VerdictFailed) process.exitCode = 1;
+    deferBenchmarkBudgetPublication({ report, manifest, runRoot });
+    text = renderBenchmarkText(report);
+    writePrivate(path.join(runRoot, "report.html"), renderBenchmarkHtml(report));
+    writePrivate(path.join(runRoot, "report.md"), renderBenchmarkMarkdown(report));
+    writePrivate(path.join(runRoot, "summary.txt"), text);
+    for (const marker of ["paused.json", "stage-diagnostic.json", "interrupted.json", "aborted.json", "stopped.json"]) fs.rmSync(path.join(runRoot, marker), { force: true });
+    writePrivateAtomic(path.join(runRoot, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
+  } catch (error) {
+    rollbackProductionCampaignPublication({ productionCampaign, manifest, runRoot, error });
+    throw error;
+  }
+  try {
+    process.stdout.write(options.json ? `${JSON.stringify(report, null, 2)}\n` : text);
+    process.stdout.write(`Reports: ${runRoot}\n`);
+  } catch { /* Canonical report and campaign outcome are already durable. */ }
 }

package/scripts/benchmark-runner-provider-boundary.mjs

@@ -0,0 +1,44 @@
+import { assertCodexRuntimeCredential } from "../packages/piagent-core/benchmark/benchmark-runtime.js";
+import { exactBenchmarkAttemptUsage } from "../packages/piagent-core/benchmark/benchmark-usage.js";
+import { registeredRuntimeVerifierReceipt } from "./benchmark-runner-configuration.mjs";
+
+function boundaryFailure(message) {
+  return Object.assign(new Error(message), { exitCode: 1 });
+}
+
+export function bindRegisteredRuntimeVerifiers({ registeredMeasurement, registeredMeasurementRun,
+  bootstrapMetadata, runtimeCommands, verificationPlan }) {
+  if (registeredMeasurementRun && (runtimeCommands.docker?.resolvedPath !== verificationPlan?.dockerCommand?.path
+    || runtimeCommands.docker?.contentDigest !== verificationPlan?.dockerCommand?.sha256)) {
+    throw boundaryFailure("Registered Docker command identity does not match the signed verification plans");
+  }
+  return registeredRuntimeVerifierReceipt({ registeredMeasurement, bootstrapMetadata, runtimeCommands });
+}
+
+export function createBenchmarkProviderBoundaryGuard({ executionGuard, registeredMeasurement,
+  registeredMeasurementRun, registeredRuntimeVerifiers, bootstrapMetadata, runtimeCommands,
+  codexRuntime, piRuntimeHome, scenarioId, surface, repeat, infrastructureAttempt }) {
+  return stage => {
+    const error = executionGuard.check(
+      `${stage}:${scenarioId}:${surface}:r${repeat}:attempt${infrastructureAttempt}`, [piRuntimeHome]);
+    if (error) throw error;
+    if (registeredMeasurement) {
+      const current = registeredRuntimeVerifierReceipt({ registeredMeasurement, bootstrapMetadata, runtimeCommands });
+      if (JSON.stringify(current) !== JSON.stringify(registeredRuntimeVerifiers)) {
+        throw boundaryFailure("Registered runtime verifier bindings changed at a provider boundary");
+      }
+    }
+    if (surface === "codex-cli") {
+      assertCodexRuntimeCredential(codexRuntime, { required: registeredMeasurementRun });
+    }
+  };
+}
+
+export function forceTokenUnavailableForPostSessionAssetError({ sessionEvidence, record }) {
+  const exactPredispatchUsage = sessionEvidence?.fatalProviderBoundaryError instanceof Error
+    && sessionEvidence.fatalProviderBoundaryPhase === "pre-dispatch"
+    && record?.providerBoundaryPhase === "pre-dispatch"
+    && record?.usageStatus === "measured-but-unaccepted"
+    && exactBenchmarkAttemptUsage(record.usage, record.usageStatus);
+  return !exactPredispatchUsage;
+}

package/scripts/benchmark-runner-schedule.mjs

@@ -0,0 +1,256 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createCodexRuntime } from "../packages/piagent-core/benchmark/benchmark-runtime.js";
+import { withBenchmarkPiCredentialWriteback, resetBenchmarkPiRuntimeEphemeralState } from "../packages/piagent-core/benchmark/benchmark-pi-home.js";
+import { benchmarkInfrastructureFailureDisposition, benchmarkRunnerErrorRecord, recoveredBenchmarkAttemptDisposition } from "../packages/piagent-core/benchmark/benchmark-runner-policy.js";
+import { appendPrivateJsonl, retainWorkspaceForensics, safeInfrastructureDiagnostic, writeBenchmarkRunManifest } from "../packages/piagent-core/benchmark/benchmark-forensics.js";
+import { completedBenchmarkRecord } from "../packages/piagent-core/benchmark/benchmark-record-validation.js";
+import { pairedOutcomeFloorStop } from "../packages/piagent-core/benchmark/benchmark-stop-policy.js";
+import { resolveBenchmarkSuiteEntry } from "../packages/piagent-core/benchmark/benchmark-suite-runtime.js";
+import { appendBenchmarkLedger } from "../packages/piagent-core/benchmark/benchmark-ledger.js";
+import { clearRecoveredBenchmarkAttempts, persistUnacceptedBenchmarkAttempt, promoteMeasuredBenchmarkRecord, stageMeasuredBenchmarkRecord } from "../packages/piagent-core/benchmark/benchmark-resume-recovery.js";
+import { productionMeasurementInvalidatingOutcomeError } from "./benchmark-runner-completion.mjs";
+import { benchmarkRunKey, formatDuration, samePairedBlock } from "./benchmark-runner-support.mjs";
+import { createBudgetProviderCallbacks } from "./benchmark-budget-runtime.mjs";
+import { registeredBenchmarkScopedSessionRequired } from "./benchmark-scoped-session-factory.mjs";
+import { createBenchmarkProviderBoundaryGuard, forceTokenUnavailableForPostSessionAssetError } from "./benchmark-runner-provider-boundary.mjs";
+import { runBenchmarkSession, runOfflineBenchmarkSession } from "./benchmark-session.mjs";
+
+// One schedule implementation owns dispatch, WAL promotion, accounting and
+// cleanup. The production entry never selects the explicit offline session seam.
+export function runBenchmarkSchedule(context) {
+  if (context.piagentWebUiJourney !== undefined) throw new Error("Production schedule does not accept an injected journey");
+  return runSchedule(context, runBenchmarkSession);
+}
+
+// Test composition only: no CLI/environment switch, registration, verifier,
+// credential writeback or production campaign authority can select this seam.
+export function runOfflineBenchmarkSchedule(context, { piagentWebUiJourney } = {}) {
+  if (typeof piagentWebUiJourney !== "function") throw new Error("Offline schedule requires an explicit scripted journey");
+  if (context.productionCampaign || context.registeredMeasurement || context.registeredMeasurementRun
+    || context.verificationPlan || context.providerWirePlan || context.registeredScopedSessionFactory
+    || context.options?.registeredMeasurement || context.bootstrapMetadata?.piAgentHome?.operatorAuth) {
+    throw new Error("Offline schedule cannot carry production or independent-verifier authority");
+  }
+  return runSchedule(context, input => runOfflineBenchmarkSession({ ...input, piagentWebUiJourney }));
+}
+
+async function runSchedule(context, runSession) {
+  const {
+    order, fullOrder, pendingOrder, options, runs,
+    manifest, recoveredAttemptsByKey, executionGuard, budgetRuntime, codexRuntime,
+    registeredMeasurement, registeredMeasurementRun, registeredRuntimeVerifiers, bootstrapMetadata, runtimeCommands,
+    piRuntimeHome, packageRoot, runCommand, suite, suiteRoot,
+    runId, runRoot, verificationPlan, providerWirePlan, candidateGuard,
+    piCommand, codexCommand, runtime, registeredScopedSessionFactory, suiteDigest,
+    configurationDigest, productionCampaign, rootSeed, ledgerPath, infrastructureLedgerPath,
+    interrupted, state
+  } = context;
+  let { fatalRunError, pauseReason, preservePiRuntime, fatalExecutionReceipt, ledgerBinding, terminalStop } = state;
+  const fullIndexByKey = new Map(fullOrder.map((item, index) => [benchmarkRunKey(item), index + 1]));
+  const wallStartedAt = Date.now();
+  const runtimeDeadline = options.maxRuntimeMinutes === undefined
+    ? undefined
+    : wallStartedAt + options.maxRuntimeMinutes * 60_000;
+  let newRuns = 0;
+  try {
+    for (const [index, item] of order.entries()) {
+      try { budgetRuntime?.check(); }
+      catch (error) { fatalRunError = error; break; }
+      if (manifest.transportCircuitBreaker.state === "open") {
+        fatalRunError = new Error(`Transport circuit breaker is open after ${manifest.transportCircuitBreaker.failures} failures; start a new run only after provider health is re-established`);
+        break;
+      }
+      if (interrupted()) break;
+      if (runtimeDeadline !== undefined && newRuns > 0 && !samePairedBlock(order[index - 1], item) && Date.now() >= runtimeDeadline) {
+        pauseReason = `max-runtime-minutes:${options.maxRuntimeMinutes}`;
+        break;
+      }
+      const completedBefore = runs.filter(completedBenchmarkRecord).length;
+      const remainingIncludingThis = fullOrder.length - completedBefore;
+      const averageMs = newRuns > 0 ? (Date.now() - wallStartedAt) / newRuns : undefined;
+      const eta = averageMs === undefined ? "" : ` · ETA ${formatDuration(averageMs * remainingIncludingThis)}`;
+      const fullIndex = fullIndexByKey.get(benchmarkRunKey(item)) ?? index + 1;
+      process.stdout.write(`[${fullIndex}/${fullOrder.length}] ${item.scenario.id} · ${item.surface} · repeat ${item.repeat}/${options.repeats} · remaining ${remainingIncludingThis} · elapsed ${formatDuration(Date.now() - wallStartedAt)}${eta}\n`);
+      let record;
+      let sessionEvidence;
+      const infrastructureFailures = [...(recoveredAttemptsByKey.get(benchmarkRunKey(item)) ?? [])];
+      const recoveredDisposition = recoveredBenchmarkAttemptDisposition(infrastructureFailures, {
+        scenarioId: item.scenario.id,
+        surface: item.surface,
+        repeat: item.repeat,
+        retryLimit: options.infrastructureRetries
+      });
+      if (!recoveredDisposition.passed) {
+        fatalRunError = new Error(recoveredDisposition.error);
+        break;
+      }
+      const firstInfrastructureAttempt = recoveredDisposition.firstAttempt;
+      for (let infrastructureAttempt = firstInfrastructureAttempt; infrastructureAttempt <= options.infrastructureRetries + 1; infrastructureAttempt += 1) {
+        record = undefined;
+        fatalRunError = executionGuard.check(`before-session:${item.scenario.id}:${item.surface}:r${item.repeat}:attempt${infrastructureAttempt}`, [piRuntimeHome]);
+        if (fatalRunError) break;
+        let attemptError;
+        let attemptCodexRuntime = codexRuntime;
+        try {
+          if (item.surface === "codex-cli") attemptCodexRuntime = createCodexRuntime(options);
+          const assertProviderBoundary = createBenchmarkProviderBoundaryGuard({ executionGuard,
+            registeredMeasurement, registeredMeasurementRun, registeredRuntimeVerifiers, bootstrapMetadata,
+            runtimeCommands, codexRuntime: attemptCodexRuntime, piRuntimeHome, scenarioId: item.scenario.id,
+            surface: item.surface, repeat: item.repeat, infrastructureAttempt });
+          sessionEvidence = await withBenchmarkPiCredentialWriteback(bootstrapMetadata.piAgentHome, piRuntimeHome, () => runSession({
+            packageRoot,
+            runCommand,
+            resolveSuiteEntry: resolveBenchmarkSuiteEntry,
+            interrupted: interrupted,
+            suite,
+            suiteRoot,
+            ...item,
+            orderIndex: fullIndex,
+            infrastructureAttempt,
+            runId,
+            runRoot,
+            options,
+            verificationPlan,
+            ...(item.surface === "piagent" && providerWirePlan ? { providerWirePlan, candidateDigest: candidateGuard.provenance.contentDigest } : {}),
+            piCommand,
+            codexCommand,
+            codexDisabledFeatures: runtime.codexDisabledFeatures,
+            codexRuntime: attemptCodexRuntime,
+            scopedBrokerSessionFactory: registeredScopedSessionFactory
+              && registeredBenchmarkScopedSessionRequired(registeredScopedSessionFactory, item.scenario.id)
+              ? registeredScopedSessionFactory : undefined,
+            piRuntimeHome,
+            systemCommands: { node: runtimeCommands.node.resolvedPath, git: runtimeCommands.git.resolvedPath, bash: runtimeCommands.bash.resolvedPath },
+            suiteDigest, configurationDigest,
+            assertProviderDispatchReady: () => { budgetRuntime?.check(); return assertProviderBoundary("provider-dispatch"); },
+            onAfterProviderDispatch: () => assertProviderBoundary("provider-return"),
+            ...createBudgetProviderCallbacks(budgetRuntime, productionCampaign),
+            persistCompletedRecord: (candidate) => stageMeasuredBenchmarkRecord({ runRoot, manifest, ledgerBinding, record: candidate, infrastructureFailures, index: fullIndex - 1, expected: item, runId, suite, configurationDigest, runs }),
+            rootSeed
+          }));
+          record = sessionEvidence.record;
+        } catch (error) {
+          preservePiRuntime ||= error.code === "BENCHMARK_PI_CREDENTIAL_RECONCILIATION_FAILED";
+          attemptError = error;
+          const safeError = safeInfrastructureDiagnostic(error.message, [piRuntimeHome?.path, bootstrapMetadata.piAgentHome.configRoot, bootstrapMetadata.piAgentHome.runtimeParent].filter(Boolean));
+          record = benchmarkRunnerErrorRecord({
+            safeError,
+            runId,
+            orderIndex: fullIndex,
+            item,
+            suiteProfile: suite.profile,
+            infrastructureAttempt
+          });
+        } finally {
+          if (attemptCodexRuntime !== codexRuntime) attemptCodexRuntime.cleanup();
+        }
+        const guardStage = `after-session:${item.scenario.id}:${item.surface}:r${item.repeat}:attempt${infrastructureAttempt}`;
+        const postSessionReceipt = executionGuard.receipt(guardStage, [piRuntimeHome]);
+        const postSessionGuard = postSessionReceipt.stamp;
+        const assetError = postSessionReceipt.error;
+        if (!assetError && item.surface !== "codex-cli") resetBenchmarkPiRuntimeEphemeralState(piRuntimeHome);
+        if (!interrupted()) {
+          if (assetError) {
+            fatalExecutionReceipt = postSessionReceipt;
+            if (sessionEvidence) retainWorkspaceForensics({ runRoot, workspaceRoot: sessionEvidence.workspaceRoot, key: sessionEvidence.key, record });
+            if (record) {
+              persistUnacceptedBenchmarkAttempt({ runRoot, manifest, record, reason: "execution-asset-mismatch-after-provider-attempt", forceTokenUnavailable: forceTokenUnavailableForPostSessionAssetError({ sessionEvidence, record }) });
+              fs.rmSync(path.join(runRoot, "pending-record.json"), { force: true });
+              fs.rmSync(path.join(runRoot, "measured-record-ready.json"), { force: true });
+              appendPrivateJsonl(infrastructureLedgerPath, { ...record, accepted: false, contaminated: true, executionAsset: assetError.executionAsset ?? { reason: assetError.message } });
+              if (sessionEvidence) fs.rmSync(sessionEvidence.inflightPath, { force: true });
+            }
+            fatalRunError = assetError;
+            break;
+          }
+          if (sessionEvidence && !record.abortSuite) promoteMeasuredBenchmarkRecord({ runRoot, ledgerBinding, record, postSessionGuard });
+        }
+        if (interrupted() && sessionEvidence) {
+          retainWorkspaceForensics({ runRoot, workspaceRoot: sessionEvidence.workspaceRoot, key: sessionEvidence.key, record });
+          persistUnacceptedBenchmarkAttempt({ runRoot, manifest, record, reason: "interrupted-provider-attempt-not-accepted-as-a-measured-outcome" });
+          fs.rmSync(path.join(runRoot, "pending-record.json"), { force: true });
+          fs.rmSync(path.join(runRoot, "measured-record-ready.json"), { force: true });
+          appendPrivateJsonl(infrastructureLedgerPath, { ...record, accepted: false, interrupted: true });
+          fs.rmSync(sessionEvidence.inflightPath, { force: true });
+        }
+        if (!record.abortSuite || interrupted()) break;
+        const failureDisposition = benchmarkInfrastructureFailureDisposition({
+          circuit: manifest.transportCircuitBreaker,
+          record,
+          infrastructureAttempt,
+          retryLimit: options.infrastructureRetries,
+          orderIndex: fullIndex,
+          scenarioId: item.scenario.id,
+          surface: item.surface,
+          repeat: item.repeat
+        });
+        manifest.transportCircuitBreaker = failureDisposition.circuit;
+        const retryAvailable = failureDisposition.retryAvailable;
+        infrastructureFailures.push(failureDisposition.failure);
+        if (failureDisposition.unknownCost) {
+          manifest.unknownCostAttempts = Number(manifest.unknownCostAttempts ?? 0) + 1;
+          manifest.tokenClaimsUnavailableReason = "one-or-more-provider-attempts-have-unknown-usage";
+        }
+        writeBenchmarkRunManifest(runRoot, manifest);
+        // A runner error before runBenchmarkSession creates its durable in-flight
+        // marker is positively pre-provider. Post-provider throws leave that
+        // marker behind and terminal recovery persists them fail-closed.
+        if (record.attemptId) persistUnacceptedBenchmarkAttempt({ runRoot, manifest, record });
+        appendPrivateJsonl(infrastructureLedgerPath, { ...record, accepted: false, retryAvailable });
+        if (sessionEvidence) {
+          retainWorkspaceForensics({ runRoot, workspaceRoot: sessionEvidence.workspaceRoot, key: sessionEvidence.key, record });
+          fs.rmSync(sessionEvidence.inflightPath, { force: true });
+        }
+        if (!retryAvailable) {
+          if (manifest.transportCircuitBreaker.state === "open") {
+            fatalRunError = new Error(`Transport circuit breaker opened after ${manifest.transportCircuitBreaker.failures} failures`);
+          }
+          if (attemptError) fatalRunError = new Error(record.infrastructureFailure ?? "benchmark runner infrastructure error");
+          break;
+        }
+        process.stdout.write(`           RETRY ${infrastructureAttempt}/${options.infrastructureRetries} (${record.infrastructureFailure ?? record.failure})\n`);
+        if (options.retryDelaySeconds > 0) {
+          await new Promise((resolve) => setTimeout(resolve, options.retryDelaySeconds * 1_000));
+        }
+      }
+      if (interrupted()) break;
+      if (fatalRunError) break;
+      if (record.abortSuite) {
+        if (!fatalRunError) fatalRunError = new Error(record.infrastructureFailure ?? record.failure ?? "agent startup failure");
+        break;
+      }
+      const retainWorkspace = options.keepWorkspaces || !record.resolved || sessionEvidence?.workflowFailed;
+      if (retainWorkspace && sessionEvidence) {
+        retainWorkspaceForensics({ runRoot, workspaceRoot: sessionEvidence.workspaceRoot, key: sessionEvidence.key, record });
+      }
+      runs.push(record);
+      newRuns += 1;
+      const pendingRecordPath = path.join(runRoot, "pending-record.json");
+      ledgerBinding = appendBenchmarkLedger(ledgerPath, record, ledgerBinding);
+      manifest.ledger = ledgerBinding;
+      clearRecoveredBenchmarkAttempts(manifest, record);
+      writeBenchmarkRunManifest(runRoot, manifest);
+      fs.rmSync(pendingRecordPath, { force: true });
+      if (sessionEvidence) {
+        fs.rmSync(sessionEvidence.inflightPath, { force: true });
+        if (!retainWorkspace) fs.rmSync(sessionEvidence.workspaceRoot, { recursive: true, force: true });
+      }
+      const cost = Number.isFinite(record.usage.cost) ? `$${Number(record.usage.cost).toFixed(6)}` : "cost n/a";
+      const completedAfter = runs.filter(completedBenchmarkRecord).length;
+      const remainingAfter = Math.max(0, fullOrder.length - completedAfter);
+      const averageAfterMs = (Date.now() - wallStartedAt) / Math.max(1, newRuns);
+      process.stdout.write(`           ${record.resolved ? "PASS" : `FAIL (${record.failure})`} · ${record.usage.fresh} fresh tok · ${cost} · run ${formatDuration(Number(record.durationSeconds ?? 0) * 1_000)} · remaining ${remainingAfter} · ETA ${formatDuration(averageAfterMs * remainingAfter)}\n`);
+      fatalRunError = productionMeasurementInvalidatingOutcomeError(options, suite, record);
+      if (fatalRunError) break;
+      terminalStop = pairedOutcomeFloorStop({ enabled: options.stopAfterFailedPair, suite, runs, current: item, next: order[index + 1] });
+      if (terminalStop) break;
+      if (fatalRunError) break;
+    }
+    if (!pauseReason && options.maxSessions !== undefined && pendingOrder.length > order.length) {
+      pauseReason = `max-sessions:${options.maxSessions}`;
+    }
+  } finally {
+    Object.assign(state, { fatalRunError, pauseReason, preservePiRuntime, fatalExecutionReceipt, ledgerBinding, terminalStop });
+  }
+}