@piagent/platform 1.6.1 → 1.8.0

package/scripts/benchmark-session.mjs

@@ -2,21 +2,18 @@ import crypto from "node:crypto";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
-
 import {
-  aggregateSessionUsage,
-  createCodexExecJsonlCollector,
-  evaluateWorkflowEvidence
+  aggregateSessionUsage, createCodexExecJsonlCollector, evaluateWorkflowEvidence
 } from "../packages/piagent-core/benchmark/benchmark-core.js";
 import { codexExecArgs } from "../packages/piagent-core/benchmark/benchmark-codex.js";
+import { classifyCodexAttemptOutcome } from "../packages/piagent-core/benchmark/benchmark-codex-outcome.js";
 import {
   benchmarkEnvironment,
   benchmarkGitEnvironment,
-  codexProcessEnvironment,
   piagentProcessEnvironment
 } from "../packages/piagent-core/benchmark/benchmark-runtime.js";
 import {
-  acceptedTaskStartTraceCount,
+  acceptedTaskStartTraceEvidence,
   benchmarkCausalContextReceipt,
   benchmarkOperationalEvidence,
   classifyPreUsageFailure,
@@ -30,35 +27,116 @@ import { inspectContextTelemetry } from "../packages/piagent-core/extensions/con
 import { matchesAnyPath } from "../packages/piagent-core/extensions/policy-core.js";
 import { listTaskContracts, workingTreeFiles, workingTreeSnapshot } from "../packages/piagent-core/extensions/task-state.js";
 import { benchmarkTreeIdentity } from "../packages/piagent-core/benchmark/benchmark-tree-identity.js";
-import { buildBenchmarkProviderWireEvidence } from "../packages/piagent-core/benchmark/benchmark-provider-wire.js";
+import { buildBenchmarkProviderWireEvidence, freezeBenchmarkWireInvocation, readBenchmarkWireReceipts, validateBenchmarkWireManifest } from "../packages/piagent-core/benchmark/benchmark-provider-wire.js";
+import { buildCodexInvocationReceipt, inspectCodexRolloutServiceTierEvidence } from "../packages/piagent-core/benchmark/benchmark-codex-rollout.js";
 import { createDeferredBenchmarkTimingCollector } from "../packages/piagent-core/benchmark/benchmark-timing-diagnostics.js";
-import { summarizeSession, walkJsonl } from "./pi-usage-history.mjs";
-
+import { benchmarkReportOutcomeFields } from "../packages/piagent-core/benchmark/benchmark-evaluator-v3.js";
+import { createBenchmarkSafetyEvidenceObserver,
+  inspectBenchmarkSafetyJsonlFiles } from "../packages/piagent-core/benchmark/benchmark-safety-evidence.js";
+import { walkJsonl } from "./pi-usage-history.mjs";
+import { inspectBenchmarkSessionDirectory } from "./benchmark-session-usage.mjs";
+import { runPiagentWebUiJourney } from "./benchmark-webui-journey.mjs";
+import { resolvedJourneyTurns } from "./benchmark-independent-verification.mjs";
+import { benchmarkVerificationFailure } from "../packages/piagent-core/benchmark/benchmark-independent-verification-observation.js";
+import { BENCHMARK_SCOPED_SESSION_CUSTODY_VERSION, BENCHMARK_SCOPED_SESSION_FACTORY_VERSION,
+  BENCHMARK_SCOPED_SESSION_REQUEST_VERSION, controlledCodexEnvironment, isCodexScopedBrokerTurnFactory,
+  requireScopedCodexHome,
+  runCodexUserJourney } from "./benchmark-codex-journey.mjs";
+import { assertBenchmarkSessionProviderBoundary, providerBoundaryFailureDisposition,
+  runPostDispatchCheckedCommand } from "./benchmark-session-provider-boundary.mjs";
+import { candidateOutcomeFailureReason, persistedJourneyReceipt } from "./benchmark-journey-outcome.mjs";
+import { buildProductionV3SessionGraderInput,
+  finalizeProductionV3SessionOutcome } from "./benchmark-session-evaluator.mjs";
+export { runCodexUserJourney } from "./benchmark-codex-journey.mjs";
+export { candidateOutcomeFailureReason, persistedJourneyReceipt } from "./benchmark-journey-outcome.mjs";
 const coldStartRuntimeManagedPaths = [
   ".pi/project-context.md",
   ".pi/context-index.json",
   ".pi/piagent-state/project-onboarding.json",
   ".pi/piagent-state/context-engine.json"
 ];
-
 function fail(message) {
   const error = new Error(message);
   error.exitCode = 1;
   throw error;
 }
+const scopedHash = /^[a-f0-9]{64}$/;
+const scopedId = /^[A-Za-z0-9][A-Za-z0-9:._~-]{0,159}$/;
+function scopedRequire(value, code) {
+  if (!value) throw Object.assign(new Error(code), { brokerCode: code });
+}
+function scopedExact(value, names, code) {
+  scopedRequire(value && typeof value === "object" && !Array.isArray(value)
+    && Object.keys(value).length === names.length && names.every(name => Object.hasOwn(value, name)), code);
+}
+function frozenScopedTurns(turns) {
+  scopedRequire(Array.isArray(turns) && turns.length > 0, "session-boundary-turns");
+  return Object.freeze(turns.map(turn => {
+    scopedRequire(turn && typeof turn === "object" && typeof turn.id === "string" && scopedId.test(turn.id)
+      && typeof turn.message === "string" && turn.message.length > 0 && turn.message.isWellFormed(),
+    "session-boundary-turns");
+    return Object.freeze({ id: turn.id, message: turn.message, workflow: turn.workflow ?? null,
+      reconnectBefore: turn.reconnectBefore === true, receiptUncertain: turn.receiptUncertain === true });
+  }));
+}
+function validScopedPiRouter(value) {
+  return Boolean(value && typeof value === "object" && !Array.isArray(value)
+    && value.version === "scoped-pi-operation-router-v1" && value.authority === "none"
+    && Array.isArray(value.toolNames) && typeof value.extensionFactory === "function"
+    && typeof value.beginOperation === "function" && typeof value.settlementEvidence === "function"
+    && typeof value.assertProviderDispatchReady === "function"
+    && typeof value.takeFatalProviderBoundaryError === "function"
+    && typeof value.assertOwnership === "function" && typeof value.dispose === "function"
+    && typeof value.status === "function");
+}
+/** Resolves the only scoped controls a registered journey may receive. A
+ * factory denial happens before the caller records or starts provider work. */
+export async function resolveBenchmarkScopedSessionControls({ factory, directCodexScopedBroker,
+  directPiScopedBrokerRouter, journeyTurns, runId, suiteId, scenarioId, surface, repeat,
+  infrastructureAttempt, configurationSha256, workspace, model, thinking, serviceTier }) {
+  if (factory == null) return Object.freeze({ codexScopedBroker: directCodexScopedBroker ?? null,
+    piScopedBrokerRouter: directPiScopedBrokerRouter ?? null });
+  scopedExact(factory, ["version", "authority", "openSession"], "session-boundary-factory");
+  scopedRequire(factory.version === BENCHMARK_SCOPED_SESSION_FACTORY_VERSION && factory.authority === "none"
+    && typeof factory.openSession === "function", "session-boundary-factory");
+  scopedRequire(directCodexScopedBroker == null && directPiScopedBrokerRouter == null,
+    "session-boundary-direct-control-conflict");
+  scopedRequire(Array.isArray(journeyTurns) && journeyTurns.length > 0, "session-boundary-journey-required");
+  scopedRequire([runId, suiteId, scenarioId].every(value => typeof value === "string" && scopedId.test(value))
+    && ["piagent", "codex-cli"].includes(surface) && Number.isSafeInteger(repeat) && repeat >= 1
+    && repeat <= 10 && Number.isSafeInteger(infrastructureAttempt) && infrastructureAttempt >= 1
+    && infrastructureAttempt <= 3 && scopedHash.test(configurationSha256)
+    && typeof model === "string" && model.length > 0 && typeof thinking === "string" && thinking.length > 0
+    && (serviceTier === null || typeof serviceTier === "string" && serviceTier.length > 0),
+  "session-boundary-request");
+  scopedRequire(typeof workspace === "string" && path.isAbsolute(workspace)
+    && fs.realpathSync.native(workspace) === workspace, "session-boundary-workspace");
+  const request = Object.freeze({ version: BENCHMARK_SCOPED_SESSION_REQUEST_VERSION, authority: "none",
+    runId, suiteId, scenarioId, surface, repeat, infrastructureAttempt, configurationSha256, workspace,
+    model, thinking, serviceTier, turns: frozenScopedTurns(journeyTurns) });
+  const controller = await factory.openSession(request);
+  scopedExact(controller, ["version", "authority", "surface", "piScopedBrokerRouter", "codexScopedBroker"],
+    "session-boundary-controller");
+  scopedRequire(controller.version === BENCHMARK_SCOPED_SESSION_CUSTODY_VERSION
+    && controller.authority === "none" && controller.surface === surface, "session-boundary-controller");
+  if (surface === "piagent") scopedRequire(controller.codexScopedBroker === null
+    && validScopedPiRouter(controller.piScopedBrokerRouter), "session-boundary-pi-controller");
+  else scopedRequire(controller.piScopedBrokerRouter === null
+    && isCodexScopedBrokerTurnFactory(controller.codexScopedBroker), "session-boundary-codex-controller");
+  return Object.freeze({ codexScopedBroker: controller.codexScopedBroker,
+    piScopedBrokerRouter: controller.piScopedBrokerRouter });
+}
 
 function privateDirectory(target) {
   fs.mkdirSync(target, { recursive: true, mode: 0o700 });
   try { fs.chmodSync(target, 0o700); } catch { /* Non-POSIX filesystem. */ }
   return target;
 }
-
 function privateTemporaryDirectory(prefix) {
   const target = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
   try { fs.chmodSync(target, 0o700); } catch { /* Non-POSIX filesystem. */ }
   return target;
 }
-
 function makeFixtureWritable(root) {
   const pending = [root];
   while (pending.length > 0) {
@@ -284,11 +362,15 @@ function parseGraderResult(stdout) {
   try { value = JSON.parse(lines.at(-1) ?? ""); } catch { fail("Benchmark grader did not return a JSON object"); }
   if (!value || typeof value !== "object" || Array.isArray(value) || typeof value.passed !== "boolean" || !Array.isArray(value.checks)) fail("Benchmark grader result must contain passed and checks");
   for (const check of value.checks) if (!check || typeof check.id !== "string" || typeof check.passed !== "boolean") fail("Benchmark grader returned an invalid check");
-  return {
+  const parsed = {
     passed: value.passed,
     score: Number.isFinite(value.score) ? Math.max(0, Math.min(10, value.score)) : value.passed ? 10 : 0,
     checks: value.checks.map((check) => ({ id: check.id, passed: check.passed, detail: typeof check.detail === "string" ? check.detail.slice(0, 500) : undefined }))
   };
+  for (const field of ["semanticStatus", "gradeStatus", "failureClass"]) {
+    if (typeof value[field] === "string") parsed[field] = value[field];
+  }
+  return parsed;
 }
 
 async function gradeWorkspace(runCommand, nodeCommand, grader, workspace, scenario, timeoutSeconds, oracleSerialized) {
@@ -307,11 +389,20 @@ async function gradeWorkspace(runCommand, nodeCommand, grader, workspace, scenar
   }
 }
 
-function sessionSummaries(sessionDir) {
-  return walkJsonl(sessionDir).map((file) => summarizeSession(file, { strictUsage: true })).filter(Boolean);
+function observedSubstrings(value, candidates) {
+  const text = String(value ?? "");
+  return candidates.filter((candidate) => text.includes(candidate));
 }
 
-export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuiteEntry, interrupted, persistCompletedRecord, suite, suiteRoot, scenario, surface, repeat, orderIndex, infrastructureAttempt = 1, runId, runRoot, options, piCommand, codexCommand, codexDisabledFeatures, codexRuntime, piRuntimeHome, systemCommands, suiteDigest, configurationDigest, rootSeed }) {
+async function runBenchmarkSessionInternal({ packageRoot, runCommand, resolveSuiteEntry, interrupted, persistCompletedRecord,
+  assertProviderDispatchReady, onProviderAttemptStart = () => {}, onAfterProviderDispatch,
+  onProviderAttemptReturned = () => {}, suite, suiteRoot, scenario, surface, repeat, orderIndex, infrastructureAttempt = 1,
+  runId, runRoot, options, verificationPlan, providerWirePlan, candidateDigest, piCommand, codexCommand,
+  codexDisabledFeatures, codexRuntime, codexScopedBroker, piScopedBrokerRouter, scopedBrokerSessionFactory,
+  piRuntimeHome, systemCommands, suiteDigest, configurationDigest, rootSeed,
+  piagentWebUiJourney = runPiagentWebUiJourney }, allowInjectedJourney) {
+  assertBenchmarkSessionProviderBoundary({ assertProviderDispatchReady, onAfterProviderDispatch, surface, piagentWebUiJourney,
+    defaultPiagentWebUiJourney: runPiagentWebUiJourney, allowInjectedJourney });
   if (surface !== "codex-cli" && !piRuntimeHome?.path) fail("Pi benchmark session is missing its controlled writable runtime home");
   const attemptSuffix = infrastructureAttempt > 1 ? `-infra-${infrastructureAttempt}` : "";
   const key = `${String(repeat).padStart(2, "0")}-${scenario.id}-${surface}${attemptSuffix}`;
@@ -319,6 +410,7 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
   const workspace = path.join(workspaceRoot, "project");
   const sessions = privateDirectory(path.join(workspaceRoot, "sessions"));
   fs.cpSync(resolveSuiteEntry(suiteRoot, scenario.fixture, "fixture"), workspace, { recursive: true, errorOnExist: true });
+  fs.chmodSync(workspace, 0o700);
   makeFixtureWritable(workspace);
   applySetupFiles(workspace, scenario.setupFiles);
   const profile = scenario.profile ?? suite.profile;
@@ -342,9 +434,30 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
     }
   }
   await initializeGit(runCommand, systemCommands.git, workspace, scenario.setupFiles);
-  const prompt = fs.readFileSync(resolveSuiteEntry(suiteRoot, scenario.prompt, "prompt"), "utf8").trim();
+  const registeredInput = verificationPlan?.registeredInput?.(scenario.id) ?? null;
+  const prompt = registeredInput?.prompt
+    ?? fs.readFileSync(resolveSuiteEntry(suiteRoot, scenario.prompt, "prompt"), "utf8").trim();
+  const journeyTurns = registeredInput?.turns
+    ?? resolvedJourneyTurns(scenario, suiteRoot, resolveSuiteEntry);
+  const scopedSessionControls = await resolveBenchmarkScopedSessionControls({ factory: scopedBrokerSessionFactory,
+    directCodexScopedBroker: codexScopedBroker, directPiScopedBrokerRouter: piScopedBrokerRouter,
+    journeyTurns, runId, suiteId: verificationPlan?.registeredSuiteId ?? suite.id,
+    scenarioId: scenario.id, surface, repeat, infrastructureAttempt,
+    configurationSha256: verificationPlan?.measurementConfigurationDigest ?? configurationDigest,
+    workspace: fs.realpathSync.native(workspace), model: options.model,
+    thinking: options.thinking, serviceTier: options.serviceTier ?? null });
+  const effectiveCodexScopedBroker = scopedSessionControls.codexScopedBroker;
+  const effectivePiScopedBrokerRouter = scopedSessionControls.piScopedBrokerRouter;
+  const independent = verificationPlan?.prepare({ scenarioId: scenario.id, surface, projectRoot: workspace,
+    directory: surface === "piagent" ? path.join(privateDirectory(path.join(fs.realpathSync.native(runRoot), "independent-verification")), key) : undefined,
+    approved: options.approveVerification });
   const graderPath = resolveSuiteEntry(suiteRoot, scenario.grader, "grader");
-  const sessionId = crypto.randomUUID();
+  // The journey owns normalization and cleanup. Passing the session custody
+  // through unchanged avoids a second preparation pass and lets malformed
+  // later-turn custody release every already-materialized turn before admission.
+  const codexJourneyScopedBrokers = journeyTurns && surface === "codex-cli"
+    ? effectiveCodexScopedBroker : null;
+  let sessionId = crypto.randomUUID();
   const attemptId = crypto.randomUUID();
   const piArgs = ["--print", "--mode", "json", "--session-dir", sessions, "--session-id", sessionId, "--name", `BENCH ${scenario.id} ${surface} r${repeat}`, "--approve", "--no-skills", "--no-prompt-templates", "--no-extensions", "--no-context-files"];
   if (surface === "piagent") {
@@ -356,48 +469,191 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
   if (options.thinking) piArgs.push("--thinking", options.thinking);
   piArgs.push(prompt);
   const command = surface === "codex-cli" ? codexCommand : piCommand;
-  const args = surface === "codex-cli" ? codexExecArgs({ workspace, options, disabledFeatures: codexDisabledFeatures }) : piArgs;
+  const args = surface === "codex-cli" ? codexExecArgs({
+    workspace,
+    options,
+    disabledFeatures: codexDisabledFeatures,
+    scopedBroker: journeyTurns ? undefined : effectiveCodexScopedBroker,
+    persistent: options.serviceTier === "fast"
+  }) : piArgs;
   const codexForbiddenHits = new Set();
-  const codexCollector = surface === "codex-cli" ? createCodexExecJsonlCollector({ model: options.model, thinkingLevel: options.thinking, onEvent: (event) => inspectForbiddenValue(event, forbiddenOutputSubstrings, codexForbiddenHits) }) : undefined;
+  const safetyObserver = createBenchmarkSafetyEvidenceObserver(scenario.id);
+  const codexCollector = surface === "codex-cli" ? createCodexExecJsonlCollector({
+    model: options.model,
+    thinkingLevel: options.thinking,
+    requestedServiceTier: options.serviceTier,
+    eventContract: options.codexBaseline === "stock" ? "production-v3" : undefined,
+    onEvent: (event) => {
+      inspectForbiddenValue(event, forbiddenOutputSubstrings, codexForbiddenHits);
+      safetyObserver.observe(event);
+    }
+  }) : undefined;
   const timingCollector = createDeferredBenchmarkTimingCollector({ surface });
   const environment = {
     PIAGENT_NO_UPDATE_CHECK: "1", PIAGENT_BENCHMARK_RUN_ID: runId, PIAGENT_BENCHMARK_SCENARIO: scenario.id,
     PIAGENT_BENCHMARK_SURFACE: surface, PIAGENT_BENCHMARK_SESSION_ID: sessionId, PIAGENT_BENCHMARK_PROFILE: profile,
-    PIAGENT_BENCHMARK_LIFECYCLE: lifecycle, NO_COLOR: "1"
+    PIAGENT_BENCHMARK_LIFECYCLE: lifecycle,
+    PIAGENT_FAST_MODE: options.serviceTier === "fast" ? "1" : "0",
+    NO_COLOR: "1"
   };
   const inflightPath = path.join(workspaceRoot, "inflight.json");
-  writePrivateAtomic(inflightPath, `${JSON.stringify({ schemaVersion: 1, runId, attemptId, orderIndex, scenarioId: scenario.id, surface, repeat, infrastructureAttempt, stage: "provider-may-start", recordedAt: new Date().toISOString() }, null, 2)}\n`);
+  const attemptIdentity = { attemptId, orderIndex, scenarioId: scenario.id, surface, repeat, infrastructureAttempt };
+  let providerDispatchAdmitted = false;
+  const admitProviderDispatch = () => {
+    if (providerDispatchAdmitted) return;
+    writePrivateAtomic(inflightPath, `${JSON.stringify({ schemaVersion: 1, runId, ...attemptIdentity, stage: "provider-may-start", recordedAt: new Date().toISOString() }, null, 2)}\n`);
+    onProviderAttemptStart(attemptIdentity);
+    providerDispatchAdmitted = true;
+  };
+  if (providerWirePlan && (surface !== "piagent" || !journeyTurns)) fail("Provider-wire manifest requires the qualified Piagent WebUI route");
+  if (providerWirePlan) validateBenchmarkWireManifest(providerWirePlan.manifest, { template: true,
+    model: options.model, thinking: options.thinking, requestedTier: options.serviceTier });
+  const wireInvocation = providerWirePlan ? freezeBenchmarkWireInvocation({ plan: providerWirePlan, workingDirectory: workspace, platformRoot: packageRoot, custodyRoot: fs.realpathSync.native(runRoot), configurationDigest, candidateDigest }) : null;
+  const processEnvironment = surface === "codex-cli"
+    ? controlledCodexEnvironment(codexRuntime, environment)
+    : surface === "piagent"
+      ? piagentProcessEnvironment(options.piagentTreatment, { ...environment, ...independent?.environment, ...wireInvocation?.environment, PI_CODING_AGENT_DIR: piRuntimeHome.path })
+      : benchmarkEnvironment({ ...environment, PI_CODING_AGENT_DIR: piRuntimeHome.path });
+  if (surface === "codex-cli" && !journeyTurns) {
+    requireScopedCodexHome(effectiveCodexScopedBroker, codexRuntime, processEnvironment);
+  }
   let agent;
+  let usageOverride;
+  let journeyReceipt = null;
+  let codexDiagnostics = [];
+  let codexContractFailure = null;
+  let fatalProviderBoundaryError = null;
+  let fatalProviderBoundaryPhase = null;
   try {
-    agent = await runCommand(command, args, {
-      cwd: workspace, input: surface === "codex-cli" ? prompt : undefined, timeoutMs: options.timeoutSeconds * 1_000,
-      forbiddenSubstrings: forbiddenOutputSubstrings, requiredSubstrings: requiredOutputSubstrings,
-      onStdoutChunk: (chunk, observation) => {
-        timingCollector.write(chunk, observation?.observedAtSeconds);
-        codexCollector?.write(chunk);
-      },
-      env: surface === "codex-cli" ? codexProcessEnvironment(codexRuntime, environment) : surface === "piagent" ? piagentProcessEnvironment(options.piagentTreatment, { ...environment, PI_CODING_AGENT_DIR: piRuntimeHome.path }) : benchmarkEnvironment({ ...environment, PI_CODING_AGENT_DIR: piRuntimeHome.path })
-    });
+    if (journeyTurns && surface === "piagent") {
+      agent = await piagentWebUiJourney({
+        packageRoot,
+        agentDir: piRuntimeHome.path,
+        workspace,
+        model: options.model,
+        thinking: options.thinking,
+        turns: journeyTurns,
+        expectedTerminalSettlement: scenario.userJourney.expectedTerminalSettlement,
+        timeoutMs: options.timeoutSeconds * 1_000,
+        environment: processEnvironment,
+        onBeforeProviderDispatch: assertProviderDispatchReady,
+        onBeforeFirstProviderDispatch: admitProviderDispatch,
+        ...(effectivePiScopedBrokerRouter ? { scopedBrokerRouter: effectivePiScopedBrokerRouter } : {})
+      });
+      if (!providerDispatchAdmitted) fail("Piagent journey ended before its first provider dispatch");
+      journeyReceipt = agent.journeyReceipt;
+      fatalProviderBoundaryError = agent.fatalProviderBoundaryError ?? null;
+      fatalProviderBoundaryPhase = agent.fatalProviderBoundaryPhase ?? null;
+      agent.forbiddenHits = observedSubstrings(agent.stdout, forbiddenOutputSubstrings);
+      agent.requiredHits = observedSubstrings(agent.stdout, requiredOutputSubstrings);
+    } else if (journeyTurns && surface === "codex-cli") {
+      const journey = await runCodexUserJourney({
+        runCommand,
+        codexCommand,
+        workspace,
+        turns: journeyTurns,
+        options,
+        disabledFeatures: codexDisabledFeatures, codexRuntime,
+        scopedBroker: codexJourneyScopedBrokers,
+        environment: processEnvironment,
+        timeoutMs: options.timeoutSeconds * 1_000,
+        forbiddenOutputSubstrings,
+        onBeforeProviderDispatch: assertProviderDispatchReady,
+        onAfterProviderDispatch,
+        onBeforeFirstProviderDispatch: admitProviderDispatch,
+        onEvent: event => safetyObserver.observe(event)
+      });
+      if (!providerDispatchAdmitted) fail("Codex journey ended before its first provider dispatch");
+      agent = journey.agent;
+      usageOverride = journey.usage;
+      codexDiagnostics = journey.diagnostics;
+      journeyReceipt = journey.journeyReceipt;
+      fatalProviderBoundaryError = journey.fatalProviderBoundaryError;
+      fatalProviderBoundaryPhase = journey.fatalProviderBoundaryPhase;
+      agent.requiredHits = observedSubstrings(agent.stdout, requiredOutputSubstrings);
+    } else {
+      await assertProviderDispatchReady(Object.freeze({ turnIndex: 1, turnId: null }));
+      admitProviderDispatch();
+      agent = await runPostDispatchCheckedCommand({ runCommand, command, args, workspace, prompt, surface,
+        timeoutMs: options.timeoutSeconds * 1_000, forbiddenOutputSubstrings, requiredOutputSubstrings,
+        processEnvironment, timingCollector, codexCollector, onAfterProviderDispatch });
+    }
   } catch (error) {
     timingCollector.discard();
     throw error;
   }
   const timingDiagnostics = timingCollector.finish(agent.durationSeconds);
-  const sessionFiles = surface === "codex-cli" ? [] : walkJsonl(sessions);
+  const sessionRoot = journeyTurns && surface === "piagent" ? path.join(piRuntimeHome.path, "sessions") : sessions;
+  const sessionFiles = surface === "codex-cli" ? [] : walkJsonl(sessionRoot);
+  const safetyEvidence = surface === "codex-cli" ? safetyObserver.summary()
+    : inspectBenchmarkSafetyJsonlFiles(sessionFiles, scenario.id);
+  const piSessionInspection = surface === "codex-cli"
+    ? { summaries: [], diagnostics: [] }
+    : inspectBenchmarkSessionDirectory(sessionRoot);
+  const piSummaries = piSessionInspection.summaries;
+  if (journeyTurns && surface === "piagent") {
+    const main = piSummaries.find((summary) => !summary.isSubagent && summary.cwd === workspace)
+      ?? piSummaries.find((summary) => !summary.isSubagent);
+    if (main?.id) sessionId = main.id;
+  }
   let usage;
-  let codexDiagnostics = [];
-  if (surface === "codex-cli") {
-    try { usage = codexCollector.finish(); }
-    catch (error) { if (agent.code === 0 && !agent.timedOut) throw error; usage = aggregateSessionUsage([]); }
+  if (usageOverride) usage = usageOverride;
+  else if (surface === "codex-cli") {
+    try {
+      usage = codexCollector.finish({ processExitCode: agent.code });
+      agent.responseText = !agent.timedOut ? codexCollector.terminalResponseText() : "";
+    } catch (error) {
+      if (error?.code === "BENCHMARK_CODEX_EVENT_CONTRACT_INVALID") {
+        codexContractFailure = error;
+        usage = error.usage ?? { ...aggregateSessionUsage([]), codexEventSummary: error.codexEventSummary,
+          codexEventOutcome: error.codexEventOutcome };
+      } else if (agent.code === 0 && !agent.timedOut) throw error;
+      else usage = aggregateSessionUsage([]);
+    }
+    usage = { ...usage, codexInvocationReceipts: [buildCodexInvocationReceipt({
+      command, args, runtime: codexRuntime, environment: processEnvironment, workspace, requestedModel: options.model,
+      requestedThinking: options.thinking, requestedServiceTier: options.serviceTier, resumed: false, result: agent, usage })] };
     codexDiagnostics = codexCollector.diagnostics();
-  } else usage = aggregateSessionUsage(sessionSummaries(sessions));
+    if (codexContractFailure) codexDiagnostics.push({ type: "codex-event-contract", message: codexContractFailure.codexEventOutcome?.reasonCodes?.join(",") ?? "invalid" });
+  } else usage = aggregateSessionUsage(piSummaries);
+  if (surface === "codex-cli" && options.serviceTier !== undefined) {
+    usage = {
+      ...usage,
+      serviceTierEvidence: inspectCodexRolloutServiceTierEvidence({
+        codexHome: codexRuntime?.mode === "controlled" ? codexRuntime.home : null, threadId: usage.providerSessionId, workspace,
+        requestedModel: options.model, requestedThinking: options.thinking, requestedServiceTier: options.serviceTier,
+        providerStartedAttempts: usage.execution?.providerStartedAttempts, invocationReceipts: usage.codexInvocationReceipts
+      })
+    };
+  }
   const piTerminalError = surface === "codex-cli" ? undefined : terminalPiSessionError(sessionFiles, sessionId);
-  const diagnosticInput = codexDiagnostics.length > 0
-    ? JSON.stringify(codexDiagnostics)
-    : piTerminalError ?? `${agent.stderr ?? ""}\n${agent.stdout ?? ""}`;
-  const preUsageFailure = classifyPreUsageFailure(agent, usage, diagnosticInput, { terminalProviderError: Boolean(piTerminalError) });
-  writePrivateAtomic(inflightPath, `${JSON.stringify({ schemaVersion: 1, runId, attemptId, orderIndex, scenarioId: scenario.id, surface, repeat, infrastructureAttempt, stage: "provider-returned", usage, recordedAt: new Date().toISOString() }, null, 2)}\n`);
+  const diagnosticInput = surface === "codex-cli"
+    ? [JSON.stringify(codexDiagnostics), agent.stderr].filter(Boolean).join("\n")
+    : [piTerminalError, ...piSessionInspection.diagnostics, agent.stderr, agent.stdout].filter(Boolean).join("\n");
+  const providerBoundaryFailure = providerBoundaryFailureDisposition(
+    fatalProviderBoundaryError, fatalProviderBoundaryPhase);
+  const preUsageFailure = providerBoundaryFailure
+    ?? classifyPreUsageFailure(agent, usage, diagnosticInput, { terminalProviderError: Boolean(piTerminalError),
+      usageParsingError: piSessionInspection.diagnostics.length > 0, candidateOutcome: agent.candidateOutcome ?? null });
+  const candidateOutcomeFailure = candidateOutcomeFailureReason(agent.candidateOutcome);
+  writePrivateAtomic(inflightPath, `${JSON.stringify({
+    schemaVersion: 1,
+    runId,
+    ...attemptIdentity,
+    stage: "provider-returned",
+    usage,
+    usageStatus: preUsageFailure?.usageStatus ?? "measured",
+    infrastructureFailure: preUsageFailure?.failure ?? null,
+    infrastructureClass: preUsageFailure?.class ?? null,
+    infrastructureRetryable: preUsageFailure?.retryable === true,
+    durationSeconds: agent.durationSeconds,
+    recordedAt: new Date().toISOString()
+  }, null, 2)}\n`);
+  onProviderAttemptReturned({ ...attemptIdentity, usage, usageStatus: preUsageFailure?.usageStatus ?? "measured" });
   const forbiddenHits = [...new Set([...(agent.forbiddenHits ?? []), ...(surface === "codex-cli" ? [...codexForbiddenHits] : forbiddenSessionHits(sessionFiles, forbiddenOutputSubstrings))])];
+  if (surface === "codex-cli" && suite.id === "production-v3") {
+    agent.requiredHits = observedSubstrings(agent.responseText, requiredOutputSubstrings);
+  }
   const requiredHits = new Set(agent.requiredHits ?? []);
   if (surface !== "codex-cli") for (const value of forbiddenSessionHits(sessionFiles, requiredOutputSubstrings)) requiredHits.add(value);
   const missingRequired = requiredOutputSubstrings.filter((value) => !requiredHits.has(value));
@@ -426,41 +682,78 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
     telemetryTruncated: contextTelemetryInspection.inputTruncated
   }) : null;
   let workflow = null;
+  const tasks = surface === "piagent" ? listTaskContracts(workspace).filter((item) => item.sessionId === sessionId) : [];
+  const independentVerification = independent?.observe(tasks);
+  const independentFailure = benchmarkVerificationFailure(independentVerification);
   if (surface === "piagent" && scenario.kind !== "safety-refusal") {
-    const task = listTaskContracts(workspace).find((item) => item.sessionId === sessionId);
-    workflow = evaluateWorkflowEvidence(task, changedFiles, usage.toolNames, { scenarioKind: scenario.kind, acceptedTaskStartCount: acceptedTaskStartTraceCount(sessionFiles, sessionId) });
+    workflow = evaluateWorkflowEvidence(tasks, changedFiles, usage.toolNames, {
+      scenarioKind: scenario.kind,
+      taskStartEvidence: acceptedTaskStartTraceEvidence(sessionFiles, sessionId),
+      expectedTurnCount: journeyTurns?.length ?? 1
+    });
     workflow.operational = benchmarkOperationalEvidence(contextTelemetry);
   }
   const beforeGrade = workingTreeSnapshot(workspace);
   const outsideScope = changedFiles.filter((file) => !matchesAnyPath(file, scenario.allowedChanges));
+  const graderInput = suite.id === "production-v3" && !preUsageFailure && !interrupted()
+    ? buildProductionV3SessionGraderInput({ suiteId: suite.id, oracleSerialized: variant.oracleSerialized,
+      scenario, surface, sessionId, agent, usage, journeyReceipt, changedFiles, outsideScope, missingRequired,
+      forbiddenHits, safetyEvidence })
+    : null;
+  const graderInputSerialized = graderInput ? `${JSON.stringify(graderInput)}\n` : variant.oracleSerialized;
   const grade = preUsageFailure
     ? { passed: false, score: 0, checks: [], error: preUsageFailure.failure }
     : interrupted()
     ? { passed: false, score: 0, checks: [], error: "interrupted-after-provider-start" }
-    : await gradeWorkspace(runCommand, systemCommands.node, graderPath, workspace, scenario, options.timeoutSeconds, variant.oracleSerialized);
+    : await gradeWorkspace(runCommand, systemCommands.node, graderPath, workspace, scenario, options.timeoutSeconds, graderInputSerialized);
   const graderIntegrity = { passed: JSON.stringify(beforeGrade) === JSON.stringify(workingTreeSnapshot(workspace)) };
   const scope = { passed: outsideScope.length === 0, changedFiles, outsideScope, allChangedFiles, runtimeManagedChanges };
+  const codexOutcomeFailure = surface === "codex-cli" ? classifyCodexAttemptOutcome({ eventOutcome: usage.codexEventOutcome, scenarioKind: scenario.kind, changedFiles }) : null;
   const outputSafety = { passed: forbiddenHits.length === 0, forbiddenHits: forbiddenHits.map((value) => crypto.createHash("sha256").update(value).digest("hex")) };
   const outputEvidence = { passed: missingRequired.length === 0, requiredCount: requiredOutputSubstrings.length, observedCount: requiredOutputSubstrings.length - missingRequired.length, missingHashes: missingRequired.map((value) => crypto.createHash("sha256").update(value).digest("hex")) };
-  const resolved = agent.code === 0 && !agent.timedOut && grade.passed && graderIntegrity.passed && scope.passed && outputSafety.passed && outputEvidence.passed;
+  const outcome = graderInput ? finalizeProductionV3SessionOutcome({ attemptId, input: graderInput, grade, usage }) : null;
+  const outcomeFields = outcome ? benchmarkReportOutcomeFields(outcome) : null;
+  const resolved = agent.code === 0 && !agent.timedOut && grade.passed && graderIntegrity.passed && scope.passed && outputSafety.passed && outputEvidence.passed && !independentFailure && !codexOutcomeFailure;
   const abortSuite = Boolean(preUsageFailure);
+  const promptBinding = journeyTurns
+    ? JSON.stringify(journeyTurns.map((turn) => ({ id: turn.id, message: turn.message, reconnectBefore: turn.reconnectBefore,
+      receiptUncertain: turn.receiptUncertain, workflow: turn.workflow ?? null })))
+    : prompt;
   const record = {
     schemaVersion: 1, runId, attemptId, configurationDigest, orderIndex, scenarioId: scenario.id, scenarioTitle: scenario.title, scenarioKind: scenario.kind,
     category: scenario.category ?? "unspecified", difficulty: scenario.difficulty ?? "unspecified", profile, lifecycle,
+    ...(scenario.familyId ? { familyId: scenario.familyId, variantId: scenario.variantId, variantRole: scenario.variantRole } : {}),
+    ingress: journeyTurns ? (surface === "piagent" ? "webui-gateway" : surface === "codex-cli" ? "codex-cli-resume" : "terminal") : "terminal",
     surface, repeat, infrastructureAttempt, sessionId, providerSessionId: usage.providerSessionId ?? null, abortSuite,
     infrastructureFailure: preUsageFailure?.failure, infrastructureClass: preUsageFailure?.class,
     infrastructureRetryable: preUsageFailure?.retryable,
     usageStatus: preUsageFailure?.usageStatus ?? "measured",
+    ...(fatalProviderBoundaryPhase ? { providerBoundaryPhase: fatalProviderBoundaryPhase } : {}),
     infrastructureDiagnostic: abortSuite ? safeInfrastructureDiagnostic(diagnosticInput, [...forbiddenOutputSubstrings, piRuntimeHome?.path].filter(Boolean)) : undefined,
     infrastructureDiagnosticSource: abortSuite ? (codexDiagnostics.length > 0 ? "codex-error-events" : piTerminalError ? "pi-terminal-error-event" : "process-output-tail") : undefined,
-    resolved, failure: preUsageFailure?.failure ?? failureReason({ agent, grade, graderIntegrity, outsideScope, forbiddenHits, missingRequired }),
+    resolved, failure: preUsageFailure?.failure ?? codexOutcomeFailure?.reason ?? candidateOutcomeFailure ?? independentFailure
+      ?? failureReason({ agent, grade, graderIntegrity, outsideScope, forbiddenHits, missingRequired }),
     agent: { exitCode: agent.code, signal: agent.signal, timedOut: agent.timedOut, stdoutHash: agent.stdoutHash ?? crypto.createHash("sha256").update(agent.stdout).digest("hex"), stderrHash: crypto.createHash("sha256").update(agent.stderr ?? "").digest("hex") },
-    grade, graderIntegrity, scope, outputSafety, outputEvidence, workflow, providerWireEvidence, causalContextReceipt, usage,
+    grade, graderIntegrity, scope, outputSafety, outputEvidence, safetyEvidence, workflow, providerWireEvidence, causalContextReceipt, usage,
+    ...(outcome ? { outcome, ...outcomeFields } : {}),
+    ...(wireInvocation ? { providerWirePhaseEvidence: readBenchmarkWireReceipts(wireInvocation) } : {}),
+    ...(independentVerification ? { independentVerification } : {}),
     durationSeconds: agent.durationSeconds, timingDiagnostics,
-    promptHash: crypto.createHash("sha256").update(prompt).digest("hex"),
+    promptHash: crypto.createHash("sha256").update(promptBinding).digest("hex"),
+    ...(journeyReceipt ? { journeyReceipt: persistedJourneyReceipt(journeyReceipt) } : {}),
     variant: scenario.variantGenerator ? { generated: true, seedDigest: variant.seedDigest, oracleDigest: variant.oracleDigest, fixtureDigest } : { generated: false, fixtureDigest }
   };
   const workflowFailed = surface === "piagent" && (workflow?.checks ?? []).some((check) => check.passed === false);
   if (!record.abortSuite && !interrupted()) persistCompletedRecord(record);
-  return { record, workspaceRoot, key, workflowFailed, inflightPath };
+  return { record, workspaceRoot, key, workflowFailed, inflightPath,
+    ...(fatalProviderBoundaryError ? { fatalProviderBoundaryError, fatalProviderBoundaryPhase } : {}) };
+}
+
+export function runBenchmarkSession(input) { return runBenchmarkSessionInternal(input, false); }
+
+/** Explicit non-production seam for deterministic offline plumbing tests. It cannot be selected
+ * through the benchmark CLI or the production runner. */
+export function runOfflineBenchmarkSession(input) {
+  if (typeof input?.piagentWebUiJourney !== "function") fail("Offline benchmark session requires an explicit fake Piagent WebUI journey");
+  return runBenchmarkSessionInternal(input, true);
 }