@piagent/platform 1.6.0 → 1.7.1

package/packages/piagent-core/benchmark/benchmark-summary-support.js

@@ -30,12 +30,18 @@ export const RELEASE_FAILURE_MESSAGES = Object.freeze({
   "efficiency-category-coverage": "Comparable token evidence does not cover every required category.",
   "failure-aware-efficiency": "Tokens per resolved outcome exceed the suite limit or failed-attempt usage is unknown.",
   "primary-efficiency": "The predeclared primary efficiency estimand lacks complete family/category evidence or its upper 95% ratio exceeds the suite limit.",
+  "codex-relative-efficiency": "The fixed-workload Piagent/Codex CLI comparison lacks exact parity or quality evidence, exceeds 0.70 total token traffic or API-equivalent cost at the point/upper-95 bound, or exceeds the subagent budget.",
+  "production-subagent-budget": "Production Piagent attempts lack exact child-session accounting, exceed one subagent per attempt, or spend more than 5% of candidate provider traffic on subagents.",
+  "provider-free-evidence": "The production run lacks same-source clean-commit architecture, runtime-conformance, long-horizon, or WebUI-parity evidence bound to exact lane configuration and runner bytes.",
+  "adaptive-context-runtime-coverage": "At least one Piagent runtime receipt reports partial adaptive-context evidence, or not-observed evidence lacks a same-source provider-free coverage lane.",
   "repeat-count": "The run used fewer repeats than the suite minimum.",
   "efficiency-confidence": "The upper 95% token-ratio bound exceeds the suite limit.",
   "efficiency-band-ratio": "At least one category, profile, lifecycle, or difficulty fresh-token ratio exceeds the suite limit.",
   "efficiency-family-ratio": "At least one comparable scenario family uses more fresh tokens than the suite family guardrail permits.",
   "accepted-usage-completeness": "Every accepted benchmark attempt must have exact terminal token buckets.",
   "all-attempt-usage-completeness": "Every accepted or failed provider attempt must have exact invariant-checked token buckets.",
+  "all-attempt-net-efficiency": "All accepted plus exact failed/retry fresh tokens must remain at or below the predeclared pooled Piagent/Codex CLI ratio.",
+  "fast-execution-configuration-parity": "Fast execution configuration must be verified on every run: exact Piagent outbound telemetry plus identity-bound effective Codex CLI thread settings; missing or default evidence blocks the claim.",
   "normalized-cost-configuration": "Normalized-cost claims require a valid versioned pricing snapshot and all predeclared ratio thresholds.",
   "normalized-cost-evidence": "Normalized API-equivalent text-token cost evidence is incomplete for at least one declared scenario family.",
   "normalized-cost-pricing-applicability": "Exact usage or per-request pricing applicability is unavailable for at least one accepted run.",
@@ -50,7 +56,7 @@ export const RELEASE_FAILURE_MESSAGES = Object.freeze({
   "infrastructure-retries": "Accepted benchmark runs used more recovered infrastructure retries than the suite permits.",
   "infrastructure-failure-ledger": "Accepted benchmark runs do not contain an exact attempt/retry/failure ledger.",
   "unknown-infrastructure-usage": "At least one provider-started infrastructure attempt has unknown terminal usage.",
-  "canonical-production-identity": "The production-v1 suite id is reserved for the canonical built-in suite identity.",
+  "canonical-production-identity": "Production suite ids are reserved for their canonical built-in suite identities.",
   "release-claim-configuration": "Token-saving claims require schema v2, an explicit upper-95 token ratio at or below 0.80, full-suite enforcement, and provider-wire stability evidence.",
   "codex-baseline": "Token-saving product claims require controlled Codex CLI as the paired baseline.",
   "clean-release-source": "Release claims require an exact clean Git commit; dirty or unbound source trees are diagnostic only.",
@@ -374,3 +380,29 @@ export function pairedQualityNoninferiorityEvidence({ suite, repeats, baselineRu
     families
   };
 }
+
+export function conditionalBothPassFreshUsage({ suite, repeats, allPairs, pairs, tokenPairs, baselineRuns, candidateRuns, allSuccessfulPairsFreshRatio }) {
+  return {
+    outcomeDefinition: "both-accepted-runs-resolved-including-safety-refusals",
+    usageDefinition: "exact-positive-fresh-same-model-and-thinking",
+    attemptScope: "accepted-attempts-only; all-attempt-costs-remain-in-tokenAccounting",
+    estimator: "geometric-mean-of-candidate-over-baseline-pair-ratios",
+    expectedPairs: suite.scenarios.length * repeats,
+    observedPairs: allPairs.length,
+    bothResolvedPairs: pairs.length,
+    eligiblePairs: tokenPairs.length,
+    excludedUnresolvedPairs: allPairs.length - pairs.length,
+    excludedUsagePairs: pairs.length - tokenPairs.length,
+    unpairedAcceptedRuns: baselineRuns.length + candidateRuns.length - 2 * allPairs.length,
+    ratio: rounded(allSuccessfulPairsFreshRatio, 4),
+    interpretation: "conditional-observation-only; not-overall-token-savings"
+  };
+}
+
+export function benchmarkOutcomePairs(baselineRuns, candidateRuns) {
+  const baselineByKey = new Map(baselineRuns.map((run) => [`${run.scenarioId}:${run.repeat}`, run]));
+  const allPairs = candidateRuns
+    .map((run) => ({ baseline: baselineByKey.get(`${run.scenarioId}:${run.repeat}`), candidate: run }))
+    .filter((pair) => pair.baseline);
+  return allPairs;
+}

package/packages/piagent-core/benchmark/benchmark-timing-contract.js

@@ -0,0 +1,189 @@
+import { median, rounded } from "./benchmark-statistics.js";
+
+export const BENCHMARK_TIMING_PHASES = Object.freeze([
+  "processStartup", "modelTurnWait", "toolExecution", "other"
+]);
+export const BENCHMARK_EXECUTION_COUNTER_FIELDS = Object.freeze([
+  "providerStartedAttempts", "toolCalls", "toolResults", "toolFailures", "blockedToolCalls",
+  "declinedToolCalls", "explicitRetries", "retryFailures", "compactions", "abortedCompactions",
+  "summarizationRetries", "repeatedToolCalls", "subagentAttempts", "subagentFailures"
+]);
+
+function plainObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function finiteNonnegative(value) {
+  return Number.isFinite(value) && value >= 0;
+}
+
+function validPhase(value, name) {
+  const expectedBoundary = {
+    processStartup: "process-start-to-turn-start",
+    modelTurnWait: "request-ready-to-next-model-output-boundary",
+    toolExecution: "matched-tool-start-to-tool-completion",
+    other: "process-duration-minus-available-observed-phases"
+  }[name];
+  const unavailableReasons = {
+    processStartup: new Set(["invalid-event-stream", "missing-turn-start-boundary", "unavailable"]),
+    modelTurnWait: new Set(["invalid-event-stream", "incomplete-model-turn-boundaries", "unavailable"]),
+    toolExecution: new Set(["invalid-event-stream", "incomplete-tool-boundaries", "unavailable"]),
+    other: new Set(["one-or-more-phase-boundaries-unavailable"])
+  }[name];
+  const expectedAvailableReason = name === "other" ? "exact-unattributed-remainder" : "observed-boundaries";
+  return plainObject(value)
+    && Object.keys(value).length === 4
+    && ["available", "unavailable"].includes(value.status)
+    && (value.status === "available" ? finiteNonnegative(value.seconds) : value.seconds === null)
+    && value.boundary === expectedBoundary
+    && (value.status === "available" ? value.reason === expectedAvailableReason : unavailableReasons.has(value.reason));
+}
+
+function validExecution(value, observations) {
+  const keys = ["schemaVersion", "source", "completeness", ...BENCHMARK_EXECUTION_COUNTER_FIELDS];
+  if (!plainObject(value) || Object.keys(value).length !== keys.length || !keys.every((key) => Object.hasOwn(value, key))
+    || value.schemaVersion !== 1 || value.source !== "stdout-jsonl-lifecycle"
+    || !plainObject(value.completeness)) return false;
+  const completenessKeys = ["providerAttempts", "tools", "retries", "compactions", "subagents"];
+  if (Object.keys(value.completeness).length !== completenessKeys.length
+    || !completenessKeys.every((key) => Object.hasOwn(value.completeness, key))) return false;
+  if (!["exact", "lower-bound", "partial", "unavailable"].includes(value.completeness.providerAttempts)
+    || !["exact", "partial", "unavailable"].includes(value.completeness.tools)
+    || !["exact", "partial", "unavailable"].includes(value.completeness.retries)
+    || !["exact", "partial", "unavailable"].includes(value.completeness.compactions)
+    || !["exact", "partial", "unavailable"].includes(value.completeness.subagents)) return false;
+  if (!BENCHMARK_EXECUTION_COUNTER_FIELDS.every((field) => Number.isSafeInteger(value[field]) && value[field] >= 0)) return false;
+  if (value.providerStartedAttempts !== observations.turnStarts || value.toolCalls < observations.toolStarts
+    || value.toolResults !== observations.toolCompletions || value.toolFailures > value.toolResults
+    || value.blockedToolCalls > value.toolResults || value.declinedToolCalls > value.toolResults
+    || value.repeatedToolCalls >= value.toolCalls && value.toolCalls > 0
+    || value.subagentFailures > value.subagentAttempts || value.retryFailures > value.explicitRetries
+    || value.abortedCompactions > value.compactions) return false;
+  if (value.completeness.tools === "exact" && value.toolCalls !== value.toolResults) return false;
+  return true;
+}
+
+export function validBenchmarkTimingDiagnostics(value, surface, durationSeconds) {
+  const topLevelKeys = ["schemaVersion", "authority", "gateImpact", "clock", "surface", "status", "processDurationSeconds", "phases", "observations", "execution", "privacy"];
+  if (!plainObject(value) || Object.keys(value).length !== topLevelKeys.length || !topLevelKeys.every((key) => Object.hasOwn(value, key))
+    || value.schemaVersion !== 2 || value.authority !== "observational-only"
+    || value.gateImpact !== "none" || value.clock !== "benchmark-process-monotonic-receipt"
+    || value.surface !== surface || !["complete", "partial", "unavailable"].includes(value.status)
+    || !finiteNonnegative(value.processDurationSeconds)
+    || !plainObject(value.phases) || Object.keys(value.phases).length !== BENCHMARK_TIMING_PHASES.length
+    || !BENCHMARK_TIMING_PHASES.every((name) => validPhase(value.phases[name], name))) return false;
+  if (finiteNonnegative(durationSeconds) && Math.abs(value.processDurationSeconds - durationSeconds) > 1e-5) return false;
+  const observationKeys = ["jsonEvents", "sessionHeaders", "agentStarts", "agentEnds", "agentSettled", "threadStarts",
+    "turnStarts", "turnCompletions", "matchedModelTurnIntervals", "toolStarts", "toolCompletions", "matchedToolIntervals",
+    "unmatchedToolStarts", "unmatchedToolCompletions", "unmatchedModelTurnStarts", "malformedLines", "invalidClockBoundaries", "coalescedBoundaries"];
+  if (!plainObject(value.observations) || Object.keys(value.observations).length !== observationKeys.length
+    || !observationKeys.every((key) => Number.isSafeInteger(value.observations[key]) && value.observations[key] >= 0)) return false;
+  const observations = value.observations;
+  if (!validExecution(value.execution, observations)) return false;
+  const lifecycleEvents = surface === "codex-cli"
+    ? observations.threadStarts
+    : observations.sessionHeaders + observations.agentStarts + observations.agentEnds + observations.agentSettled;
+  if ([observations.sessionHeaders, observations.agentSettled, observations.threadStarts]
+    .some((count) => count > 1)
+    || (surface === "codex-cli"
+      ? observations.sessionHeaders + observations.agentStarts + observations.agentEnds + observations.agentSettled !== 0
+      : observations.threadStarts !== 0 || observations.agentEnds > observations.agentStarts)
+    || observations.turnCompletions > observations.turnStarts
+    || (surface !== "codex-cli" && observations.matchedModelTurnIntervals > observations.turnStarts)
+    || observations.matchedToolIntervals > observations.toolStarts
+    || observations.matchedToolIntervals > observations.toolCompletions
+    || observations.matchedModelTurnIntervals > observations.jsonEvents
+    || (surface === "codex-cli" && (observations.turnStarts > 1 || observations.turnCompletions > 1
+      || observations.matchedModelTurnIntervals > observations.toolStarts + observations.turnCompletions))
+    || lifecycleEvents + observations.turnStarts + observations.turnCompletions
+      + observations.toolStarts + observations.toolCompletions > observations.jsonEvents) return false;
+  const surfaceHeaderObserved = surface === "codex-cli" ? observations.threadStarts === 1 : observations.sessionHeaders === 1;
+  const lifecycleComplete = surface === "codex-cli"
+    ? surfaceHeaderObserved && observations.turnStarts === 1 && observations.turnCompletions === 1
+    : surfaceHeaderObserved && observations.agentStarts > 0 && observations.agentEnds === observations.agentStarts
+      && observations.agentSettled === 1 && observations.turnStarts > 0 && observations.turnCompletions === observations.turnStarts;
+  const privacyKeys = ["rawPayloadStored", "promptsStored", "commandsStored", "pathsStored", "identifiersStored"];
+  if (!plainObject(value.privacy) || Object.keys(value.privacy).length !== privacyKeys.length
+    || !privacyKeys.every((key) => value.privacy[key] === false)) return false;
+  const available = BENCHMARK_TIMING_PHASES.filter((name) => value.phases[name].status === "available").length;
+  const expectedStatus = available === BENCHMARK_TIMING_PHASES.length ? "complete" : available > 0 ? "partial" : "unavailable";
+  if (value.status !== expectedStatus) return false;
+  if (BENCHMARK_TIMING_PHASES.some((name) => value.phases[name].status === "available" && value.phases[name].seconds > value.processDurationSeconds + 1e-6)) return false;
+  if ((value.observations.malformedLines > 0 || value.observations.invalidClockBoundaries > 0) && available > 0) return false;
+  if (value.status === "complete" && ["unmatchedToolStarts", "unmatchedToolCompletions", "unmatchedModelTurnStarts",
+    "malformedLines", "invalidClockBoundaries", "coalescedBoundaries"].some((key) => value.observations[key] > 0)) return false;
+  if (value.status === "complete" && (!lifecycleComplete
+    || observations.matchedModelTurnIntervals === 0
+    || (surface !== "codex-cli" && observations.matchedModelTurnIntervals !== observations.turnStarts)
+    || observations.toolStarts !== observations.toolCompletions
+    || observations.matchedToolIntervals !== observations.toolStarts)) return false;
+  if (value.phases.processStartup.status === "available" && (!surfaceHeaderObserved || observations.turnStarts === 0
+    || surface !== "codex-cli" && observations.agentStarts === 0)) return false;
+  if (value.phases.modelTurnWait.status === "available" && (!lifecycleComplete
+    || observations.matchedModelTurnIntervals === 0
+    || (surface !== "codex-cli" && observations.matchedModelTurnIntervals !== observations.turnStarts)
+    || observations.unmatchedModelTurnStarts > 0)) return false;
+  if (value.phases.toolExecution.status === "available" && (!lifecycleComplete
+    || observations.toolStarts !== observations.toolCompletions
+    || observations.matchedToolIntervals !== observations.toolStarts
+    || observations.unmatchedToolStarts > 0 || observations.unmatchedToolCompletions > 0)) return false;
+  const attributedPhasesAvailable = ["processStartup", "modelTurnWait", "toolExecution"]
+    .every((name) => value.phases[name].status === "available");
+  if ((value.phases.other.status === "available") !== attributedPhasesAvailable) return false;
+  if (value.phases.other.status === "available") {
+    const sum = BENCHMARK_TIMING_PHASES.reduce((total, name) => total + value.phases[name].seconds, 0);
+    if (Math.abs(sum - value.processDurationSeconds) > 5e-5) return false;
+  }
+  return true;
+}
+
+export function canonicalBenchmarkTimingDiagnostics(value, surface, durationSeconds) {
+  try {
+    return validBenchmarkTimingDiagnostics(value, surface, durationSeconds) ? value : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export function summarizeBenchmarkTimingDiagnostics(runs) {
+  const values = Array.isArray(runs) ? runs : [];
+  const surfaces = {};
+  for (const surface of [...new Set(values.map((run) => run?.surface).filter((value) => typeof value === "string"))].sort()) {
+    const surfaceRuns = values.filter((run) => run?.surface === surface);
+    const valid = surfaceRuns.filter((run) => canonicalBenchmarkTimingDiagnostics(run.timingDiagnostics, surface, run.durationSeconds));
+    const phaseSummary = Object.fromEntries(BENCHMARK_TIMING_PHASES.map((name) => {
+      const available = valid.map((run) => run.timingDiagnostics.phases[name]).filter((item) => item.status === "available");
+      return [name, {
+        availableRuns: available.length,
+        unavailableRuns: surfaceRuns.length - available.length,
+        medianSeconds: available.length > 0 ? rounded(median(available.map((item) => item.seconds)), 6) : null
+      }];
+    }));
+    surfaces[surface] = {
+      runs: surfaceRuns.length,
+      validDiagnostics: valid.length,
+      completeDiagnostics: valid.filter((run) => run.timingDiagnostics.status === "complete").length,
+      execution: {
+        coveredRuns: valid.length,
+        unavailableRuns: surfaceRuns.length - valid.length,
+        counters: Object.fromEntries(BENCHMARK_EXECUTION_COUNTER_FIELDS.map((field) => [field,
+          valid.reduce((sum, run) => sum + run.timingDiagnostics.execution[field], 0)])),
+        exactRuns: {
+          providerAttempts: valid.filter((run) => run.timingDiagnostics.execution.completeness.providerAttempts === "exact").length,
+          tools: valid.filter((run) => run.timingDiagnostics.execution.completeness.tools === "exact").length,
+          retries: valid.filter((run) => run.timingDiagnostics.execution.completeness.retries === "exact").length,
+          compactions: valid.filter((run) => run.timingDiagnostics.execution.completeness.compactions === "exact").length,
+          subagents: valid.filter((run) => run.timingDiagnostics.execution.completeness.subagents === "exact").length
+        }
+      },
+      phases: phaseSummary
+    };
+  }
+  return {
+    schemaVersion: 2,
+    authority: "observational-only",
+    gateImpact: "none",
+    clock: "benchmark-process-monotonic-receipt",
+    surfaces
+  };
+}

package/packages/piagent-core/benchmark/benchmark-timing-diagnostics.js

@@ -1,6 +1,16 @@
 import { StringDecoder } from "node:string_decoder";
 
-import { median, rounded } from "./benchmark-statistics.js";
+import { rounded } from "./benchmark-statistics.js";
+import {
+  BENCHMARK_EXECUTION_COUNTER_FIELDS as EXECUTION_COUNTER_FIELDS,
+  BENCHMARK_TIMING_PHASES as PHASES
+} from "./benchmark-timing-contract.js";
+
+export {
+  canonicalBenchmarkTimingDiagnostics,
+  summarizeBenchmarkTimingDiagnostics,
+  validBenchmarkTimingDiagnostics
+} from "./benchmark-timing-contract.js";
 
 const CODEX_MODEL_OUTPUT_ITEMS = new Set(["agent_message", "reasoning"]);
 const CODEX_CONTROL_ITEMS = new Set(["todo_list"]);
@@ -22,7 +32,6 @@ const PI_CONTROL_EVENTS = new Set([
 ]);
 const PI_MESSAGE_ROLES = new Set(["user", "assistant", "toolResult", "custom"]);
 const SURFACES = new Set(["raw-pi", "piagent", "codex-cli"]);
-const PHASES = ["processStartup", "modelTurnWait", "toolExecution", "other"];
 const MAX_JSONL_BYTES = 64 * 1024 * 1024;
 const MAX_TIMING_REPLAY_BYTES = 4 * 1024 * 1024;
 
@@ -30,6 +39,51 @@ function plainObject(value) {
   return Boolean(value) && typeof value === "object" && !Array.isArray(value);
 }
 
+function stableValue(value) {
+  if (Array.isArray(value)) return value.map(stableValue);
+  if (!plainObject(value)) return value;
+  return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableValue(value[key])]));
+}
+
+function executionFingerprint(name, args) {
+  try { return JSON.stringify([name, stableValue(args)]); }
+  catch { return String(name); }
+}
+
+function piSubagentAttempt(toolName) {
+  const name = String(toolName ?? "").toLowerCase();
+  return ["spawn_agent", "create_agent", "delegate_task", "subagent"].includes(name);
+}
+
+function codexExecutionIdentity(item) {
+  if (item.type === "command_execution") return item.command;
+  if (item.type === "mcp_tool_call") return [item.server, item.tool, item.arguments];
+  if (item.type === "collab_tool_call") return [item.tool, item.prompt];
+  if (item.type === "web_search") return item.action ?? item.query;
+  return item.type;
+}
+
+function structuredStatus(value) {
+  const status = [value?.status, value?.details?.status, value?.result?.status]
+    .find((candidate) => typeof candidate === "string");
+  return status?.toLowerCase() ?? "";
+}
+
+function emptyExecution(source) {
+  return {
+    schemaVersion: 1,
+    source,
+    completeness: {
+      providerAttempts: "unavailable",
+      tools: "unavailable",
+      retries: "unavailable",
+      compactions: "unavailable",
+      subagents: "unavailable"
+    },
+    ...Object.fromEntries(EXECUTION_COUNTER_FIELDS.map((field) => [field, 0]))
+  };
+}
+
 function finiteNonnegative(value) {
   return Number.isFinite(value) && value >= 0;
 }
@@ -145,6 +199,7 @@ function consumePiEvent(state, event, at) {
       return;
     }
     state.turnStarts += 1;
+    state.execution.providerStartedAttempts += 1;
     if (state.startupBoundary === null) state.startupBoundary = at;
     state.turnOpen = true;
     state.assistantSeenInTurn = false;
@@ -243,8 +298,14 @@ function consumePiEvent(state, event, at) {
       return;
     }
     state.toolStarts += 1;
+    state.execution.toolCalls += 1;
     state.turnToolStarts += 1;
-    state.openTools.set(id, { at, toolName: event.toolName });
+    const subagentAttempt = piSubagentAttempt(event.toolName);
+    const fingerprint = executionFingerprint(event.toolName, event.args);
+    if (state.toolFingerprints.has(fingerprint)) state.execution.repeatedToolCalls += 1;
+    else state.toolFingerprints.add(fingerprint);
+    if (subagentAttempt) state.execution.subagentAttempts += 1;
+    state.openTools.set(id, { at, toolName: event.toolName, subagentAttempt });
     return;
   }
   if (event.type === "tool_execution_end") {
@@ -255,8 +316,15 @@ function consumePiEvent(state, event, at) {
       return;
     }
     state.toolCompletions += 1;
+    state.execution.toolResults += 1;
     state.turnToolCompletions += 1;
     const started = state.openTools.get(id);
+    const status = structuredStatus(event.result);
+    const failed = event.isError === true || status === "failed" || status === "error";
+    if (failed) state.execution.toolFailures += 1;
+    if (status === "blocked") state.execution.blockedToolCalls += 1;
+    if (status === "declined") state.execution.declinedToolCalls += 1;
+    if (started?.subagentAttempt && (failed || status === "blocked" || status === "declined")) state.execution.subagentFailures += 1;
     if (started === undefined || started.toolName !== event.toolName || state.completedTurnTools.has(id)) {
       if (started !== undefined && started.toolName !== event.toolName) invalidateEvent(state);
       state.unmatchedToolCompletions += 1;
@@ -333,6 +401,11 @@ function consumePiEvent(state, event, at) {
   }
   if (PI_CONTROL_EVENTS.has(event.type)) {
     if (!validPiControlEvent(event)) invalidateEvent(state);
+    else if (event.type === "auto_retry_start") state.execution.explicitRetries += 1;
+    else if (event.type === "auto_retry_end" && event.success === false) state.execution.retryFailures += 1;
+    else if (event.type === "compaction_start") state.execution.compactions += 1;
+    else if (event.type === "compaction_end" && event.aborted === true) state.execution.abortedCompactions += 1;
+    else if (event.type === "summarization_retry_scheduled") state.execution.summarizationRetries += 1;
     return;
   }
   invalidateEvent(state);
@@ -459,6 +532,7 @@ function consumeCodexEvent(state, event, at) {
       return;
     }
     state.turnStarts += 1;
+    state.execution.providerStartedAttempts += 1;
     if (state.startupBoundary === null) state.startupBoundary = at;
     state.turnOpen = true;
     state.modelTurnStart = at;
@@ -536,7 +610,13 @@ function consumeCodexEvent(state, event, at) {
     state.seenCodexItemIds.set(id, event.item.type);
     closeCodexModelTurnInterval(state, at);
     state.toolStarts += 1;
-    state.openTools.set(id, { at, type: event.item.type });
+    state.execution.toolCalls += 1;
+    const fingerprint = executionFingerprint(event.item.type, codexExecutionIdentity(event.item));
+    if (state.toolFingerprints.has(fingerprint)) state.execution.repeatedToolCalls += 1;
+    else state.toolFingerprints.add(fingerprint);
+    const subagentAttempt = event.item.type === "collab_tool_call" && event.item.tool === "spawn_agent";
+    if (subagentAttempt) state.execution.subagentAttempts += 1;
+    state.openTools.set(id, { at, type: event.item.type, subagentAttempt });
     return;
   }
   if (kind === "control") {
@@ -554,6 +634,9 @@ function consumeCodexEvent(state, event, at) {
   if (kind === "completed-only-control") return;
   if (kind === "completed-only-tool") {
     state.toolCompletions += 1;
+    state.execution.toolCalls += 1;
+    state.execution.toolResults += 1;
+    if (event.item.status === "failed") state.execution.toolFailures += 1;
     state.unmatchedToolCompletions += 1;
     state.modelTurnIntervalsInvalid = true;
     state.toolIntervalsInvalid = true;
@@ -561,6 +644,7 @@ function consumeCodexEvent(state, event, at) {
   }
   if (kind === "model-output") return;
   state.toolCompletions += 1;
+  state.execution.toolResults += 1;
   const id = safeId(event.item.id);
   if (!id) {
     invalidateEvent(state);
@@ -569,6 +653,13 @@ function consumeCodexEvent(state, event, at) {
   }
   const seenType = state.seenCodexItemIds.get(id);
   const started = state.openTools.get(id);
+  const status = typeof event.item.status === "string" ? event.item.status.toLowerCase() : "";
+  const failed = status === "failed" || event.item.type === "command_execution"
+    && Number.isSafeInteger(event.item.exit_code) && event.item.exit_code !== 0;
+  if (failed) state.execution.toolFailures += 1;
+  if (status === "blocked") state.execution.blockedToolCalls += 1;
+  if (status === "declined") state.execution.declinedToolCalls += 1;
+  if (started?.subagentAttempt && (failed || status === "blocked" || status === "declined")) state.execution.subagentFailures += 1;
   let matched = false;
   if (started === undefined) {
     if (seenType !== undefined) invalidateEvent(state);
@@ -607,7 +698,7 @@ export function unavailableBenchmarkTimingDiagnostics(surface, durationSeconds)
   if (!SURFACES.has(surface)) throw new Error(`Unsupported benchmark timing surface: ${surface}`);
   const processDurationSeconds = finiteNonnegative(durationSeconds) ? rounded(durationSeconds, 6) : null;
   return {
-    schemaVersion: 1,
+    schemaVersion: 2,
     authority: "observational-only",
     gateImpact: "none",
     clock: "benchmark-process-monotonic-receipt",
@@ -627,6 +718,7 @@ export function unavailableBenchmarkTimingDiagnostics(surface, durationSeconds)
       unmatchedToolStarts: 0, unmatchedToolCompletions: 0, unmatchedModelTurnStarts: 0,
       malformedLines: 1, invalidClockBoundaries: 0, coalescedBoundaries: 0
     },
+    execution: emptyExecution("stdout-jsonl-lifecycle"),
     privacy: { rawPayloadStored: false, promptsStored: false, commandsStored: false, pathsStored: false, identifiersStored: false }
   };
 }
@@ -663,8 +755,17 @@ function finish(state, durationSeconds) {
   const otherSeconds = otherAvailable ? Math.max(0, durationSeconds - attributed) : null;
   const availablePhases = [startupAvailable, modelTurnAvailable, toolAvailable, otherAvailable].filter(Boolean).length;
   const status = availablePhases === PHASES.length ? "complete" : availablePhases > 0 ? "partial" : "unavailable";
+  const executionComplete = state.malformedLines === 0 && !state.providerErrorSeen && lifecycleComplete;
+  state.execution.completeness.providerAttempts = executionComplete
+    ? state.surface === "codex-cli" ? "lower-bound" : "exact"
+    : "partial";
+  state.execution.completeness.tools = executionComplete && state.unmatchedToolStarts === 0
+    && state.unmatchedToolCompletions === 0 ? "exact" : "partial";
+  state.execution.completeness.retries = executionComplete && state.surface !== "codex-cli" ? "exact" : "unavailable";
+  state.execution.completeness.compactions = executionComplete && state.surface !== "codex-cli" ? "exact" : "unavailable";
+  state.execution.completeness.subagents = state.execution.completeness.tools;
   return {
-    schemaVersion: 1,
+    schemaVersion: 2,
     authority: "observational-only",
     gateImpact: "none",
     clock: "benchmark-process-monotonic-receipt",
@@ -705,6 +806,7 @@ function finish(state, durationSeconds) {
       invalidClockBoundaries: state.invalidClockBoundaries,
       coalescedBoundaries: state.coalescedBoundaries
     },
+    execution: state.execution,
     privacy: {
       rawPayloadStored: false,
       promptsStored: false,
@@ -730,6 +832,8 @@ export function createBenchmarkTimingCollector({ surface }) {
     toolIntervals: [],
     toolIntervalsInvalid: false,
     openTools: new Map(),
+    toolFingerprints: new Set(),
+    execution: emptyExecution("stdout-jsonl-lifecycle"),
     completedTurnTools: new Map(),
     turnToolResultIdentities: new Map(),
     openToolResultIdentity: null,
@@ -836,136 +940,3 @@ export function createDeferredBenchmarkTimingCollector({
     }
   };
 }
-
-function validPhase(value, name) {
-  const expectedBoundary = {
-    processStartup: "process-start-to-turn-start",
-    modelTurnWait: "request-ready-to-next-model-output-boundary",
-    toolExecution: "matched-tool-start-to-tool-completion",
-    other: "process-duration-minus-available-observed-phases"
-  }[name];
-  const unavailableReasons = {
-    processStartup: new Set(["invalid-event-stream", "missing-turn-start-boundary", "unavailable"]),
-    modelTurnWait: new Set(["invalid-event-stream", "incomplete-model-turn-boundaries", "unavailable"]),
-    toolExecution: new Set(["invalid-event-stream", "incomplete-tool-boundaries", "unavailable"]),
-    other: new Set(["one-or-more-phase-boundaries-unavailable"])
-  }[name];
-  const expectedAvailableReason = name === "other" ? "exact-unattributed-remainder" : "observed-boundaries";
-  return plainObject(value)
-    && Object.keys(value).length === 4
-    && ["available", "unavailable"].includes(value.status)
-    && (value.status === "available" ? finiteNonnegative(value.seconds) : value.seconds === null)
-    && value.boundary === expectedBoundary
-    && (value.status === "available" ? value.reason === expectedAvailableReason : unavailableReasons.has(value.reason));
-}
-
-export function validBenchmarkTimingDiagnostics(value, surface, durationSeconds) {
-  const topLevelKeys = ["schemaVersion", "authority", "gateImpact", "clock", "surface", "status", "processDurationSeconds", "phases", "observations", "privacy"];
-  if (!plainObject(value) || Object.keys(value).length !== topLevelKeys.length || !topLevelKeys.every((key) => Object.hasOwn(value, key))
-    || value.schemaVersion !== 1 || value.authority !== "observational-only"
-    || value.gateImpact !== "none" || value.clock !== "benchmark-process-monotonic-receipt"
-    || value.surface !== surface || !["complete", "partial", "unavailable"].includes(value.status)
-    || !finiteNonnegative(value.processDurationSeconds)
-    || !plainObject(value.phases) || Object.keys(value.phases).length !== PHASES.length
-    || !PHASES.every((name) => validPhase(value.phases[name], name))) return false;
-  if (finiteNonnegative(durationSeconds) && Math.abs(value.processDurationSeconds - durationSeconds) > 1e-5) return false;
-  const observationKeys = ["jsonEvents", "sessionHeaders", "agentStarts", "agentEnds", "agentSettled", "threadStarts",
-    "turnStarts", "turnCompletions", "matchedModelTurnIntervals", "toolStarts", "toolCompletions", "matchedToolIntervals",
-    "unmatchedToolStarts", "unmatchedToolCompletions", "unmatchedModelTurnStarts", "malformedLines", "invalidClockBoundaries", "coalescedBoundaries"];
-  if (!plainObject(value.observations) || Object.keys(value.observations).length !== observationKeys.length
-    || !observationKeys.every((key) => Number.isSafeInteger(value.observations[key]) && value.observations[key] >= 0)) return false;
-  const observations = value.observations;
-  const lifecycleEvents = surface === "codex-cli"
-    ? observations.threadStarts
-    : observations.sessionHeaders + observations.agentStarts + observations.agentEnds + observations.agentSettled;
-  if ([observations.sessionHeaders, observations.agentSettled, observations.threadStarts]
-    .some((count) => count > 1)
-    || (surface === "codex-cli"
-      ? observations.sessionHeaders + observations.agentStarts + observations.agentEnds + observations.agentSettled !== 0
-      : observations.threadStarts !== 0 || observations.agentEnds > observations.agentStarts)
-    || observations.turnCompletions > observations.turnStarts
-    || (surface !== "codex-cli" && observations.matchedModelTurnIntervals > observations.turnStarts)
-    || observations.matchedToolIntervals > observations.toolStarts
-    || observations.matchedToolIntervals > observations.toolCompletions
-    || observations.matchedModelTurnIntervals > observations.jsonEvents
-    || (surface === "codex-cli" && (observations.turnStarts > 1 || observations.turnCompletions > 1
-      || observations.matchedModelTurnIntervals > observations.toolStarts + observations.turnCompletions))
-    || lifecycleEvents + observations.turnStarts + observations.turnCompletions
-      + observations.toolStarts + observations.toolCompletions > observations.jsonEvents) return false;
-  const surfaceHeaderObserved = surface === "codex-cli" ? observations.threadStarts === 1 : observations.sessionHeaders === 1;
-  const lifecycleComplete = surface === "codex-cli"
-    ? surfaceHeaderObserved && observations.turnStarts === 1 && observations.turnCompletions === 1
-    : surfaceHeaderObserved && observations.agentStarts > 0 && observations.agentEnds === observations.agentStarts
-      && observations.agentSettled === 1 && observations.turnStarts > 0 && observations.turnCompletions === observations.turnStarts;
-  const privacyKeys = ["rawPayloadStored", "promptsStored", "commandsStored", "pathsStored", "identifiersStored"];
-  if (!plainObject(value.privacy) || Object.keys(value.privacy).length !== privacyKeys.length
-    || !privacyKeys.every((key) => value.privacy[key] === false)) return false;
-  const available = PHASES.filter((name) => value.phases[name].status === "available").length;
-  const expectedStatus = available === PHASES.length ? "complete" : available > 0 ? "partial" : "unavailable";
-  if (value.status !== expectedStatus) return false;
-  if (PHASES.some((name) => value.phases[name].status === "available" && value.phases[name].seconds > value.processDurationSeconds + 1e-6)) return false;
-  if ((value.observations.malformedLines > 0 || value.observations.invalidClockBoundaries > 0) && available > 0) return false;
-  if (value.status === "complete" && ["unmatchedToolStarts", "unmatchedToolCompletions", "unmatchedModelTurnStarts",
-    "malformedLines", "invalidClockBoundaries", "coalescedBoundaries"].some((key) => value.observations[key] > 0)) return false;
-  if (value.status === "complete" && (!lifecycleComplete
-    || observations.matchedModelTurnIntervals === 0
-    || (surface !== "codex-cli" && observations.matchedModelTurnIntervals !== observations.turnStarts)
-    || observations.toolStarts !== observations.toolCompletions
-    || observations.matchedToolIntervals !== observations.toolStarts)) return false;
-  if (value.phases.processStartup.status === "available" && (!surfaceHeaderObserved || observations.turnStarts === 0
-    || surface !== "codex-cli" && observations.agentStarts === 0)) return false;
-  if (value.phases.modelTurnWait.status === "available" && (!lifecycleComplete
-    || observations.matchedModelTurnIntervals === 0
-    || (surface !== "codex-cli" && observations.matchedModelTurnIntervals !== observations.turnStarts)
-    || observations.unmatchedModelTurnStarts > 0)) return false;
-  if (value.phases.toolExecution.status === "available" && (!lifecycleComplete
-    || observations.toolStarts !== observations.toolCompletions
-    || observations.matchedToolIntervals !== observations.toolStarts
-    || observations.unmatchedToolStarts > 0 || observations.unmatchedToolCompletions > 0)) return false;
-  const attributedPhasesAvailable = ["processStartup", "modelTurnWait", "toolExecution"]
-    .every((name) => value.phases[name].status === "available");
-  if ((value.phases.other.status === "available") !== attributedPhasesAvailable) return false;
-  if (value.phases.other.status === "available") {
-    const sum = PHASES.reduce((total, name) => total + value.phases[name].seconds, 0);
-    if (Math.abs(sum - value.processDurationSeconds) > 5e-5) return false;
-  }
-  return true;
-}
-
-export function canonicalBenchmarkTimingDiagnostics(value, surface, durationSeconds) {
-  try {
-    return validBenchmarkTimingDiagnostics(value, surface, durationSeconds) ? value : undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-export function summarizeBenchmarkTimingDiagnostics(runs) {
-  const values = Array.isArray(runs) ? runs : [];
-  const surfaces = {};
-  for (const surface of [...new Set(values.map((run) => run?.surface).filter((value) => typeof value === "string"))].sort()) {
-    const surfaceRuns = values.filter((run) => run?.surface === surface);
-    const valid = surfaceRuns.filter((run) => canonicalBenchmarkTimingDiagnostics(run.timingDiagnostics, surface, run.durationSeconds));
-    const phaseSummary = Object.fromEntries(PHASES.map((name) => {
-      const available = valid.map((run) => run.timingDiagnostics.phases[name]).filter((item) => item.status === "available");
-      return [name, {
-        availableRuns: available.length,
-        unavailableRuns: surfaceRuns.length - available.length,
-        medianSeconds: available.length > 0 ? rounded(median(available.map((item) => item.seconds)), 6) : null
-      }];
-    }));
-    surfaces[surface] = {
-      runs: surfaceRuns.length,
-      validDiagnostics: valid.length,
-      completeDiagnostics: valid.filter((run) => run.timingDiagnostics.status === "complete").length,
-      phases: phaseSummary
-    };
-  }
-  return {
-    schemaVersion: 1,
-    authority: "observational-only",
-    gateImpact: "none",
-    clock: "benchmark-process-monotonic-receipt",
-    surfaces
-  };
-}