@piagent/platform 1.6.0 → 1.7.1

package/packages/piagent-core/benchmark/benchmark-goal-report.js

@@ -0,0 +1,87 @@
+import { assessProductionV3Goal } from "./benchmark-goal-assessment.js";
+
+const number = (value, digits = 2) => Number.isFinite(value) ? value.toFixed(digits) : "n/a";
+const outcome = value => value === true ? "PASS" : "FAIL";
+const html = value => String(value ?? "").replaceAll("&", "&amp;").replaceAll("<", "&lt;")
+  .replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#39;");
+const markdown = value => html(value).replaceAll("|", "&#124;").replaceAll("`", "&#96;")
+  .replaceAll("\\", "&#92;").replaceAll("\n", " ").replaceAll("\r", " ");
+
+// Bind only from the real frozen manifest and the re-read verified ledger.
+// No report-derived stand-ins for absent candidate/configuration/order bindings.
+export function attachBenchmarkGoalAssessment({ report, suite, manifest, verifiedLedgerRecords }) {
+  if (report?.suite?.id !== "production-v3") return report;
+  report.goalAssessment = assessProductionV3Goal({ report, suite, bindings: {
+    expectedOrder: manifest?.order,
+    expectedLedger: manifest?.ledger,
+    verifiedLedgerRecords,
+    expectedCandidateDigest: manifest?.candidateProvenance?.contentDigest,
+    expectedConfigurationDigest: manifest?.configurationDigest
+  } });
+  return report;
+}
+
+function tables(goal) {
+  const timing = value => `${number(value?.medianSeconds, 3)} / ${number(value?.p95Seconds, 3)}`;
+  const counted = (value, field) => `${value?.[field] ?? 0}/${value?.attempts ?? 0}`;
+  return [
+    { title: "All-attempt latency", headers: ["Surface", "Available / recorded", "Median seconds", "P95 seconds"],
+      rows: ["piagent", "codex-cli"].map(surface => {
+        const item = goal.latency?.[surface];
+        return [surface, `${item?.availableAttempts ?? 0}/${item?.attempts ?? 0}`,
+          number(item?.medianSeconds, 3), number(item?.p95Seconds, 3)];
+      }) },
+    { title: "Every scenario (both repeats retained)", headers: ["Scenario", "Pi fresh", "Codex fresh", "Fresh ratio",
+      "Resolved Pi / Codex", "Grade passes Pi / Codex", "Pi median / P95 s", "Codex median / P95 s", "Goal"],
+      rows: (goal.scenarios ?? []).map(item => [item.scenarioId,
+        number(item.usage?.piagent?.freshTokens, 0), number(item.usage?.["codex-cli"]?.freshTokens, 0),
+        number(item.freshTokenRatio, 6), `${counted(item.outcomes?.piagent, "resolved")} / ${counted(item.outcomes?.["codex-cli"], "resolved")}`,
+        `${counted(item.outcomes?.piagent, "gradePassed")} / ${counted(item.outcomes?.["codex-cli"], "gradePassed")}`,
+        timing(item.latency?.piagent), timing(item.latency?.["codex-cli"]), outcome(item.passed)]) },
+    { title: "Every matched scenario / repeat pair", headers: ["Scenario", "Repeat", "Fresh ratio", "Duration ratio",
+      "Quality", "Safety", "Workflow", "Goal", "Failures / missing evidence"],
+      rows: (goal.pairs ?? []).map(item => [item.scenarioId, item.repeat, number(item.freshTokenRatio, 6), number(item.durationRatio, 6),
+        outcome(item.qualityGate), outcome(item.safetyGate), outcome(item.workflowGate), item.complete ? outcome(item.passed) : "UNPROVEN",
+        [...(item.failedRequirements ?? []), ...(item.evidenceFailures ?? [])].join(", ") || "none"]) }
+  ];
+}
+
+function summary(goal) {
+  return [
+    `Stricter goal status: ${goal.status ?? "GOAL_UNPROVEN"}`,
+    `Observed requirements: ${goal.observedRequirementsPassed === true ? "PASS" : goal.complete === true ? "FAIL" : "UNPROVEN"}`,
+    `Goal claim eligible under existing gates: ${goal.claimAllowed === true ? "yes" : "no"}`,
+    `Existing production verdict (unchanged): ${goal.existingVerdict ?? "unavailable"}`,
+    `Passing pairs: ${goal.observed?.passingPairs ?? 0}/54; passing scenarios: ${goal.observed?.passingScenarios ?? 0}/27`,
+    "Every matched pair must use at most 0.65x fresh tokens and at most 1.00x wall time, without quality, safety, or workflow regression.",
+    "Latency includes valid failed attempts. P95 uses nearest-rank; two repeats per scenario do not establish future-task confidence.",
+    "Stock Codex has no Pi workflow score: compare native terminal outcomes and enforce Pi's additional required workflow continuity.",
+    `Measurement evidence failures: ${(goal.evidenceFailures ?? []).join(", ") || "none"}`,
+    `Upstream claim restrictions: ${(goal.claimFailures ?? []).join(", ") || "none"}`,
+    `Retained limitations: ${(goal.claimBoundary?.limitations ?? []).join(", ") || "none"}`,
+    "Scope: observed public-regression workload only; no universal, causal, private-generalization, or member-production claim."
+  ];
+}
+
+export function renderBenchmarkGoalMarkdown(goal) {
+  if (!goal) return "";
+  return ["## Stricter every-pair goal assessment", "", ...summary(goal).flatMap(line => [markdown(line), ""]),
+    ...tables(goal).flatMap(table => [`### ${table.title}`, "", `| ${table.headers.map(markdown).join(" | ")} |`,
+      `| ${table.headers.map(() => "---").join(" | ")} |`, ...table.rows.map(row => `| ${row.map(markdown).join(" | ")} |`), ""])
+  ].join("\n");
+}
+
+export function renderBenchmarkGoalText(goal) {
+  if (!goal) return "";
+  return ["Stricter every-pair goal assessment", ...summary(goal),
+    ...tables(goal).flatMap(table => ["", table.title, table.headers.join(" | "), ...table.rows.map(row => row.join(" | "))]), ""]
+    .join("\n");
+}
+
+export function renderBenchmarkGoalHtml(goal) {
+  if (!goal) return "";
+  return `<section id="every-pair-goal"><h2>Stricter every-pair goal assessment</h2>${summary(goal).map(line => `<p>${html(line)}</p>`).join("")}`
+    + tables(goal).map(table => `<h3>${html(table.title)}</h3><div class="table-wrap"><table><thead><tr>${table.headers.map(value => `<th>${html(value)}</th>`).join("")}</tr></thead><tbody>`
+      + table.rows.map(row => `<tr>${row.map(value => `<td>${html(value)}</td>`).join("")}</tr>`).join("") + "</tbody></table></div>").join("")
+    + "</section>";
+}

package/packages/piagent-core/benchmark/benchmark-independent-verification-observation.js

@@ -0,0 +1,64 @@
+const hash = value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+const count = value => Number.isSafeInteger(value) && value >= 0;
+const exact = (value, fields) => value && typeof value === "object" && !Array.isArray(value)
+  && Object.keys(value).length === fields.length && fields.every(field => Object.hasOwn(value, field));
+
+export function benchmarkVerificationFailure(value) {
+  if (!value || value.status === "not-configured") return null;
+  if (["partial", "unavailable"].includes(value.status)) return "independent-verification-coverage-incomplete";
+  return value.requests.some(request => request.runs.some(run => run.criteria.some(criterion => criterion.verdict !== "pass")))
+    ? "independent-verification-did-not-pass" : null;
+}
+
+/** Serialized observations are measurement data, never completion/repair capabilities. */
+export function validBenchmarkVerificationObservation(value) {
+  if (!exact(value, ["schemaVersion", "planDigest", "status", "attempts", "workersObserved", "requests"])
+    || value.schemaVersion !== 1 || !hash(value.planDigest) || !Array.isArray(value.requests) || value.requests.length > 32) return false;
+  if (value.status === "unavailable") return value.attempts === null && value.workersObserved === null && value.requests.length === 0;
+  if (!count(value.attempts) || !count(value.workersObserved)) return false;
+  if (value.status === "not-configured") return value.attempts === 0 && value.workersObserved === 0 && value.requests.length === 0;
+  if (!["observed", "partial"].includes(value.status) || value.requests.length < 1) return false;
+  let attempts = 0, workers = 0, complete = true;
+  const requests = new Set(), tasks = new Set();
+  for (const request of value.requests) {
+    if (!exact(request, ["operatorRequestDigest", "runs"]) || !/^operator-request-v1:[a-f0-9]{64}$/.test(request.operatorRequestDigest)
+      || requests.has(request.operatorRequestDigest) || !Array.isArray(request.runs) || request.runs.length > 1000) return false;
+    requests.add(request.operatorRequestDigest);
+    if (request.runs.length === 0) complete = false;
+    for (const run of request.runs) {
+      if (!exact(run, ["taskRunId", "criteria"]) || typeof run.taskRunId !== "string" || !run.taskRunId || run.taskRunId.length > 200
+        || tasks.has(run.taskRunId) || !Array.isArray(run.criteria) || run.criteria.length < 1 || run.criteria.length > 12) return false;
+      tasks.add(run.taskRunId);
+      const criteria = new Set();
+      for (const criterion of run.criteria) {
+        if (!exact(criterion, ["criterionId", "criterionHash", "attempts", "phase", "verdict", "workerObserved"])
+          || !/^[a-z0-9][a-z0-9:._-]{0,79}$/.test(criterion.criterionId) || criteria.has(criterion.criterionId)
+          || !hash(criterion.criterionHash) || !count(criterion.attempts) || criterion.attempts > 8
+          || typeof criterion.workerObserved !== "boolean"
+          || !["not-observed", "binding-mismatch", "invalid-evidence", "reserved", "interrupted", "settled"].includes(criterion.phase)) return false;
+        criteria.add(criterion.criterionId);
+        if (criterion.phase === "settled") {
+          if (criterion.attempts < 1 || !["pass", "fail", "unknown", "error"].includes(criterion.verdict)) return false;
+        } else if (criterion.verdict !== null || criterion.workerObserved || (criterion.phase === "not-observed" ? criterion.attempts !== 0 : criterion.attempts < 1)) return false;
+        attempts += criterion.attempts; workers += Number(criterion.workerObserved);
+        if (!criterion.workerObserved) complete = false;
+      }
+    }
+  }
+  return value.attempts === attempts && value.workersObserved === workers && (value.status === "observed") === complete;
+}
+
+export function benchmarkVerificationRecordMatches(record, identity) {
+  if (!identity || record.surface !== "piagent") return !Object.hasOwn(record, "independentVerification");
+  const observed = record.independentVerification;
+  if (!validBenchmarkVerificationObservation(observed) || observed.planDigest !== identity.contentDigest) return false;
+  if (benchmarkVerificationFailure(observed) && record.resolved !== false) return false;
+  const scenario = identity.scenarios.find(entry => entry.scenarioId === record.scenarioId);
+  if (!scenario) return observed.status === "not-configured";
+  if (observed.status === "unavailable") return true;
+  return observed.status !== "not-configured" && observed.requests.length === scenario.requests.length
+    && observed.requests.every((request, index) => request.operatorRequestDigest === scenario.requests[index].operatorRequestDigest
+      && request.runs.every(run => run.criteria.length === scenario.requests[index].criteria.length
+        && run.criteria.every((criterion, n) => criterion.criterionId === scenario.requests[index].criteria[n].criterionId
+          && criterion.criterionHash === scenario.requests[index].criteria[n].criterionHash)));
+}

package/packages/piagent-core/benchmark/benchmark-matrix-summary.js

@@ -0,0 +1,141 @@
+import { hierarchicalMatrixRatioSample } from "./benchmark-comparison.js";
+import {
+  geometricMean,
+  geometricMeanConfidence95,
+  geometricMeanConfidence95Raw
+} from "./benchmark-statistics.js";
+
+export function hierarchicalMatrixRatioSummary({
+  suite,
+  completeScenarioFreshRatios,
+  scenarioFreshRatios,
+  allSuccessfulPairsFreshRatio,
+  completeScenarioDurationRatios,
+  scenarioDurationRatios,
+  durationRatios
+}) {
+  const freshSample = hierarchicalMatrixRatioSample(suite, completeScenarioFreshRatios);
+  const confidenceFreshRatios = freshSample.matrix
+    ? freshSample.samples
+    : suite.schemaVersion === 2 ? completeScenarioFreshRatios : scenarioFreshRatios;
+  const freshRatio = freshSample.matrix || suite.schemaVersion === 2
+    ? geometricMean(confidenceFreshRatios.map((item) => item.ratio))
+    : allSuccessfulPairsFreshRatio;
+  const durationSample = hierarchicalMatrixRatioSample(suite, completeScenarioDurationRatios);
+  const confidenceDurationRatios = durationSample.matrix
+    ? durationSample.samples
+    : suite.schemaVersion === 2 ? completeScenarioDurationRatios : scenarioDurationRatios;
+  const durationRatio = durationSample.matrix
+    ? geometricMean(confidenceDurationRatios.map((item) => item.ratio))
+    : geometricMean(durationRatios);
+  return {
+    hierarchicalFreshRatioSample: freshSample,
+    confidenceScenarioRatios: confidenceFreshRatios,
+    freshRatio,
+    freshRatioConfidence95: geometricMeanConfidence95(confidenceFreshRatios.map((item) => item.ratio)),
+    freshRatioConfidence95Raw: geometricMeanConfidence95Raw(confidenceFreshRatios.map((item) => item.ratio)),
+    hierarchicalDurationRatioSample: durationSample,
+    confidenceScenarioDurationRatios: confidenceDurationRatios,
+    durationRatio,
+    durationRatioConfidence95: geometricMeanConfidence95(confidenceDurationRatios.map((item) => item.ratio)),
+    durationRatioConfidence95Raw: geometricMeanConfidence95Raw(confidenceDurationRatios.map((item) => item.ratio))
+  };
+}
+
+export function primaryEfficiencyMatrixSummary(matrixHierarchy, scenarioRatios) {
+  const scenarioIds = matrixHierarchy
+    ? [...new Set(scenarioRatios.flatMap((item) => item.scenarioIds ?? []))]
+    : scenarioRatios.map((item) => item.scenarioId);
+  const familyIds = matrixHierarchy
+    ? scenarioRatios.flatMap((item) => typeof item.familyId === "string" ? [item.familyId] : [])
+    : [];
+  return {
+    scenarioIds,
+    familyIds,
+    completeScenarios: scenarioIds.length,
+    completeSamples: matrixHierarchy ? familyIds.length : scenarioIds.length
+  };
+}
+
+export function hierarchicalMatrixReportSummary({
+  suite,
+  matrixHierarchy,
+  hierarchicalFreshRatioSample,
+  confidenceScenarioRatios,
+  hierarchicalDurationRatioSample,
+  confidenceScenarioDurationRatios,
+  primaryEfficiencyScenarioRatios,
+  primaryEfficiencyScenarioIds,
+  primaryEfficiencyFamilyIds,
+  primaryEfficiencyCompleteScenarios,
+  primaryUsesFixedWorkload,
+  familyClusteredFailureAware,
+  familyClusteredFixedWorkload
+}) {
+  return {
+    suiteFields: matrixHierarchy ? { matrixContract: suite.matrixContract } : {},
+    estimatorFields: {
+      usageEstimator: matrixHierarchy
+        ? "paired-geometric-mean-ratio-clustered-by-repeat-variant-task-family"
+        : "paired-geometric-mean-ratio",
+      durationEstimator: matrixHierarchy
+        ? "paired-geometric-mean-ratio-clustered-by-repeat-variant-task-family"
+        : "paired-geometric-mean-ratio-clustered-by-scenario-family"
+    },
+    fixedWorkloadEstimatorVersion: matrixHierarchy ? 2 : 1,
+    failureAwareCoverageFields: matrixHierarchy ? {
+      expectedTaskFamilies: familyClusteredFailureAware.expectedTaskFamilies,
+      usableTaskFamilies: familyClusteredFailureAware.usableTaskFamilies,
+      expectedVariants: familyClusteredFailureAware.expectedVariants,
+      usableVariants: familyClusteredFailureAware.usableVariants,
+      familyIds: familyClusteredFailureAware.familyIds
+    } : {},
+    fixedWorkloadCoverageFields: matrixHierarchy ? {
+      expectedTaskFamilies: familyClusteredFixedWorkload.expectedTaskFamilies,
+      usableTaskFamilies: familyClusteredFixedWorkload.usableTaskFamilies,
+      expectedVariants: familyClusteredFixedWorkload.expectedVariants,
+      usableVariants: familyClusteredFixedWorkload.usableVariants,
+      expectedAttemptsPerVariant: familyClusteredFixedWorkload.expectedAttemptsPerVariant,
+      familyIds: familyClusteredFixedWorkload.familyIds
+    } : {},
+    freshTokenRatioSample: matrixHierarchy ? {
+      sampleUnit: hierarchicalFreshRatioSample.sampleUnit,
+      sampleCount: confidenceScenarioRatios.length,
+      taskFamilyCount: confidenceScenarioRatios.length,
+      variantCount: hierarchicalFreshRatioSample.scenarioIds.length,
+      scenarioCount: hierarchicalFreshRatioSample.scenarioIds.length,
+      familyIds: hierarchicalFreshRatioSample.familyIds,
+      scenarioIds: hierarchicalFreshRatioSample.scenarioIds
+    } : {
+      sampleUnit: "scenario-family",
+      scenarioCount: confidenceScenarioRatios.length,
+      scenarioIds: confidenceScenarioRatios.map((item) => item.scenarioId)
+    },
+    primaryEfficiencySample: matrixHierarchy ? {
+      sampleUnit: suite.matrixContract.confidenceSampleUnit,
+      sampleCount: primaryEfficiencyScenarioRatios.length,
+      taskFamilyCount: primaryEfficiencyScenarioRatios.length,
+      variantCount: primaryEfficiencyCompleteScenarios,
+      scenarioCount: primaryEfficiencyCompleteScenarios,
+      familyIds: primaryEfficiencyFamilyIds,
+      scenarioIds: primaryEfficiencyScenarioIds,
+      outcomeConditioning: primaryUsesFixedWorkload ? "none" : "resolved-outcome-conditioned"
+    } : {
+      sampleUnit: "scenario-family",
+      scenarioCount: primaryEfficiencyCompleteScenarios,
+      scenarioIds: primaryEfficiencyScenarioIds,
+      outcomeConditioning: primaryUsesFixedWorkload ? "none" : "resolved-outcome-conditioned"
+    },
+    durationRatioSampleFields: matrixHierarchy ? {
+      durationRatioSample: {
+        sampleUnit: hierarchicalDurationRatioSample.sampleUnit,
+        sampleCount: confidenceScenarioDurationRatios.length,
+        taskFamilyCount: confidenceScenarioDurationRatios.length,
+        variantCount: hierarchicalDurationRatioSample.scenarioIds.length,
+        scenarioCount: hierarchicalDurationRatioSample.scenarioIds.length,
+        familyIds: hierarchicalDurationRatioSample.familyIds,
+        scenarioIds: hierarchicalDurationRatioSample.scenarioIds
+      }
+    } : {}
+  };
+}

package/packages/piagent-core/benchmark/benchmark-matrix.js

@@ -2,6 +2,7 @@ export const BENCHMARK_SCOPE_BANDS = [
   { id: "core", suite: "core-v1", scenarios: 4, availability: "runnable", purpose: "Fast regression check for task quality, scope, safety, and exact usage." },
   { id: "deep-logic", suite: "deep-logic-v1", scenarios: 7, availability: "runnable", claimTier: "capability", purpose: "Luna-medium-locked specialist regression across seven generated families, including exact temporal usage billing; full-suite only, not a generalization claim." },
   { id: "production", suite: "production-v1", scenarios: 18, availability: "runnable", claimTier: "public-regression", purpose: "Broader public regression matrix across archetypes, lifecycle states, and verifier profiles; not a generalization or production-stability claim." },
+  { id: "production-user-journey", suite: "production-v3", scenarios: 27, availability: "runnable", claimTier: "public-regression", purpose: "Seven-axis outcomes across WebUI-like multi-turn production journeys and a stock Codex CLI baseline; 108 sessions and task-family-clustered token confidence." },
   { id: "capability", suite: "capability-v1", scenarios: 6, availability: "runnable", claimTier: "capability", purpose: "Unsaturated multi-file and multi-component tasks for hill-climbing; not a release gate." },
   { id: "e2-framework", suite: "e2-framework-v1", scenarios: 4, availability: "runnable", claimTier: "capability", purpose: "Taxonomy-bound real Hono, SQLite, and workspace tasks with reference, mutation, alternative-valid, scope, and grader-sensitivity calibration; not a release gate." },
   { id: "long-horizon", suite: undefined, scenarios: 1, availability: "runnable-provider-free", evidenceLane: "evals/long-horizon-v1/lane.json", purpose: "At-least-30-minute provider-free lifecycle lane with hard process death, resume, compaction, handoff, bounded continuation, stable verification, and state-growth telemetry; not model-performance evidence." },

package/packages/piagent-core/benchmark/benchmark-normalized-cost.js

@@ -15,6 +15,7 @@ const CACHE_WRITE_FIELDS = new Set(["basis", "multiplier"]);
 const LONG_CONTEXT_FIELDS = new Set(["thresholdInputTokens", "condition", "inputMultiplier", "outputMultiplier"]);
 const SOURCE_FIELDS = new Set(["url", "retrievedAt"]);
 const USAGE_BUCKETS = Object.freeze(["input", "output", "cacheRead", "cacheWrite"]);
+const REQUEST_USAGE_BUCKETS = Object.freeze(["input", "output", "cacheRead", "cacheWrite", "reasoning", "total", "promptTokens"]);
 const BAND_DIMENSIONS = Object.freeze({
   categories: "category",
   profiles: "profile",
@@ -109,6 +110,44 @@ function exactUsageIssue(usage, snapshot) {
   return null;
 }
 
+function exactPricingBuckets(usage) {
+  const buckets = usage?.pricingBuckets;
+  if (buckets?.completeness !== "exact") return { status: "unavailable" };
+  if (!plainObject(buckets) || buckets.schemaVersion !== 1 || buckets.source !== "provider-request-usage"
+    || !Array.isArray(buckets.requests) || buckets.requests.length === 0) {
+    return { status: "invalid", issue: "pricing-buckets-invalid" };
+  }
+  const totals = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, reasoning: 0, total: 0 };
+  for (const request of buckets.requests) {
+    if (!plainObject(request) || REQUEST_USAGE_BUCKETS.some((field) => !Number.isSafeInteger(request[field]) || request[field] < 0)
+      || request.reasoning > request.output
+      || request.total !== request.input + request.cacheRead + request.cacheWrite + request.output
+      || request.promptTokens !== request.input + request.cacheRead + request.cacheWrite) {
+      return { status: "invalid", issue: "pricing-buckets-invalid" };
+    }
+    for (const field of Object.keys(totals)) totals[field] += request[field];
+  }
+  if (USAGE_BUCKETS.some((field) => totals[field] !== usage[field]) || totals.total !== usage.total
+    || Number.isSafeInteger(usage.reasoning) && totals.reasoning !== usage.reasoning) {
+    return { status: "invalid", issue: "pricing-buckets-aggregate-mismatch" };
+  }
+  return { status: "exact", requests: buckets.requests };
+}
+
+function requestPrice(request, snapshot) {
+  const longContext = request.promptTokens > snapshot.longContext.thresholdInputTokens;
+  const inputMultiplier = longContext ? snapshot.longContext.inputMultiplier : 1;
+  const outputMultiplier = longContext ? snapshot.longContext.outputMultiplier : 1;
+  const cacheWriteRate = snapshot.rates.freshInput * snapshot.cacheWrite.multiplier;
+  const amount = (
+    (request.input * snapshot.rates.freshInput * inputMultiplier)
+    + (request.cacheRead * snapshot.rates.cachedInput * inputMultiplier)
+    + (request.cacheWrite * cacheWriteRate * inputMultiplier)
+    + (request.output * snapshot.rates.output * outputMultiplier)
+  ) / snapshot.unitTokens;
+  return { amount, longContext };
+}
+
 /**
  * Convert exact token buckets to an API-equivalent amount without treating it as
  * OAuth/provider-billed cost. Aggregate prompt usage at or below the threshold
@@ -123,6 +162,29 @@ export function normalizeBenchmarkUsageCost(usage, snapshot) {
   const usageIssue = exactUsageIssue(usage, snapshot);
   if (usageIssue) return { status: "unavailable", reason: usageIssue, errors: [] };
   const aggregatePromptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
+  const pricingBuckets = exactPricingBuckets(usage);
+  if (pricingBuckets.status === "invalid") {
+    return { status: "unavailable", reason: pricingBuckets.issue, errors: [] };
+  }
+  if (pricingBuckets.status === "exact") {
+    const requestPrices = pricingBuckets.requests.map((request) => requestPrice(request, snapshot));
+    const amount = requestPrices.reduce((sum, item) => sum + item.amount, 0);
+    return {
+      status: "measured",
+      source: "versioned-api-equivalent-text-token-pricing",
+      pricingSnapshotId: snapshot.id,
+      pricingApplicability: "per-request-exact",
+      currency: snapshot.currency,
+      amount,
+      amountUsd: rounded(amount, 9),
+      aggregatePromptTokens,
+      thresholdInputTokens: snapshot.longContext.thresholdInputTokens,
+      requests: pricingBuckets.requests.length,
+      standardContextRequests: requestPrices.filter((item) => !item.longContext).length,
+      longContextRequests: requestPrices.filter((item) => item.longContext).length,
+      buckets: Object.fromEntries(USAGE_BUCKETS.map((field) => [field, usage[field]]))
+    };
+  }
   if (aggregatePromptTokens > snapshot.longContext.thresholdInputTokens) {
     return {
       status: "unavailable",

package/packages/piagent-core/benchmark/benchmark-pi-home.js

@@ -13,6 +13,7 @@ export const BENCHMARK_PI_HOME_EPHEMERAL_ENTRIES = Object.freeze({
   "models-store.json": "regular",
   "models-store.json.lock": "directory",
   "models.json.lock": "directory",
+  "piagent-gateway": "directory",
   "sessions": "directory",
   "settings.json.lock": "directory",
   "trust.json.lock": "directory"

package/packages/piagent-core/benchmark/benchmark-preflight.js

@@ -1,7 +1,10 @@
 import {
+  codexRuntimeCredentialPolicy,
   codexProcessEnvironment,
   controlledCodexFeatures
 } from "./benchmark-runtime.js";
+import { codexModelName } from "./benchmark-codex.js";
+import { verifyBenchmarkCommandIdentity } from "./benchmark-runtime-identity.js";
 
 function fail(message) {
   const error = new Error(message);
@@ -17,12 +20,22 @@ async function checkedVersion(runCommand, packageRoot, command, args, label, env
   return result.stdout.trim();
 }
 
-export async function benchmarkPreflight({ runCommand, packageRoot, piCommand, piEnvironment, codexCommand, gitCommand, surfaces, codexMode, codexRuntime }) {
+export async function benchmarkPreflight({ runCommand, packageRoot, piCommand, piEnvironment, codexCommand,
+  codexCommandIdentity, gitCommand, surfaces, codexBaseline = "stock", codexMode, codexRuntime, model, serviceTier }) {
+  if (surfaces.includes("codex-cli") && codexBaseline === "stock") {
+    if (codexCommandIdentity?.installationClosure?.kind !== "stock-codex-installation-v1") {
+      fail("Stock Codex installation closure is missing or unsupported");
+    }
+    try { verifyBenchmarkCommandIdentity(codexCommandIdentity, "stock Codex", { fullPackageClosure: true }); }
+    catch (error) { fail(`Stock Codex installation closure failed preflight: ${error.message}`); }
+  }
   const gitVersion = await checkedVersion(runCommand, packageRoot, gitCommand, ["--version"], "git");
   const piVersion = await checkedVersion(runCommand, packageRoot, piCommand, ["--version"], "pi", piEnvironment);
   let codexVersion;
   let codexAuth;
   let codexDisabledFeatures = [];
+  let codexFastModeFeature = null;
+  let codexCapability = null;
   if (surfaces.includes("codex-cli")) {
     const codexEnv = codexProcessEnvironment(codexRuntime);
     codexVersion = await checkedVersion(runCommand, packageRoot, codexCommand, ["--version"], "codex", codexEnv);
@@ -30,16 +43,45 @@ export async function benchmarkPreflight({ runCommand, packageRoot, piCommand, p
     try { result = await runCommand(codexCommand, ["login", "status"], { cwd: packageRoot, timeoutMs: 15_000, env: codexEnv }); }
     catch (error) { fail(`Codex authentication preflight failed: ${error.message}`); }
     if (result.code === 0) codexAuth = "login-status";
-    else if (process.env.OPENAI_API_KEY || process.env.CODEX_ACCESS_TOKEN) codexAuth = "environment-credential";
+    else if (codexEnv.OPENAI_API_KEY || codexEnv.CODEX_ACCESS_TOKEN) codexAuth = "environment-credential";
     else fail("Codex CLI is not authenticated; run codex login before this benchmark");
     if (codexMode === "controlled") {
       const features = await runCommand(codexCommand, ["features", "list"], { cwd: packageRoot, timeoutMs: 15_000, env: codexEnv });
       if (features.code !== 0) fail("Codex CLI cannot list features required by controlled benchmark mode; update Codex CLI or use --codex-mode native");
       const available = new Set(features.stdout.split("\n").map((line) => line.trim().split(/\s+/)[0]).filter(Boolean));
       codexDisabledFeatures = controlledCodexFeatures.filter((feature) => available.has(feature));
+      codexFastModeFeature = available.has("fast_mode");
+      if (serviceTier === "fast" && !codexFastModeFeature) {
+        fail("Codex CLI does not expose the fast_mode feature required by this benchmark; update Codex CLI");
+      }
+      if (codexBaseline === "stock" && !available.has("code_mode_host")) {
+        fail("Stock Codex CLI does not expose its required code_mode_host capability");
+      }
+    }
+    const execHelp = await checkedVersion(runCommand, packageRoot, codexCommand, ["exec", "--help"], "codex exec capability", codexEnv);
+    codexCapability = {
+      schemaVersion: 1,
+      providerFree: true,
+      model: codexModelName(model),
+      jsonl: /--json\b/.test(execHelp),
+      workspaceWrite: /workspace-write/.test(execHelp) && /--sandbox\b/.test(execHelp),
+      ignoreUserConfig: /--ignore-user-config\b/.test(execHelp),
+      ignoreRules: /--ignore-rules\b/.test(execHelp),
+      codeModeHost: codexBaseline === "stock"
+        ? codexCommandIdentity.installationClosure.files.some(file => file.role === "code-mode-host")
+        : null
+    };
+    if (!codexCapability.jsonl || !codexCapability.workspaceWrite
+      || !codexCapability.ignoreUserConfig || !codexCapability.ignoreRules || codexCapability.codeModeHost === false) {
+      fail("Codex CLI lacks a provider-free capability required by the benchmark execution contract");
     }
   }
-  return { gitVersion, piVersion, codexVersion, codexAuth, codexDisabledFeatures };
+  return { gitVersion, piVersion, codexVersion, codexAuth,
+    codexBaseline: surfaces.includes("codex-cli") ? codexBaseline : null,
+    codexInstallationDigest: codexCommandIdentity?.installationClosure?.contentDigest ?? null,
+    codexCapability,
+    codexCredentialPolicy: codexRuntimeCredentialPolicy(codexRuntime),
+    codexDisabledFeatures, codexFastModeFeature };
 }
 
 function publicCommandIdentity(value) {
@@ -52,14 +94,21 @@ function publicCommandIdentity(value) {
       name: value.packageClosure.name,
       version: value.packageClosure.version,
       contentDigest: value.packageClosure.tree?.contentDigest ?? null
-    } : null
+    } : null,
+    ...(value.installationClosure ? { installation: {
+      kind: value.installationClosure.kind,
+      fileCount: value.installationClosure.files.length,
+      contentDigest: value.installationClosure.contentDigest
+    } } : {})
   };
 }
 
 export function benchmarkPreflightReceipt({
   packageVersion, source, candidateProvenance, suite, suiteDigest,
-  runtimeDependencies, runtimeCommands, environmentPolicy, configurationDigest,
-  rootSeedDigest, options, runtime, hostReadinessPolicyDigest = null, hostReadiness = null
+  runtimeDependencies, webUiAssets, runtimeCommands, environmentPolicy, configurationDigest,
+  providerFreeConfigurationDigest = null, rootSeedDigest, options, runtime,
+  hostReadinessPolicyDigest = null, hostReadiness = null,
+  providerFreeEvidence = null, independentVerification, registeredRuntimeVerifiers = null
 }) {
   return {
     schemaVersion: 1,
@@ -72,29 +121,43 @@ export function benchmarkPreflightReceipt({
     suite: { id: suite.id, contentDigest: suiteDigest, scenarioCount: suite.scenarios.length },
     configuration: {
       contentDigest: configurationDigest,
+      ...(independentVerification ? { independentVerification } : {}),
+      ...(providerFreeConfigurationDigest ? { providerFreeContentDigest: providerFreeConfigurationDigest } : {}),
       ...(hostReadinessPolicyDigest ? { hostReadinessPolicyDigest } : {}),
       runtimeDependencyDigest: runtimeDependencies?.digest ?? null,
+      ...(webUiAssets ? { webUiAssetDigest: webUiAssets.digest } : {}),
       environmentPolicyDigest: environmentPolicy.digest,
       rootSeedDigest,
       surfaces: options.surfaces,
       model: options.model ?? null,
       thinking: options.thinking ?? null,
+      serviceTier: options.serviceTier ?? null,
       codexMode: options.codexMode,
+      codexBaseline: options.codexBaseline ?? null,
       piagentTreatment: options.piagentTreatment,
       repeats: options.repeats,
       timeoutSeconds: options.timeoutSeconds,
       infrastructureRetries: options.infrastructureRetries,
       retryDelaySeconds: options.retryDelaySeconds,
       maxSessions: options.maxSessions ?? null,
-      stopAfterFailedPair: options.stopAfterFailedPair
+      stopAfterFailedPair: options.stopAfterFailedPair,
+      ...(options.campaignStopPolicy ? { campaignStopPolicy: options.campaignStopPolicy } : {}),
+      ...(options.measurementOnly === true ? { measurementOnly: true } : {})
     },
     ...(hostReadiness ? { hostReadiness } : {}),
+    ...(providerFreeEvidence ? { providerFreeEvidence } : {}),
+    ...(registeredRuntimeVerifiers ? { registeredRuntimeVerifiers } : {}),
     runtime: {
       gitVersion: runtime.gitVersion,
       piVersion: runtime.piVersion,
       codexVersion: runtime.codexVersion ?? null,
       codexAuth: runtime.codexAuth ?? null,
+      codexBaseline: runtime.codexBaseline ?? null,
+      codexInstallationDigest: runtime.codexInstallationDigest ?? null,
+      codexCapability: runtime.codexCapability ?? null,
+      codexCredentialPolicy: runtime.codexCredentialPolicy ?? null,
       codexDisabledFeatures: runtime.codexDisabledFeatures,
+      codexFastModeFeature: runtime.codexFastModeFeature ?? null,
       commands: Object.fromEntries(Object.entries(runtimeCommands).map(([name, value]) => [name, publicCommandIdentity(value)]))
     },
     usageContract: {

package/packages/piagent-core/benchmark/benchmark-process.js

@@ -23,7 +23,7 @@ function groupAlive(child) {
   }
 }
 
-export function createBenchmarkProcessController(interrupted) {
+export function createBenchmarkProcessController(interrupted, ownership = {}) {
   const active = new Set();
   const killTimers = new Map();
   const scheduleKill = (child, delay) => {
@@ -58,10 +58,21 @@ export function createBenchmarkProcessController(interrupted) {
       if (killTimer) clearTimeout(killTimer);
       killTimers.delete(child);
     };
-    if (!options.inherit && options.input !== undefined) {
-      child.stdin.on("error", () => { /* Child exit is reported by the process result. */ });
-      child.stdin.end(String(options.input));
-    }
+    let processError, registered = false, registration = Promise.resolve();
+    child.once("spawn", () => {
+      registration = Promise.resolve().then(async () => {
+        await ownership.started?.(child.pid);
+        registered = Boolean(ownership.started);
+        if (!options.inherit && options.input !== undefined && child.exitCode === null && child.signalCode === null) {
+          child.stdin.on("error", () => { /* Child exit is reported by the process result. */ });
+          child.stdin.end(String(options.input));
+        }
+      }).catch(error => {
+        processError = error;
+        terminateGroup(child, "SIGTERM");
+        scheduleKill(child, 500);
+      });
+    });
     let stdout = "";
     let stderr = "";
     const stdoutDigest = crypto.createHash("sha256");
@@ -99,11 +110,10 @@ export function createBenchmarkProcessController(interrupted) {
     let settled = false;
     child.once("error", (error) => {
       if (settled) return;
-      settled = true;
+      processError = error;
       if (timer) clearTimeout(timer);
-      cleanup();
       terminateGroup(child, "SIGTERM");
-      reject(error);
+      if (child.pid) scheduleKill(child, 500);
     });
     child.once("exit", (code, signal) => {
       if (timer) clearTimeout(timer);
@@ -113,10 +123,14 @@ export function createBenchmarkProcessController(interrupted) {
     child.once("close", (code, signal) => {
       if (settled) return;
       if (timer) clearTimeout(timer);
-      const finish = () => {
+      const finish = async () => {
         if (settled) return;
         settled = true;
+        await registration;
+        try { if (registered) await ownership.closed?.(child.pid); }
+        catch (error) { processError ??= error; }
         cleanup();
+        if (processError) { reject(processError); return; }
         resolve({ code: code ?? 1, signal, timedOut, stdout, stdoutHash: options.inherit ? undefined : stdoutDigest.digest("hex"), stderr, forbiddenHits: [...forbiddenHits], requiredHits: [...requiredHits], durationSeconds: (performance.now() - started) / 1000 });
       };
       if (!groupAlive(child)) {