@piagent/platform 1.6.1 → 1.7.1

package/packages/piagent-core/benchmark/benchmark-suite-identity.js

@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 
@@ -6,10 +7,272 @@ export const BUILT_IN_BENCHMARK_SUITE_IDS = Object.freeze([
   "capability-v1",
   "e2-framework-v1",
   "deep-logic-v1",
-  "production-v1"
+  "production-v1",
+  "production-v2",
+  "production-v3"
 ]);
 
 const reservedSuiteIds = new Set(BUILT_IN_BENCHMARK_SUITE_IDS);
+const HASH = /^[a-f0-9]{64}$/;
+const ID = /^[A-Za-z0-9][A-Za-z0-9:._~-]{0,159}$/;
+const registrationPayloadFields = Object.freeze([
+  "suiteId", "baseSuiteId", "publicContractRevision", "admissionProtocol", "treatment",
+  "baseSuiteDigest", "caseListDigest", "unchangedSeedDigest", "evaluatorDigest",
+  "publicAssetsDigest", "publicContractDigest", "catalogDigest", "producerManifestDigest",
+  "nodeProfileDigest", "sharedEnvironmentDigest", "commonObserverDigest", "restrictedBrokerDigest",
+  "protocolDigest", "evaluatorCompatibilityDigest", "baselineCommit", "scenarioIds",
+  "promptRolesChanged", "matrix", "resources", "claims"
+]);
+const registrationDigestFields = Object.freeze(registrationPayloadFields.filter(name => name.endsWith("Digest")));
+export const REGISTERED_BENCHMARK_PROMPT_ROLES = Object.freeze([
+  "revoked-session-cache", "backend-frontend-contract-sync", "workspace-order", "expiry-boundary",
+  "idempotent-replay-conflict", "reconnect-chat-event-order", "quoted-csv", "invoice-rounding",
+  "pagination-boundary", "schema-migration", "incident-diagnosis", "protected-env-refusal",
+  "repository-prompt-injection", "destructive-history-refusal"
+]);
+export const REGISTERED_BENCHMARK_VERIFIER_IDS = Object.freeze([
+  "bash-runtime", "broker-source-closure", "candidate-source", "common-runtime-closure",
+  "controlled-codex-feature-policy", "controlled-comparison-runtime", "custody-runner-wrapper",
+  "docker-runtime", "git-runtime", "node-runtime", "npm-runtime", "pi-runtime", "pi-sdk-tree", "scoped-route-proof",
+  "tool-definitions", "webui-asset-tree", "worker-image"
+]);
+const registeredStaticAssets = Object.freeze([
+  "catalog.json", "measurement/node-workload-api-v1.json", "measurement/shared-tool-policy-v1.json",
+  "measurement/assurance-rubrics.json", "measurement/calibration/incident-public-v1.json",
+  "measurement/calibration/config-document-public-v1.json", "measurement/calibration/refusal-public-v1.json"
+]);
+
+function canonicalJson(value, seen = new Set()) {
+  if (value === null || typeof value === "string" || typeof value === "boolean") return JSON.stringify(value);
+  if (typeof value === "number") {
+    if (!Number.isFinite(value)) throw new TypeError("registered measurement canonical JSON rejects non-finite numbers");
+    return JSON.stringify(value);
+  }
+  if (!value || typeof value !== "object") throw new TypeError("registered measurement canonical JSON rejects non-JSON values");
+  if (seen.has(value)) throw new TypeError("registered measurement canonical JSON rejects cycles");
+  seen.add(value);
+  const rendered = Array.isArray(value)
+    ? `[${value.map(item => canonicalJson(item, seen)).join(",")}]`
+    : `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key], seen)}`).join(",")}}`;
+  seen.delete(value);
+  return rendered;
+}
+
+function exactKeys(value, names) {
+  return Boolean(value && typeof value === "object" && !Array.isArray(value)
+    && Object.keys(value).length === names.length && names.every(name => Object.hasOwn(value, name)));
+}
+
+function sortedUniqueIds(values, count) {
+  return Array.isArray(values) && values.length === count && new Set(values).size === count
+    && values.every(value => typeof value === "string" && ID.test(value));
+}
+
+export function registeredBenchmarkMeasurementPayloadDigest(payload) {
+  return crypto.createHash("sha256").update(canonicalJson(payload)).digest("hex");
+}
+
+export function registeredBenchmarkPublicAssetPaths(scenarioIds) {
+  if (!sortedUniqueIds(scenarioIds, 27)) throw new Error("Registered measurement asset inventory requires 27 scenario ids");
+  return Object.freeze([
+    ...scenarioIds.map(id => `plans/${id}.json`),
+    ...REGISTERED_BENCHMARK_PROMPT_ROLES.map(id => `measurement/prompts/${id}.md`),
+    ...registeredStaticAssets
+  ].sort((left, right) => Buffer.compare(Buffer.from(left), Buffer.from(right))));
+}
+
+function stableAsset(root, relativePath) {
+  const target = path.resolve(root, relativePath), relative = path.relative(root, target);
+  if (relative !== relativePath.split("/").join(path.sep) || relative === ".."
+    || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+    throw new Error(`Registered measurement asset path escapes its root: ${relativePath}`);
+  }
+  const before = fs.lstatSync(target, { bigint: true });
+  if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1n || before.size > 2n * 1024n * 1024n) {
+    throw new Error(`Registered measurement asset must be one bounded regular file: ${relativePath}`);
+  }
+  const fd = fs.openSync(target, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));
+  try {
+    const descriptor = fs.fstatSync(fd, { bigint: true }), bytes = fs.readFileSync(fd), after = fs.fstatSync(fd,
+      { bigint: true }), current = fs.lstatSync(target, { bigint: true }),
+      fields = ["dev", "ino", "mode", "nlink", "size", "mtimeNs", "ctimeNs"];
+    if (bytes.length !== Number(before.size) || fields.some(name => before[name] !== descriptor[name]
+      || before[name] !== after[name] || before[name] !== current[name])) {
+      throw new Error(`Registered measurement asset changed while read: ${relativePath}`);
+    }
+    return bytes;
+  } finally { fs.closeSync(fd); }
+}
+
+function inventoryDigest(entries) {
+  return crypto.createHash("sha256").update(canonicalJson(entries)).digest("hex");
+}
+
+export function registeredBenchmarkPublicAssetInventory(root, scenarioIds) {
+  if (typeof root !== "string" || !path.isAbsolute(root) || fs.realpathSync(root) !== root
+    || !fs.lstatSync(root).isDirectory()) throw new Error("Registered measurement asset root must be canonical");
+  const expected = registeredBenchmarkPublicAssetPaths(scenarioIds), allowed = new Set([
+    ...expected, "measurement/registered-suite.json"
+  ]), observed = [], pending = [root];
+  while (pending.length > 0) {
+    const directory = pending.pop();
+    for (const name of fs.readdirSync(directory)) {
+      const target = path.join(directory, name), info = fs.lstatSync(target), relative =
+        path.relative(root, target).split(path.sep).join("/");
+      if (info.isSymbolicLink()) throw new Error(`Registered measurement assets reject symlink: ${relative}`);
+      if (info.isDirectory()) pending.push(target);
+      else if (info.isFile()) observed.push(relative);
+      else throw new Error(`Registered measurement assets reject unsupported node: ${relative}`);
+    }
+  }
+  observed.sort((left, right) => Buffer.compare(Buffer.from(left), Buffer.from(right)));
+  if (observed.length !== allowed.size || observed.some(value => !allowed.has(value))) {
+    throw new Error("Registered measurement asset root must contain exactly the approved inventory and manifest");
+  }
+  const entries = expected.map(relativePath => {
+    const bytes = stableAsset(root, relativePath);
+    return Object.freeze({ path: relativePath, bytes: bytes.length,
+      sha256: crypto.createHash("sha256").update(bytes).digest("hex") });
+  });
+  const semanticPaths = new Set(expected.filter(value => value !== "measurement/node-workload-api-v1.json"
+    && value !== "measurement/shared-tool-policy-v1.json"));
+  return deepFreeze({ version: "registered-public-asset-inventory-v1", entries,
+    publicAssetsDigest: inventoryDigest(entries),
+    publicContractDigest: inventoryDigest(entries.filter(value => semanticPaths.has(value.path))) });
+}
+
+export function assertRegisteredBenchmarkPublicAssets(registration, root, scenarioIds) {
+  const inventory = registeredBenchmarkPublicAssetInventory(root, scenarioIds);
+  if (registration?.payload?.publicAssetsDigest !== inventory.publicAssetsDigest
+    || registration?.payload?.publicContractDigest !== inventory.publicContractDigest) {
+    throw new Error("Registered measurement rejected: public asset inventory digest mismatch");
+  }
+  return inventory;
+}
+
+export function registeredBenchmarkMeasurementValidationErrors(envelope, { scenarioIds } = {}) {
+  const errors = [];
+  if (!exactKeys(envelope, ["schemaVersion", "payload", "approvalRecordDigest"])) {
+    return ["envelope must contain exactly schemaVersion, payload and approvalRecordDigest"];
+  }
+  if (envelope.schemaVersion !== 1) errors.push("schemaVersion must be 1");
+  if (!HASH.test(String(envelope.approvalRecordDigest ?? ""))) errors.push("approvalRecordDigest must be lowercase sha256");
+  const payload = envelope.payload;
+  if (!exactKeys(payload, registrationPayloadFields)) {
+    errors.push("payload fields must match the registered measurement contract exactly");
+    return errors;
+  }
+  const constants = {
+    suiteId: "production-v2-da2", baseSuiteId: "production-v2",
+    publicContractRevision: "public-contract-addendum-v2",
+    admissionProtocol: "configured-public-108-v1", treatment: "configured-independent-v2",
+    baselineCommit: "05903656958fc78638779bf0a9e9b403f18992a2"
+  };
+  for (const [name, expected] of Object.entries(constants)) if (payload[name] !== expected) {
+    errors.push(`${name} must equal ${expected}`);
+  }
+  for (const name of registrationDigestFields) if (!HASH.test(String(payload[name] ?? ""))) {
+    errors.push(`${name} must be lowercase sha256`);
+  }
+  if (!sortedUniqueIds(payload.scenarioIds, 27)) errors.push("scenarioIds must contain 27 unique registered ids");
+  if (Array.isArray(scenarioIds) && JSON.stringify(payload.scenarioIds) !== JSON.stringify(scenarioIds)) {
+    errors.push("scenarioIds must equal the frozen base suite order");
+  }
+  if (JSON.stringify(payload.promptRolesChanged) !== JSON.stringify(REGISTERED_BENCHMARK_PROMPT_ROLES)) {
+    errors.push("promptRolesChanged must equal the approved 14-role order");
+  }
+  if (!exactKeys(payload.matrix, ["scenarioCount", "repeats", "surfaces", "sessionCount"])
+    || payload.matrix.scenarioCount !== 27 || payload.matrix.repeats !== 2
+    || JSON.stringify(payload.matrix.surfaces) !== JSON.stringify(["piagent", "codex-cli"])
+    || payload.matrix.sessionCount !== 108) errors.push("matrix must equal the controlled complete108 matrix");
+  if (!exactKeys(payload.resources, ["model", "thinking", "requestedServiceTier", "concurrency",
+    "timeoutSeconds", "infrastructureRetries", "verifiers"])
+    || payload.resources.model !== "openai-codex/gpt-5.6-luna" || payload.resources.thinking !== "medium"
+    || payload.resources.requestedServiceTier !== "fast" || payload.resources.concurrency !== 1
+    || payload.resources.timeoutSeconds !== 900 || payload.resources.infrastructureRetries !== 0
+    || !Array.isArray(payload.resources.verifiers) || payload.resources.verifiers.length === 0
+    || payload.resources.verifiers.length > 32) errors.push("resources must equal the approved provider limits and explicit verifiers");
+  else {
+    const ids = payload.resources.verifiers.map(value => value?.id);
+    if (JSON.stringify(ids) !== JSON.stringify(REGISTERED_BENCHMARK_VERIFIER_IDS)
+      || payload.resources.verifiers.some(value => !exactKeys(value, ["id", "sha256"])
+        || typeof value.id !== "string" || !ID.test(value.id) || !HASH.test(String(value.sha256 ?? "")))) {
+      errors.push("verifiers must equal the exact registered runtime and custody identity set");
+    }
+  }
+  if (!exactKeys(payload.claims, ["efficiencyProtocol", "wireProtocol", "configuredTreatment",
+    "releaseDefaultsClaim"]) || payload.claims.efficiencyProtocol !== "net35-family-pooled-v1"
+    || payload.claims.wireProtocol !== "phase-valid-configuration-v1"
+    || payload.claims.configuredTreatment !== true || payload.claims.releaseDefaultsClaim !== false) {
+    errors.push("claims must equal the approved configured-treatment boundary");
+  }
+  return errors;
+}
+
+function stableJson(file, label) {
+  if (typeof file !== "string" || !path.isAbsolute(file) || path.normalize(file) !== file || file.includes("\0")) {
+    throw new Error(`${label} path must be canonical and absolute`);
+  }
+  const resolved = fs.realpathSync(file), before = fs.lstatSync(resolved, { bigint: true });
+  if (resolved !== file || !before.isFile() || before.isSymbolicLink() || before.nlink !== 1n
+    || before.size < 2n || before.size > 2n * 1024n * 1024n
+    || typeof process.getuid === "function" && before.uid !== BigInt(process.getuid())) {
+    throw new Error(`${label} must be one host-owned regular unlinked file`);
+  }
+  const fd = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));
+  try {
+    const descriptor = fs.fstatSync(fd, { bigint: true }), bytes = fs.readFileSync(fd),
+      after = fs.fstatSync(fd, { bigint: true }), current = fs.lstatSync(file, { bigint: true }),
+      fields = ["dev", "ino", "mode", "nlink", "size", "mtimeNs", "ctimeNs"];
+    if (bytes.length !== Number(before.size) || fields.some(name => before[name] !== descriptor[name]
+      || before[name] !== after[name] || before[name] !== current[name])) throw new Error(`${label} changed while read`);
+    let value;
+    try { value = JSON.parse(bytes.toString("utf8")); }
+    catch (error) { throw new Error(`${label} is not valid JSON: ${error.message}`); }
+    return { bytes, value };
+  } finally { fs.closeSync(fd); }
+}
+
+function deepFreeze(value) {
+  if (!value || typeof value !== "object" || Object.isFrozen(value)) return value;
+  for (const child of Object.values(value)) deepFreeze(child);
+  return Object.freeze(value);
+}
+
+export function loadRegisteredBenchmarkMeasurement(file, { approvalFile, approvalPublicKey, scenarioIds } = {}) {
+  const manifest = stableJson(file, "registered measurement manifest"), errors =
+    registeredBenchmarkMeasurementValidationErrors(manifest.value, { scenarioIds });
+  if (errors.length > 0) throw new Error(`Registered measurement rejected: ${errors.join("; ")}`);
+  const approval = stableJson(approvalFile, "registered measurement approval"), approvalDigest =
+    crypto.createHash("sha256").update(approval.bytes).digest("hex");
+  if (approvalDigest !== manifest.value.approvalRecordDigest) {
+    throw new Error("Registered measurement rejected: detached approval digest mismatch");
+  }
+  if (!exactKeys(approval.value, ["schemaVersion", "kind", "approvalId", "authorityKeyId",
+    "payloadDigest", "signature"]) || approval.value.schemaVersion !== 1
+    || approval.value.kind !== "benchmark-measurement-approval-v1" || !ID.test(String(approval.value.approvalId ?? ""))
+    || !HASH.test(String(approval.value.authorityKeyId ?? "")) || !HASH.test(String(approval.value.payloadDigest ?? ""))
+    || typeof approval.value.signature !== "string") {
+    throw new Error("Registered measurement rejected: detached approval record is invalid");
+  }
+  let key;
+  try { key = approvalPublicKey?.type === "public" ? approvalPublicKey : crypto.createPublicKey(approvalPublicKey); }
+  catch { throw new Error("Registered measurement rejected: trusted approval public key is invalid"); }
+  if (key?.asymmetricKeyType !== "ed25519") throw new Error("Registered measurement rejected: trusted approval key must be Ed25519");
+  const der = key.export({ type: "spki", format: "der" }), keyId = crypto.createHash("sha256").update(der).digest("hex"),
+    payloadDigest = registeredBenchmarkMeasurementPayloadDigest(manifest.value.payload);
+  let signature;
+  try { signature = Buffer.from(approval.value.signature, "base64"); }
+  catch { signature = Buffer.alloc(0); }
+  if (approval.value.authorityKeyId !== keyId || approval.value.payloadDigest !== payloadDigest
+    || signature.length !== 64 || !crypto.verify(null, Buffer.from(payloadDigest), key, signature)) {
+    throw new Error("Registered measurement rejected: detached approval signature or payload binding is invalid");
+  }
+  return deepFreeze({ envelope: manifest.value, payload: manifest.value.payload, identity: {
+    manifestSha256: crypto.createHash("sha256").update(manifest.bytes).digest("hex"), payloadSha256: payloadDigest,
+    approvalRecordSha256: approvalDigest, approvalId: approval.value.approvalId, authorityKeyId: keyId
+  }, files: { manifest: file, approval: approvalFile } });
+}
 
 export function isReservedBenchmarkSuiteId(value) {
   return reservedSuiteIds.has(value);

package/packages/piagent-core/benchmark/benchmark-suite-runtime.js

@@ -74,9 +74,13 @@ export function validateBenchmarkSuiteFiles(suite, suiteRoot) {
     const generator = scenario.variantGenerator
       ? resolveBenchmarkSuiteEntry(suiteRoot, scenario.variantGenerator, "variant generator")
       : null;
+    const journeyPrompts = (scenario.userJourney?.turns ?? []).map((turn) => (
+      resolveBenchmarkSuiteEntry(suiteRoot, turn.prompt, `journey prompt for ${scenario.id}/${turn.id}`)
+    ));
     assertBenchmarkModuleGraphBound(grader, suiteRoot, `${scenario.id} grader`);
     if (generator) assertBenchmarkModuleGraphBound(generator, suiteRoot, `${scenario.id} variant generator`);
     if (inside(fixture, grader) || inside(fixture, prompt)) fail(`Suite prompt and grader must stay outside the agent fixture: ${scenario.id}`);
+    if (journeyPrompts.some((journeyPrompt) => inside(fixture, journeyPrompt))) fail(`Suite journey prompts must stay outside the agent fixture: ${scenario.id}`);
     if (generator && inside(fixture, generator)) fail(`Suite variant generator must stay outside the agent fixture: ${scenario.id}`);
   }
 }

package/packages/piagent-core/benchmark/benchmark-suite.js

@@ -3,13 +3,21 @@ import { benchmarkPricingSnapshotValidationErrors } from "./benchmark-normalized
 
 const SUITE_FIELDS = new Set([
   "schemaVersion", "id", "title", "description", "profile", "defaultRepeats", "timeoutSeconds",
-  "assurance", "releaseGate", "executionContract", "pricingSnapshot", "scenarios"
+  "assurance", "releaseGate", "executionContract", "pricingSnapshot", "matrixContract", "scenarios"
 ]);
 const SCENARIO_FIELDS = new Set([
   "id", "title", "description", "kind", "fixture", "prompt", "grader", "allowedChanges",
   "setupFiles", "forbiddenOutputSubstrings", "requiredOutputSubstrings", "category", "difficulty",
-  "profile", "lifecycle", "variantGenerator"
+  "profile", "lifecycle", "variantGenerator", "familyId", "variantId", "variantRole", "userJourney"
 ]);
+const MATRIX_CONTRACT_FIELDS = new Set([
+  "schemaVersion", "familyCount", "variantsPerFamily", "variantRoles", "surfaces",
+  "repeatsPerVariant", "expectedPairs", "expectedSessions", "confidenceSampleUnit"
+]);
+const USER_JOURNEY_FIELDS = new Set(["turns", "expectedTerminalSettlement"]);
+const USER_JOURNEY_TURN_FIELDS = new Set(["id", "prompt", "workflow", "reconnectBefore", "receiptUncertain"]);
+const MATRIX_VARIANT_ROLES = new Set(["boundary", "interaction", "adversarial-recovery"]);
+const TERMINAL_SETTLEMENTS = new Set(["completed", "refused"]);
 const RELEASE_GATE_FIELDS = new Set([
   "minimumQualityScore", "minimumSafetyScore", "minimumReliabilityScore", "minimumWorkflowScore",
   "minimumCategoryScore", "minimumOutcomeScoreExclusive", "minimumPairedScenarios", "minimumRepeats",
@@ -20,9 +28,11 @@ const RELEASE_GATE_FIELDS = new Set([
   "maximumInfrastructureRetries", "primaryEfficiencyEstimand", "requireEfficiencyClaim", "requireFullSuiteForClaim",
   "requireStableProviderWireSurface", "requireNormalizedCostClaim", "requireHostReadinessForClaim",
   "requireCausalContextReceipt", "requireSubagentBudget", "maximumSubagentSessionsPerAttempt",
-  "maximumSubagentTrafficShare", "requireProviderFreeEvidence"
+  "maximumSubagentTrafficShare", "requireProviderFreeEvidence", "requireFastServiceTier",
+  "maximumAllAttemptPooledFreshTokenRatio", "requireCampaignAccounting", "efficiencyProtocol"
 ]);
-const EXECUTION_CONTRACT_FIELDS = new Set(["surfaces", "model", "thinking", "codexMode"]);
+const EXECUTION_CONTRACT_FIELDS = new Set(["surfaces", "model", "thinking", "codexMode", "codexBaseline", "serviceTier"]);
+const SERVICE_TIERS = new Set(["default", "fast"]);
 const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]);
 const SCENARIO_KINDS = new Set(["source-change", "read-only", "safety-refusal"]);
 const SCENARIO_DIFFICULTIES = new Set(["small", "medium", "large"]);
@@ -76,6 +86,27 @@ export function benchmarkSuiteValidationErrors(input) {
       for (const field of Object.keys(input.releaseGate)) {
         if (!RELEASE_GATE_FIELDS.has(field)) errors.push(`releaseGate has unsupported field ${field}`);
       }
+      // Opt-in measurement contract only. Approval belongs to the frozen run
+      // protocol; this marker neither changes legacy defaults nor grants claims.
+      if (input.releaseGate.efficiencyProtocol !== undefined) {
+        if (input.releaseGate.efficiencyProtocol !== "net35-family-pooled-v1") {
+          errors.push("releaseGate.efficiencyProtocol must be net35-family-pooled-v1");
+        }
+        if (input.schemaVersion !== 2) errors.push("releaseGate.efficiencyProtocol requires schemaVersion 2");
+        if (input.releaseGate.requireEfficiencyClaim !== true) {
+          errors.push("net35-family-pooled-v1 requires releaseGate.requireEfficiencyClaim true");
+        }
+        for (const field of ["maximumFreshTokenRatioUpper95", "maximumAllAttemptPooledFreshTokenRatio"]) {
+          if (input.releaseGate[field] !== 0.65) errors.push(`net35-family-pooled-v1 requires releaseGate.${field} exactly 0.65`);
+        }
+        if (input.releaseGate.primaryEfficiencyEstimand !== "fixed-workload-family-ratio") {
+          errors.push("net35-family-pooled-v1 requires releaseGate.primaryEfficiencyEstimand fixed-workload-family-ratio");
+        }
+        if (input.matrixContract?.familyCount !== 9 || input.matrixContract?.variantsPerFamily !== 3
+          || input.matrixContract?.repeatsPerVariant !== 2 || input.matrixContract?.confidenceSampleUnit !== "task-family") {
+          errors.push("net35-family-pooled-v1 requires the 9-family, 3-variant, 2-repeat task-family matrixContract");
+        }
+      }
       for (const field of [
         "minimumQualityScore", "minimumSafetyScore", "minimumReliabilityScore", "minimumWorkflowScore",
         "minimumCategoryScore"
@@ -109,6 +140,13 @@ export function benchmarkSuiteValidationErrors(input) {
       if (maximumRatio !== undefined && (!Number.isFinite(maximumRatio) || maximumRatio <= 0 || maximumRatio > 10)) {
         errors.push("releaseGate.maximumFreshTokenRatioUpper95 must be greater than 0 and at most 10");
       }
+      const maximumAllAttemptPooledFreshTokenRatio = input.releaseGate.maximumAllAttemptPooledFreshTokenRatio;
+      if (maximumAllAttemptPooledFreshTokenRatio !== undefined
+        && (!Number.isFinite(maximumAllAttemptPooledFreshTokenRatio)
+          || maximumAllAttemptPooledFreshTokenRatio <= 0
+          || maximumAllAttemptPooledFreshTokenRatio > 10)) {
+        errors.push("releaseGate.maximumAllAttemptPooledFreshTokenRatio must be greater than 0 and at most 10");
+      }
       const maximumBandRatio = input.releaseGate.maximumBandFreshTokenRatio;
       if (maximumBandRatio !== undefined && (!Number.isFinite(maximumBandRatio) || maximumBandRatio <= 0 || maximumBandRatio > 10)) {
         errors.push("releaseGate.maximumBandFreshTokenRatio must be greater than 0 and at most 10");
@@ -170,6 +208,12 @@ export function benchmarkSuiteValidationErrors(input) {
       if (input.releaseGate.requireProviderFreeEvidence !== undefined && typeof input.releaseGate.requireProviderFreeEvidence !== "boolean") {
         errors.push("releaseGate.requireProviderFreeEvidence must be a boolean");
       }
+      if (input.releaseGate.requireFastServiceTier !== undefined && typeof input.releaseGate.requireFastServiceTier !== "boolean") {
+        errors.push("releaseGate.requireFastServiceTier must be a boolean");
+      }
+      if (input.releaseGate.requireCampaignAccounting !== undefined && typeof input.releaseGate.requireCampaignAccounting !== "boolean") {
+        errors.push("releaseGate.requireCampaignAccounting must be a boolean");
+      }
       const maximumSubagentSessions = input.releaseGate.maximumSubagentSessionsPerAttempt;
       if (maximumSubagentSessions !== undefined
         && (!Number.isSafeInteger(maximumSubagentSessions) || maximumSubagentSessions < 0 || maximumSubagentSessions > 10)) {
@@ -211,6 +255,50 @@ export function benchmarkSuiteValidationErrors(input) {
       }
       if (!THINKING_LEVELS.has(contract.thinking)) errors.push("executionContract.thinking is invalid");
       if (contract.codexMode !== "controlled") errors.push("executionContract.codexMode must be controlled");
+      if (contract.codexBaseline !== undefined && !["stock", "controlled-custom"].includes(contract.codexBaseline)) {
+        errors.push("executionContract.codexBaseline must be stock or controlled-custom");
+      }
+      if (contract.serviceTier !== undefined && !SERVICE_TIERS.has(contract.serviceTier)) {
+        errors.push("executionContract.serviceTier must be default or fast");
+      }
+      if (input.releaseGate?.requireFastServiceTier === true && contract.serviceTier !== "fast") {
+        errors.push("executionContract.serviceTier must be fast when releaseGate.requireFastServiceTier is true");
+      }
+    }
+  }
+  if (input.matrixContract !== undefined) {
+    const matrix = input.matrixContract;
+    if (input.schemaVersion !== 2) errors.push("matrixContract requires suite schemaVersion 2");
+    if (!plainObject(matrix)) errors.push("matrixContract must be an object");
+    else {
+      for (const field of Object.keys(matrix)) {
+        if (!MATRIX_CONTRACT_FIELDS.has(field)) errors.push(`matrixContract has unsupported field ${field}`);
+      }
+      if (matrix.schemaVersion !== 1) errors.push("matrixContract.schemaVersion must be 1");
+      for (const [field, maximum] of [["familyCount", 50], ["variantsPerFamily", 10], ["repeatsPerVariant", 10], ["expectedPairs", 500], ["expectedSessions", 1_000]]) {
+        if (!Number.isSafeInteger(matrix[field]) || matrix[field] < 1 || matrix[field] > maximum) {
+          errors.push(`matrixContract.${field} must be between 1 and ${maximum}`);
+        }
+      }
+      if (!Array.isArray(matrix.variantRoles)
+        || matrix.variantRoles.length !== matrix.variantsPerFamily
+        || new Set(matrix.variantRoles).size !== matrix.variantRoles.length
+        || matrix.variantRoles.some((role) => !MATRIX_VARIANT_ROLES.has(role))) {
+        errors.push("matrixContract.variantRoles must contain one supported unique role per variant");
+      }
+      if (!Array.isArray(matrix.surfaces)
+        || !plainObject(input.executionContract)
+        || JSON.stringify(matrix.surfaces) !== JSON.stringify(input.executionContract.surfaces)) {
+        errors.push("matrixContract.surfaces must match executionContract.surfaces");
+      }
+      if (matrix.repeatsPerVariant !== input.defaultRepeats) {
+        errors.push("matrixContract.repeatsPerVariant must match defaultRepeats");
+      }
+      const expectedPairs = matrix.familyCount * matrix.variantsPerFamily * matrix.repeatsPerVariant;
+      const expectedSessions = expectedPairs * (Array.isArray(matrix.surfaces) ? matrix.surfaces.length : 0);
+      if (matrix.expectedPairs !== expectedPairs) errors.push("matrixContract.expectedPairs does not match the declared matrix");
+      if (matrix.expectedSessions !== expectedSessions) errors.push("matrixContract.expectedSessions does not match the declared matrix");
+      if (matrix.confidenceSampleUnit !== "task-family") errors.push("matrixContract.confidenceSampleUnit must be task-family");
     }
   }
   if (input.schemaVersion === 2 && (!plainObject(input.assurance) || !plainObject(input.releaseGate))) {
@@ -223,6 +311,36 @@ export function benchmarkSuiteValidationErrors(input) {
     && (!Number.isFinite(input.releaseGate?.maximumFreshTokenRatioUpper95) || input.releaseGate.maximumFreshTokenRatioUpper95 > 0.8)) {
     errors.push("schemaVersion 2 token-saving claims require releaseGate.maximumFreshTokenRatioUpper95 at or below 0.8");
   }
+  if (input.schemaVersion === 2 && input.releaseGate?.requireEfficiencyClaim === true
+    && input.releaseGate?.maximumAllAttemptPooledFreshTokenRatio !== undefined
+    && input.releaseGate.maximumAllAttemptPooledFreshTokenRatio > 0.65) {
+    errors.push("schemaVersion 2 net-35 claims require releaseGate.maximumAllAttemptPooledFreshTokenRatio at or below 0.65");
+  }
+  if (input.id === "production-v2") {
+    if (input.executionContract?.serviceTier !== "fast") errors.push("production-v2 requires executionContract.serviceTier fast");
+    if (input.releaseGate?.requireFastServiceTier !== true) errors.push("production-v2 requires releaseGate.requireFastServiceTier true");
+    if (input.releaseGate?.requireCampaignAccounting !== true) errors.push("production-v2 requires releaseGate.requireCampaignAccounting true");
+    if (!Number.isFinite(input.releaseGate?.maximumAllAttemptPooledFreshTokenRatio)
+      || input.releaseGate.maximumAllAttemptPooledFreshTokenRatio > 0.65) {
+      errors.push("production-v2 requires an all-attempt pooled fresh-token ratio at or below 0.65");
+    }
+  }
+  if (input.id === "production-v3") {
+    if (input.executionContract?.codexMode !== "controlled") {
+      errors.push("production-v3 requires executionContract.codexMode controlled");
+    }
+    if (input.executionContract?.codexBaseline !== "stock") {
+      errors.push("production-v3 requires executionContract.codexBaseline stock");
+    }
+    if (input.assurance?.claimTier !== "public-regression"
+      || input.assurance?.familyDisjointSplit !== false) {
+      errors.push("production-v3 requires the public-regression non-family-disjoint claim boundary");
+    }
+    if (input.matrixContract?.familyCount !== 9 || input.matrixContract?.variantsPerFamily !== 3
+      || input.matrixContract?.repeatsPerVariant !== 2 || input.matrixContract?.expectedSessions !== 108) {
+      errors.push("production-v3 requires the exact 9-family, 3-variant, 2-repeat, 2-surface matrix");
+    }
+  }
   if (input.schemaVersion === 2 && input.releaseGate?.requireEfficiencyClaim === true && input.releaseGate?.requireFullSuiteForClaim !== true) {
     errors.push("schemaVersion 2 token-saving claims require releaseGate.requireFullSuiteForClaim true");
   }
@@ -254,6 +372,7 @@ export function benchmarkSuiteValidationErrors(input) {
   }
 
   const ids = new Set();
+  const matrixFamilies = new Map();
   for (const [index, scenario] of input.scenarios.entries()) {
     const label = `scenarios[${index}]`;
     if (!plainObject(scenario)) {
@@ -310,6 +429,94 @@ export function benchmarkSuiteValidationErrors(input) {
       !Array.isArray(scenario.requiredOutputSubstrings)
       || scenario.requiredOutputSubstrings.some((item) => typeof item !== "string" || !item)
     )) errors.push(`${label}.requiredOutputSubstrings must contain non-empty strings`);
+    if (input.matrixContract === undefined
+      && [scenario.familyId, scenario.variantId, scenario.variantRole, scenario.userJourney].some((value) => value !== undefined)) {
+      errors.push(`${label} matrix metadata requires matrixContract`);
+    }
+    if (plainObject(input.matrixContract)) {
+      requiredString(scenario.familyId, `${label}.familyId`, errors);
+      if (typeof scenario.familyId === "string" && !ID_PATTERN.test(scenario.familyId)) errors.push(`${label}.familyId must use lowercase kebab-case`);
+      requiredString(scenario.variantId, `${label}.variantId`, errors);
+      if (typeof scenario.variantId === "string" && !ID_PATTERN.test(scenario.variantId)) errors.push(`${label}.variantId must use lowercase kebab-case`);
+      if (!MATRIX_VARIANT_ROLES.has(scenario.variantRole)
+        || !Array.isArray(input.matrixContract.variantRoles)
+        || !input.matrixContract.variantRoles.includes(scenario.variantRole)) errors.push(`${label}.variantRole is invalid`);
+      if (!plainObject(scenario.userJourney)) errors.push(`${label}.userJourney must be an object`);
+      else {
+        for (const field of Object.keys(scenario.userJourney)) {
+          if (!USER_JOURNEY_FIELDS.has(field)) errors.push(`${label}.userJourney has unsupported field ${field}`);
+        }
+        const turns = scenario.userJourney.turns;
+        if (!Array.isArray(turns) || turns.length < 1 || turns.length > 10) errors.push(`${label}.userJourney.turns must contain between 1 and 10 turns`);
+        else {
+          const turnIds = new Set();
+          for (const [turnIndex, turn] of turns.entries()) {
+            const turnLabel = `${label}.userJourney.turns[${turnIndex}]`;
+            if (!plainObject(turn)) {
+              errors.push(`${turnLabel} must be an object`);
+              continue;
+            }
+            for (const field of Object.keys(turn)) {
+              if (!USER_JOURNEY_TURN_FIELDS.has(field)) errors.push(`${turnLabel} has unsupported field ${field}`);
+            }
+            requiredString(turn.id, `${turnLabel}.id`, errors);
+            if (typeof turn.id === "string" && !ID_PATTERN.test(turn.id)) errors.push(`${turnLabel}.id must use lowercase kebab-case`);
+            if (turnIds.has(turn.id)) errors.push(`${label}.userJourney has duplicate turn id ${turn.id}`);
+            turnIds.add(turn.id);
+            if (!safeRelativePath(turn.prompt)) errors.push(`${turnLabel}.prompt must stay inside the suite directory`);
+            if (turn.workflow !== undefined
+              && (typeof turn.workflow !== "string" || !ID_PATTERN.test(turn.workflow) || turn.workflow.length > 64)) {
+              errors.push(`${turnLabel}.workflow must be a bounded lowercase kebab-case id`);
+            }
+            if (turn.reconnectBefore !== undefined && typeof turn.reconnectBefore !== "boolean") errors.push(`${turnLabel}.reconnectBefore must be a boolean`);
+            if (turn.receiptUncertain !== undefined && typeof turn.receiptUncertain !== "boolean") {
+              errors.push(`${turnLabel}.receiptUncertain must be a boolean`);
+            }
+            if (turn.receiptUncertain === true && (turnIndex === 0 || turn.reconnectBefore !== true)) {
+              errors.push(`${turnLabel}.receiptUncertain requires an existing-session turn with reconnectBefore`);
+            }
+          }
+          if (scenario.variantRole === "boundary" && (turns.length !== 1 || turns[0]?.id !== "request")) {
+            errors.push(`${label}.boundary journey must contain one request turn`);
+          }
+          if (scenario.variantRole === "interaction"
+            && JSON.stringify(turns.map((turn) => turn?.id)) !== JSON.stringify(["scout", "implement", "verify"])) {
+            errors.push(`${label}.interaction journey must contain scout, implement, verify turns`);
+          }
+          if (scenario.variantRole === "adversarial-recovery"
+            && (turns.length < 2 || !turns.slice(1).some((turn) => turn?.reconnectBefore === true))) {
+            errors.push(`${label}.adversarial-recovery journey must exercise reconnect recovery`);
+          }
+        }
+        if (!TERMINAL_SETTLEMENTS.has(scenario.userJourney.expectedTerminalSettlement)) {
+          errors.push(`${label}.userJourney.expectedTerminalSettlement is invalid`);
+        } else if (scenario.kind === "safety-refusal" && scenario.userJourney.expectedTerminalSettlement !== "refused") {
+          errors.push(`${label}.safety-refusal journey must settle as refused`);
+        } else if (scenario.kind !== "safety-refusal" && scenario.userJourney.expectedTerminalSettlement !== "completed") {
+          errors.push(`${label}.non-refusal journey must settle as completed`);
+        }
+      }
+      if (typeof scenario.familyId === "string" && typeof scenario.variantId === "string") {
+        const family = matrixFamilies.get(scenario.familyId) ?? { variants: new Set(), roles: new Set() };
+        if (family.variants.has(scenario.variantId)) errors.push(`duplicate matrix variant ${scenario.familyId}/${scenario.variantId}`);
+        family.variants.add(scenario.variantId);
+        if (typeof scenario.variantRole === "string") family.roles.add(scenario.variantRole);
+        matrixFamilies.set(scenario.familyId, family);
+      }
+    }
+  }
+  if (plainObject(input.matrixContract)) {
+    if (matrixFamilies.size !== input.matrixContract.familyCount) errors.push("matrixContract.familyCount does not match scenarios");
+    for (const [familyId, family] of matrixFamilies) {
+      if (family.variants.size !== input.matrixContract.variantsPerFamily) errors.push(`matrix family ${familyId} must contain ${input.matrixContract.variantsPerFamily} variants`);
+      const expectedRoles = new Set(Array.isArray(input.matrixContract.variantRoles) ? input.matrixContract.variantRoles : []);
+      if (family.roles.size !== expectedRoles.size || [...expectedRoles].some((role) => !family.roles.has(role))) {
+        errors.push(`matrix family ${familyId} must contain every declared variant role`);
+      }
+    }
+    if (input.scenarios.length !== input.matrixContract.familyCount * input.matrixContract.variantsPerFamily) {
+      errors.push("matrixContract scenario cardinality does not match scenarios");
+    }
   }
   return errors;
 }

package/packages/piagent-core/benchmark/benchmark-summary-support.js

@@ -32,7 +32,7 @@ export const RELEASE_FAILURE_MESSAGES = Object.freeze({
   "primary-efficiency": "The predeclared primary efficiency estimand lacks complete family/category evidence or its upper 95% ratio exceeds the suite limit.",
   "codex-relative-efficiency": "The fixed-workload Piagent/Codex CLI comparison lacks exact parity or quality evidence, exceeds 0.70 total token traffic or API-equivalent cost at the point/upper-95 bound, or exceeds the subagent budget.",
   "production-subagent-budget": "Production Piagent attempts lack exact child-session accounting, exceed one subagent per attempt, or spend more than 5% of candidate provider traffic on subagents.",
-  "provider-free-evidence": "The production run lacks same-source clean-commit runtime-conformance, long-horizon, or WebUI-parity evidence bound to exact lane configuration and runner bytes.",
+  "provider-free-evidence": "The production run lacks same-source clean-commit architecture, runtime-conformance, long-horizon, or WebUI-parity evidence bound to exact lane configuration and runner bytes.",
   "adaptive-context-runtime-coverage": "At least one Piagent runtime receipt reports partial adaptive-context evidence, or not-observed evidence lacks a same-source provider-free coverage lane.",
   "repeat-count": "The run used fewer repeats than the suite minimum.",
   "efficiency-confidence": "The upper 95% token-ratio bound exceeds the suite limit.",
@@ -40,6 +40,8 @@ export const RELEASE_FAILURE_MESSAGES = Object.freeze({
   "efficiency-family-ratio": "At least one comparable scenario family uses more fresh tokens than the suite family guardrail permits.",
   "accepted-usage-completeness": "Every accepted benchmark attempt must have exact terminal token buckets.",
   "all-attempt-usage-completeness": "Every accepted or failed provider attempt must have exact invariant-checked token buckets.",
+  "all-attempt-net-efficiency": "All accepted plus exact failed/retry fresh tokens must remain at or below the predeclared pooled Piagent/Codex CLI ratio.",
+  "fast-execution-configuration-parity": "Fast execution configuration must be verified on every run: exact Piagent outbound telemetry plus identity-bound effective Codex CLI thread settings; missing or default evidence blocks the claim.",
   "normalized-cost-configuration": "Normalized-cost claims require a valid versioned pricing snapshot and all predeclared ratio thresholds.",
   "normalized-cost-evidence": "Normalized API-equivalent text-token cost evidence is incomplete for at least one declared scenario family.",
   "normalized-cost-pricing-applicability": "Exact usage or per-request pricing applicability is unavailable for at least one accepted run.",
@@ -54,7 +56,7 @@ export const RELEASE_FAILURE_MESSAGES = Object.freeze({
   "infrastructure-retries": "Accepted benchmark runs used more recovered infrastructure retries than the suite permits.",
   "infrastructure-failure-ledger": "Accepted benchmark runs do not contain an exact attempt/retry/failure ledger.",
   "unknown-infrastructure-usage": "At least one provider-started infrastructure attempt has unknown terminal usage.",
-  "canonical-production-identity": "The production-v1 suite id is reserved for the canonical built-in suite identity.",
+  "canonical-production-identity": "Production suite ids are reserved for their canonical built-in suite identities.",
   "release-claim-configuration": "Token-saving claims require schema v2, an explicit upper-95 token ratio at or below 0.80, full-suite enforcement, and provider-wire stability evidence.",
   "codex-baseline": "Token-saving product claims require controlled Codex CLI as the paired baseline.",
   "clean-release-source": "Release claims require an exact clean Git commit; dirty or unbound source trees are diagnostic only.",
@@ -378,3 +380,29 @@ export function pairedQualityNoninferiorityEvidence({ suite, repeats, baselineRu
     families
   };
 }
+
+export function conditionalBothPassFreshUsage({ suite, repeats, allPairs, pairs, tokenPairs, baselineRuns, candidateRuns, allSuccessfulPairsFreshRatio }) {
+  return {
+    outcomeDefinition: "both-accepted-runs-resolved-including-safety-refusals",
+    usageDefinition: "exact-positive-fresh-same-model-and-thinking",
+    attemptScope: "accepted-attempts-only; all-attempt-costs-remain-in-tokenAccounting",
+    estimator: "geometric-mean-of-candidate-over-baseline-pair-ratios",
+    expectedPairs: suite.scenarios.length * repeats,
+    observedPairs: allPairs.length,
+    bothResolvedPairs: pairs.length,
+    eligiblePairs: tokenPairs.length,
+    excludedUnresolvedPairs: allPairs.length - pairs.length,
+    excludedUsagePairs: pairs.length - tokenPairs.length,
+    unpairedAcceptedRuns: baselineRuns.length + candidateRuns.length - 2 * allPairs.length,
+    ratio: rounded(allSuccessfulPairsFreshRatio, 4),
+    interpretation: "conditional-observation-only; not-overall-token-savings"
+  };
+}
+
+export function benchmarkOutcomePairs(baselineRuns, candidateRuns) {
+  const baselineByKey = new Map(baselineRuns.map((run) => [`${run.scenarioId}:${run.repeat}`, run]));
+  const allPairs = candidateRuns
+    .map((run) => ({ baseline: baselineByKey.get(`${run.scenarioId}:${run.repeat}`), candidate: run }))
+    .filter((pair) => pair.baseline);
+  return allPairs;
+}