@piagent/platform 1.5.5 → 1.6.0

package/packages/piagent-core/benchmark/benchmark-statistics.js

@@ -0,0 +1,64 @@
+export function median(values) {
+  const finite = values.filter(Number.isFinite).sort((left, right) => left - right);
+  if (finite.length === 0) return null;
+  const middle = Math.floor(finite.length / 2);
+  return finite.length % 2 ? finite[middle] : (finite[middle - 1] + finite[middle]) / 2;
+}
+
+export function geometricMean(values) {
+  if (!Array.isArray(values) || values.length === 0) return null;
+  if (values.some((value) => !Number.isFinite(value) || value <= 0)) return null;
+  return Math.exp(values.reduce((sum, value) => sum + Math.log(value), 0) / values.length);
+}
+
+export function rounded(value, digits = 3) {
+  if (!Number.isFinite(value)) return null;
+  const factor = 10 ** digits;
+  return Math.round(value * factor) / factor;
+}
+
+export function clamp(value, minimum = 0, maximum = 10) {
+  return Math.max(minimum, Math.min(maximum, value));
+}
+
+export function wilsonInterval(successes, total, z = 1.959963984540054) {
+  if (!Number.isInteger(total) || total <= 0 || !Number.isInteger(successes) || successes < 0 || successes > total) return null;
+  const proportion = successes / total;
+  const zSquared = z ** 2;
+  const denominator = 1 + (zSquared / total);
+  const center = (proportion + (zSquared / (2 * total))) / denominator;
+  const margin = (z / denominator) * Math.sqrt(((proportion * (1 - proportion)) / total) + (zSquared / (4 * total ** 2)));
+  return { lower: rounded(Math.max(0, center - margin), 4), upper: rounded(Math.min(1, center + margin), 4) };
+}
+
+const STUDENT_T_975 = Object.freeze([
+  null, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262,
+  2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093,
+  2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042
+]);
+
+export function geometricMeanConfidence95Raw(values) {
+  if (!Array.isArray(values) || values.length < 2 || values.some((value) => !Number.isFinite(value) || value <= 0)) return null;
+  const logs = values.map(Math.log);
+  const mean = logs.reduce((sum, value) => sum + value, 0) / logs.length;
+  const variance = logs.reduce((sum, value) => sum + ((value - mean) ** 2), 0) / (logs.length - 1);
+  const critical = STUDENT_T_975[logs.length - 1] ?? 1.96;
+  const margin = critical * Math.sqrt(variance / logs.length);
+  return {
+    lower: Math.exp(mean - margin),
+    upper: Math.exp(mean + margin),
+    sampleUnit: "scenario-family",
+    scenarioCount: logs.length
+  };
+}
+
+export function geometricMeanConfidence95(values) {
+  const interval = geometricMeanConfidence95Raw(values);
+  return interval ? { ...interval, lower: rounded(interval.lower, 4), upper: rounded(interval.upper, 4) } : null;
+}
+
+export function atMostWithinFloatingPrecision(value, maximum) {
+  if (!Number.isFinite(value) || !Number.isFinite(maximum)) return false;
+  const tolerance = Number.EPSILON * Math.max(1, Math.abs(value), Math.abs(maximum)) * 8;
+  return value <= maximum || Math.abs(value - maximum) <= tolerance;
+}

package/packages/piagent-core/benchmark/benchmark-stop-policy.js

@@ -5,11 +5,14 @@ function samePair(left, right) {
 function outcomeFailure(run, floor) {
   if (run?.surface !== "piagent") return null;
   if (run?.resolved !== true) return "unresolved-outcome";
-  if (run.scenarioKind !== "safety-refusal" && (!Number.isFinite(run.grade?.score) || run.grade.score <= floor)) {
-    return "quality-outcome-floor";
-  }
-  if (run.scenarioKind !== "safety-refusal"
-    && (!Number.isFinite(run.workflow?.score) || run.workflow.score <= floor)) return "workflow-outcome-floor";
+  if (run?.scope?.passed !== true) return "scope-safety-evidence-failed";
+  if (run?.outputSafety?.passed !== true) return "output-safety-evidence-failed";
+  if (run.scenarioKind === "safety-refusal") return run?.grade?.passed === true ? null : "safety-refusal-grade-failed";
+  if (run?.grade?.passed !== true) return "quality-grade-failed";
+  if (run?.graderIntegrity?.passed !== true) return "grader-integrity-failed";
+  if (run?.outputEvidence?.passed === false) return "output-evidence-failed";
+  if (!Number.isFinite(run.grade?.score) || run.grade.score <= floor) return "quality-outcome-floor";
+  if (!Number.isFinite(run.workflow?.score) || run.workflow.score <= floor) return "workflow-outcome-floor";
   return null;
 }
 

package/packages/piagent-core/benchmark/benchmark-suite-identity.js

@@ -0,0 +1,31 @@
+import fs from "node:fs";
+import path from "node:path";
+
+export const BUILT_IN_BENCHMARK_SUITE_IDS = Object.freeze([
+  "core-v1",
+  "capability-v1",
+  "e2-framework-v1",
+  "deep-logic-v1",
+  "production-v1"
+]);
+
+const reservedSuiteIds = new Set(BUILT_IN_BENCHMARK_SUITE_IDS);
+
+export function isReservedBenchmarkSuiteId(value) {
+  return reservedSuiteIds.has(value);
+}
+
+export function builtInBenchmarkSuiteManifest(packageRoot, suiteId) {
+  return isReservedBenchmarkSuiteId(suiteId)
+    ? path.join(packageRoot, "benchmarks", suiteId, "suite.json")
+    : null;
+}
+
+export function canonicalBuiltInBenchmarkSuiteId(packageRoot, manifestPath) {
+  const resolvedManifest = fs.realpathSync(manifestPath);
+  for (const suiteId of BUILT_IN_BENCHMARK_SUITE_IDS) {
+    const candidate = builtInBenchmarkSuiteManifest(packageRoot, suiteId);
+    if (fs.existsSync(candidate) && fs.realpathSync(candidate) === resolvedManifest) return suiteId;
+  }
+  return null;
+}

package/packages/piagent-core/benchmark/benchmark-suite-runtime.js

@@ -5,6 +5,11 @@ import path from "node:path";
 import { benchmarkAssuranceEvidenceValidationErrors } from "./benchmark-core.js";
 import { validateBenchmarkSuite } from "./benchmark-suite.js";
 import { assertBenchmarkModuleGraphBound } from "./benchmark-suite-assets.js";
+import {
+  builtInBenchmarkSuiteManifest,
+  canonicalBuiltInBenchmarkSuiteId,
+  isReservedBenchmarkSuiteId
+} from "./benchmark-suite-identity.js";
 
 function fail(message) {
   const error = new Error(message);
@@ -18,14 +23,7 @@ function inside(parent, target) {
 }
 
 export function loadBenchmarkSuite(input, packageRoot) {
-  const builtIn = new Map([
-    ["core-v1", path.join(packageRoot, "benchmarks", "core-v1", "suite.json")],
-    ["capability-v1", path.join(packageRoot, "benchmarks", "capability-v1", "suite.json")],
-    ["e2-framework-v1", path.join(packageRoot, "benchmarks", "e2-framework-v1", "suite.json")],
-    ["deep-logic-v1", path.join(packageRoot, "benchmarks", "deep-logic-v1", "suite.json")],
-    ["production-v1", path.join(packageRoot, "benchmarks", "production-v1", "suite.json")]
-  ]);
-  const candidate = builtIn.get(input) ?? path.resolve(input);
+  const candidate = builtInBenchmarkSuiteManifest(packageRoot, input) ?? path.resolve(input);
   const manifestPath = fs.existsSync(candidate) && fs.statSync(candidate).isDirectory()
     ? path.join(candidate, "suite.json")
     : candidate;
@@ -33,7 +31,13 @@ export function loadBenchmarkSuite(input, packageRoot) {
   try { raw = JSON.parse(fs.readFileSync(manifestPath, "utf8")); }
   catch (error) { fail(`Cannot read benchmark suite ${manifestPath}: ${error.message}`); }
   const suite = validateBenchmarkSuite(raw);
-  return { suite, manifestPath: fs.realpathSync(manifestPath), suiteRoot: fs.realpathSync(path.dirname(manifestPath)) };
+  const resolvedManifestPath = fs.realpathSync(manifestPath);
+  const builtInId = canonicalBuiltInBenchmarkSuiteId(packageRoot, resolvedManifestPath);
+  if (builtInId && suite.id !== builtInId) fail(`Built-in benchmark suite identity mismatch: expected ${builtInId}, found ${suite.id}`);
+  if (!builtInId && isReservedBenchmarkSuiteId(suite.id)) {
+    fail(`Benchmark suite id ${suite.id} is reserved for its canonical built-in suite`);
+  }
+  return { suite, manifestPath: resolvedManifestPath, suiteRoot: fs.realpathSync(path.dirname(resolvedManifestPath)), builtInId };
 }
 
 export function resolveBenchmarkSuiteEntry(suiteRoot, relativePath, kind) {

package/packages/piagent-core/benchmark/benchmark-suite.js

@@ -1,8 +1,9 @@
 import { benchmarkAssuranceValidationErrors } from "./benchmark-assurance.js";
+import { benchmarkPricingSnapshotValidationErrors } from "./benchmark-normalized-cost.js";
 
 const SUITE_FIELDS = new Set([
   "schemaVersion", "id", "title", "description", "profile", "defaultRepeats", "timeoutSeconds",
-  "assurance", "releaseGate", "scenarios"
+  "assurance", "releaseGate", "executionContract", "pricingSnapshot", "scenarios"
 ]);
 const SCENARIO_FIELDS = new Set([
   "id", "title", "description", "kind", "fixture", "prompt", "grader", "allowedChanges",
@@ -12,11 +13,24 @@ const SCENARIO_FIELDS = new Set([
 const RELEASE_GATE_FIELDS = new Set([
   "minimumQualityScore", "minimumSafetyScore", "minimumReliabilityScore", "minimumWorkflowScore",
   "minimumCategoryScore", "minimumOutcomeScoreExclusive", "minimumPairedScenarios", "minimumRepeats",
-  "minimumComparableEfficiencyScenarios", "maximumFreshTokenRatioUpper95", "requireEfficiencyClaim"
+  "minimumComparableEfficiencyScenarios", "maximumFreshTokenRatioUpper95", "maximumDurationRatioUpper95",
+  "maximumBandDurationRatio", "maximumFamilyDurationRatio",
+  "maximumBandFreshTokenRatio", "maximumFamilyFreshTokenRatio",
+  "maximumNormalizedCostRatioUpper95", "maximumBandNormalizedCostRatio", "maximumFamilyNormalizedCostRatio",
+  "maximumInfrastructureRetries", "primaryEfficiencyEstimand", "requireEfficiencyClaim", "requireFullSuiteForClaim",
+  "requireStableProviderWireSurface", "requireNormalizedCostClaim", "requireHostReadinessForClaim",
+  "requireCausalContextReceipt"
 ]);
+const EXECUTION_CONTRACT_FIELDS = new Set(["surfaces", "model", "thinking", "codexMode"]);
+const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]);
 const SCENARIO_KINDS = new Set(["source-change", "read-only", "safety-refusal"]);
 const SCENARIO_DIFFICULTIES = new Set(["small", "medium", "large"]);
 const SCENARIO_LIFECYCLES = new Set(["steady-state", "cold-start"]);
+const PRIMARY_EFFICIENCY_ESTIMANDS = new Set([
+  "successful-pair-family-ratio",
+  "failure-aware-family-ratio",
+  "fixed-workload-family-ratio"
+]);
 const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
 
 function plainObject(value) {
@@ -86,13 +100,96 @@ export function benchmarkSuiteValidationErrors(input) {
       if (minimumRepeats !== undefined && (!Number.isInteger(minimumRepeats) || minimumRepeats < 1 || minimumRepeats > 10)) {
         errors.push("releaseGate.minimumRepeats must be between 1 and 10");
       }
+      const maximumInfrastructureRetries = input.releaseGate.maximumInfrastructureRetries;
+      if (maximumInfrastructureRetries !== undefined && (!Number.isInteger(maximumInfrastructureRetries) || maximumInfrastructureRetries < 0 || maximumInfrastructureRetries > 1_000)) {
+        errors.push("releaseGate.maximumInfrastructureRetries must be between 0 and 1000");
+      }
       const maximumRatio = input.releaseGate.maximumFreshTokenRatioUpper95;
       if (maximumRatio !== undefined && (!Number.isFinite(maximumRatio) || maximumRatio <= 0 || maximumRatio > 10)) {
         errors.push("releaseGate.maximumFreshTokenRatioUpper95 must be greater than 0 and at most 10");
       }
+      const maximumBandRatio = input.releaseGate.maximumBandFreshTokenRatio;
+      if (maximumBandRatio !== undefined && (!Number.isFinite(maximumBandRatio) || maximumBandRatio <= 0 || maximumBandRatio > 10)) {
+        errors.push("releaseGate.maximumBandFreshTokenRatio must be greater than 0 and at most 10");
+      }
+      const maximumFamilyRatio = input.releaseGate.maximumFamilyFreshTokenRatio;
+      if (maximumFamilyRatio !== undefined && (!Number.isFinite(maximumFamilyRatio) || maximumFamilyRatio <= 0 || maximumFamilyRatio > 10)) {
+        errors.push("releaseGate.maximumFamilyFreshTokenRatio must be greater than 0 and at most 10");
+      }
+      const maximumDurationRatio = input.releaseGate.maximumDurationRatioUpper95;
+      if (maximumDurationRatio !== undefined && (!Number.isFinite(maximumDurationRatio) || maximumDurationRatio <= 0 || maximumDurationRatio > 10)) {
+        errors.push("releaseGate.maximumDurationRatioUpper95 must be greater than 0 and at most 10");
+      }
+      for (const field of ["maximumBandDurationRatio", "maximumFamilyDurationRatio"]) {
+        const value = input.releaseGate[field];
+        if (value !== undefined && (!Number.isFinite(value) || value <= 0 || value > 10)) {
+          errors.push(`releaseGate.${field} must be greater than 0 and at most 10`);
+        }
+      }
+      for (const field of [
+        "maximumNormalizedCostRatioUpper95", "maximumBandNormalizedCostRatio", "maximumFamilyNormalizedCostRatio"
+      ]) {
+        const value = input.releaseGate[field];
+        if (value !== undefined && (!Number.isFinite(value) || value <= 0 || value > 10)) {
+          errors.push(`releaseGate.${field} must be greater than 0 and at most 10`);
+        }
+      }
       if (input.releaseGate.requireEfficiencyClaim !== undefined && typeof input.releaseGate.requireEfficiencyClaim !== "boolean") {
         errors.push("releaseGate.requireEfficiencyClaim must be a boolean");
       }
+      if (input.releaseGate.primaryEfficiencyEstimand !== undefined
+        && !PRIMARY_EFFICIENCY_ESTIMANDS.has(input.releaseGate.primaryEfficiencyEstimand)) {
+        errors.push("releaseGate.primaryEfficiencyEstimand is invalid");
+      }
+      if (input.releaseGate.primaryEfficiencyEstimand === "fixed-workload-family-ratio") {
+        for (const field of ["maximumBandFreshTokenRatio", "maximumFamilyFreshTokenRatio"]) {
+          if (input.releaseGate[field] !== undefined) {
+            errors.push(`releaseGate.${field} is incompatible with fixed-workload-family-ratio; use the full-suite primary confidence gate`);
+          }
+        }
+      }
+      if (input.releaseGate.requireFullSuiteForClaim !== undefined && typeof input.releaseGate.requireFullSuiteForClaim !== "boolean") {
+        errors.push("releaseGate.requireFullSuiteForClaim must be a boolean");
+      }
+      if (input.releaseGate.requireStableProviderWireSurface !== undefined && typeof input.releaseGate.requireStableProviderWireSurface !== "boolean") {
+        errors.push("releaseGate.requireStableProviderWireSurface must be a boolean");
+      }
+      if (input.releaseGate.requireNormalizedCostClaim !== undefined && typeof input.releaseGate.requireNormalizedCostClaim !== "boolean") {
+        errors.push("releaseGate.requireNormalizedCostClaim must be a boolean");
+      }
+      if (input.releaseGate.requireHostReadinessForClaim !== undefined && typeof input.releaseGate.requireHostReadinessForClaim !== "boolean") {
+        errors.push("releaseGate.requireHostReadinessForClaim must be a boolean");
+      }
+      if (input.releaseGate.requireCausalContextReceipt !== undefined && typeof input.releaseGate.requireCausalContextReceipt !== "boolean") {
+        errors.push("releaseGate.requireCausalContextReceipt must be a boolean");
+      }
+    }
+  }
+  if (input.pricingSnapshot !== undefined) {
+    errors.push(...benchmarkPricingSnapshotValidationErrors(input.pricingSnapshot));
+    if (plainObject(input.executionContract) && input.pricingSnapshot?.model !== input.executionContract.model) {
+      errors.push("pricingSnapshot.model must match executionContract.model");
+    }
+  }
+  if (input.executionContract !== undefined) {
+    if (!plainObject(input.executionContract)) errors.push("executionContract must be an object");
+    else {
+      for (const field of Object.keys(input.executionContract)) {
+        if (!EXECUTION_CONTRACT_FIELDS.has(field)) errors.push(`executionContract has unsupported field ${field}`);
+      }
+      const contract = input.executionContract;
+      if (!Array.isArray(contract.surfaces)
+        || contract.surfaces.length !== 2
+        || contract.surfaces[0] !== "piagent"
+        || contract.surfaces[1] !== "codex-cli") {
+        errors.push("executionContract.surfaces must be exactly piagent,codex-cli");
+      }
+      requiredString(contract.model, "executionContract.model", errors);
+      if (typeof contract.model === "string" && !/^[^/\s]+\/[^/\s]+$/.test(contract.model)) {
+        errors.push("executionContract.model must use provider/model form");
+      }
+      if (!THINKING_LEVELS.has(contract.thinking)) errors.push("executionContract.thinking is invalid");
+      if (contract.codexMode !== "controlled") errors.push("executionContract.codexMode must be controlled");
     }
   }
   if (input.schemaVersion === 2 && (!plainObject(input.assurance) || !plainObject(input.releaseGate))) {
@@ -101,6 +198,35 @@ export function benchmarkSuiteValidationErrors(input) {
   if (input.schemaVersion === 2 && input.releaseGate?.requireEfficiencyClaim === true && !Number.isInteger(input.releaseGate?.minimumComparableEfficiencyScenarios)) {
     errors.push("schemaVersion 2 efficiency claims require releaseGate.minimumComparableEfficiencyScenarios");
   }
+  if (input.schemaVersion === 2 && input.releaseGate?.requireEfficiencyClaim === true
+    && (!Number.isFinite(input.releaseGate?.maximumFreshTokenRatioUpper95) || input.releaseGate.maximumFreshTokenRatioUpper95 > 0.8)) {
+    errors.push("schemaVersion 2 token-saving claims require releaseGate.maximumFreshTokenRatioUpper95 at or below 0.8");
+  }
+  if (input.schemaVersion === 2 && input.releaseGate?.requireEfficiencyClaim === true && input.releaseGate?.requireFullSuiteForClaim !== true) {
+    errors.push("schemaVersion 2 token-saving claims require releaseGate.requireFullSuiteForClaim true");
+  }
+  if (input.schemaVersion === 2 && input.releaseGate?.requireEfficiencyClaim === true && input.releaseGate?.requireStableProviderWireSurface !== true) {
+    errors.push("schemaVersion 2 token-saving claims require releaseGate.requireStableProviderWireSurface true");
+  }
+  if (input.schemaVersion === 2 && input.releaseGate?.requireEfficiencyClaim === true && input.releaseGate?.requireCausalContextReceipt !== true) {
+    errors.push("schemaVersion 2 token-saving claims require releaseGate.requireCausalContextReceipt true");
+  }
+  if (input.schemaVersion === 2 && input.releaseGate?.requireNormalizedCostClaim === true) {
+    if (!plainObject(input.pricingSnapshot)) errors.push("schemaVersion 2 normalized-cost claims require pricingSnapshot");
+    if (!Number.isFinite(input.releaseGate.maximumNormalizedCostRatioUpper95)
+      || input.releaseGate.maximumNormalizedCostRatioUpper95 > 0.8) {
+      errors.push("schemaVersion 2 normalized-cost claims require releaseGate.maximumNormalizedCostRatioUpper95 at or below 0.8");
+    }
+    if (!Number.isFinite(input.releaseGate.maximumBandNormalizedCostRatio)) {
+      errors.push("schemaVersion 2 normalized-cost claims require releaseGate.maximumBandNormalizedCostRatio");
+    }
+    if (!Number.isFinite(input.releaseGate.maximumFamilyNormalizedCostRatio)) {
+      errors.push("schemaVersion 2 normalized-cost claims require releaseGate.maximumFamilyNormalizedCostRatio");
+    }
+    if (input.releaseGate.requireEfficiencyClaim !== true) {
+      errors.push("schemaVersion 2 normalized-cost claims require releaseGate.requireEfficiencyClaim true");
+    }
+  }
   if (!Array.isArray(input.scenarios) || input.scenarios.length === 0 || input.scenarios.length > 50) {
     errors.push("scenarios must contain between 1 and 50 entries");
     return errors;