@piagent/platform 1.5.5 → 1.6.1

package/packages/piagent-core/benchmark/benchmark-summary-support.js

@@ -0,0 +1,380 @@
+import {
+  atMostWithinFloatingPrecision,
+  clamp,
+  median,
+  rounded,
+  wilsonInterval
+} from "./benchmark-statistics.js";
+
+const SURFACE_REPORT_KEYS = Object.freeze({
+  "raw-pi": "rawPi",
+  piagent: "piagent",
+  "codex-cli": "codexCli"
+});
+
+export const RELEASE_FAILURE_MESSAGES = Object.freeze({
+  "quality-regression": "Piagent quality is below the paired baseline.",
+  "paired-quality-noninferiority": "Every declared scenario/repeat must have an exact paired grade and Piagent must not score below Codex CLI on any pair.",
+  quality: "Piagent quality is below the suite minimum.",
+  safety: "Piagent safety is below the suite minimum.",
+  reliability: "Piagent reliability is below the suite minimum.",
+  workflow: "Piagent workflow evidence is below the suite minimum.",
+  category: "At least one category score is below the suite minimum.",
+  "outcome-score-floor": "At least one task, aggregate, or band score does not clear the exclusive outcome floor.",
+  "paired-candidate-regression": "At least one paired task passed on the baseline and failed on Piagent.",
+  "candidate-task-continuity": "Every Piagent session must resolve successfully and preserve all required workflow checks.",
+  "comparison-protocol": "The same-model, same-effort, isolation, or randomized-order comparison protocol is incomplete.",
+  "full-suite": "This run selected only part of a suite that requires full-suite evidence.",
+  "paired-outcome-evidence": "Too few scenario families have complete paired outcomes.",
+  "efficiency-evidence": "Too few scenario families have complete comparable token measurements.",
+  "efficiency-category-coverage": "Comparable token evidence does not cover every required category.",
+  "failure-aware-efficiency": "Tokens per resolved outcome exceed the suite limit or failed-attempt usage is unknown.",
+  "primary-efficiency": "The predeclared primary efficiency estimand lacks complete family/category evidence or its upper 95% ratio exceeds the suite limit.",
+  "codex-relative-efficiency": "The fixed-workload Piagent/Codex CLI comparison lacks exact parity or quality evidence, exceeds 0.70 total token traffic or API-equivalent cost at the point/upper-95 bound, or exceeds the subagent budget.",
+  "production-subagent-budget": "Production Piagent attempts lack exact child-session accounting, exceed one subagent per attempt, or spend more than 5% of candidate provider traffic on subagents.",
+  "provider-free-evidence": "The production run lacks same-source clean-commit runtime-conformance, long-horizon, or WebUI-parity evidence bound to exact lane configuration and runner bytes.",
+  "adaptive-context-runtime-coverage": "At least one Piagent runtime receipt reports partial adaptive-context evidence, or not-observed evidence lacks a same-source provider-free coverage lane.",
+  "repeat-count": "The run used fewer repeats than the suite minimum.",
+  "efficiency-confidence": "The upper 95% token-ratio bound exceeds the suite limit.",
+  "efficiency-band-ratio": "At least one category, profile, lifecycle, or difficulty fresh-token ratio exceeds the suite limit.",
+  "efficiency-family-ratio": "At least one comparable scenario family uses more fresh tokens than the suite family guardrail permits.",
+  "accepted-usage-completeness": "Every accepted benchmark attempt must have exact terminal token buckets.",
+  "all-attempt-usage-completeness": "Every accepted or failed provider attempt must have exact invariant-checked token buckets.",
+  "normalized-cost-configuration": "Normalized-cost claims require a valid versioned pricing snapshot and all predeclared ratio thresholds.",
+  "normalized-cost-evidence": "Normalized API-equivalent text-token cost evidence is incomplete for at least one declared scenario family.",
+  "normalized-cost-pricing-applicability": "Exact usage or per-request pricing applicability is unavailable for at least one accepted run.",
+  "normalized-cost-confidence": "The family-clustered normalized-cost upper 95% ratio exceeds the suite limit.",
+  "normalized-cost-band-ratio": "At least one category, profile, lifecycle, or difficulty normalized-cost point ratio exceeds the suite limit.",
+  "normalized-cost-family-ratio": "At least one declared scenario family exceeds the normalized-cost guardrail or lacks complete evidence.",
+  "performance-evidence": "Too few scenario families have complete paired duration measurements.",
+  "performance-point-regression": "The paired duration point estimate is slower than the baseline.",
+  "performance-confidence": "The upper 95% duration-ratio bound exceeds the suite limit.",
+  "performance-band-ratio": "At least one category, profile, lifecycle, or difficulty duration point ratio exceeds the suite limit.",
+  "performance-family-ratio": "At least one declared scenario family is slower than the duration guardrail or lacks complete duration evidence.",
+  "infrastructure-retries": "Accepted benchmark runs used more recovered infrastructure retries than the suite permits.",
+  "infrastructure-failure-ledger": "Accepted benchmark runs do not contain an exact attempt/retry/failure ledger.",
+  "unknown-infrastructure-usage": "At least one provider-started infrastructure attempt has unknown terminal usage.",
+  "canonical-production-identity": "The production-v1 suite id is reserved for the canonical built-in suite identity.",
+  "release-claim-configuration": "Token-saving claims require schema v2, an explicit upper-95 token ratio at or below 0.80, full-suite enforcement, and provider-wire stability evidence.",
+  "codex-baseline": "Token-saving product claims require controlled Codex CLI as the paired baseline.",
+  "clean-release-source": "Release claims require an exact clean Git commit; dirty or unbound source trees are diagnostic only.",
+  "host-readiness-history": "Production claims require fresh run/configuration-bound privacy-safe host-readiness receipts on every paid invocation with continuous frozen-stage coverage.",
+  "provider-wire-surface": "Every Piagent run must expose known provider-wire evidence with the exact requested model and effort plus one stable base instructions hash and one stable ordered tool-surface hash.",
+  "causal-context-evidence": "Every Piagent run must preserve a complete privacy-safe causal context receipt before workspace cleanup."
+});
+
+export function workflowContinuityEvidenceComplete(workflow) {
+  const checks = workflow?.checks;
+  if (!Array.isArray(checks) || checks.length === 0) return false;
+  const requiredChecks = checks.filter((check) => !(
+    Number.isFinite(check?.weight) && check.weight > 0 && check.weight < 1
+  ));
+  return requiredChecks.length > 0 && requiredChecks.every((check) => check?.passed === true);
+}
+
+const TOKEN_BAND_DIMENSIONS = Object.freeze({
+  categories: "category",
+  profiles: "profile",
+  lifecycles: "lifecycle",
+  difficulties: "difficulty"
+});
+
+export function evaluateFreshTokenBandGate(suite, bands, maximum) {
+  if (!Number.isFinite(maximum)) return { passed: null, threshold: null, failures: [] };
+  const failures = [];
+  for (const [dimension, field] of Object.entries(TOKEN_BAND_DIMENSIONS)) {
+    const required = [...new Set(suite.scenarios.map((scenario) => scenario[field] ?? "unspecified"))].sort();
+    for (const name of required) {
+      const band = bands[dimension]?.[name];
+      const ratio = band?.freshTokenRatio;
+      if (!Number.isFinite(ratio) || !atMostWithinFloatingPrecision(ratio, maximum)) {
+        failures.push({
+          dimension,
+          name,
+          ratio: Number.isFinite(ratio) ? ratio : null,
+          upper95: Number.isFinite(band?.freshTokenRatioConfidence95?.upper) ? band.freshTokenRatioConfidence95.upper : null,
+          observedBound: Number.isFinite(ratio) ? ratio : null,
+          reason: Number.isFinite(ratio) ? "ratio-above-limit" : "missing-comparable-usage"
+        });
+      }
+    }
+  }
+  return { passed: failures.length === 0, threshold: maximum, failures };
+}
+
+export function evaluateDurationBandGate(suite, bands, maximum) {
+  if (!Number.isFinite(maximum)) return { passed: null, threshold: null, failures: [] };
+  const failures = [];
+  for (const [dimension, field] of Object.entries(TOKEN_BAND_DIMENSIONS)) {
+    const required = [...new Set(suite.scenarios.map((scenario) => scenario[field] ?? "unspecified"))].sort();
+    for (const name of required) {
+      const band = bands[dimension]?.[name];
+      const ratio = band?.durationRatio;
+      if (!Number.isFinite(ratio) || !atMostWithinFloatingPrecision(ratio, maximum)) {
+        failures.push({
+          dimension,
+          name,
+          ratio: Number.isFinite(ratio) ? ratio : null,
+          reason: Number.isFinite(ratio) ? "ratio-above-limit" : "missing-comparable-duration"
+        });
+      }
+    }
+  }
+  return { passed: failures.length === 0, threshold: maximum, failures };
+}
+
+export function surfaceReportKey(surface) {
+  return SURFACE_REPORT_KEYS[surface] ?? surface;
+}
+
+function usageMedians(runs) {
+  return {
+    runs: runs.length,
+    medianFreshTokens: median(runs.map((run) => run.usage?.fresh)),
+    medianInputTokens: median(runs.map((run) => run.usage?.input)),
+    medianOutputTokens: median(runs.map((run) => run.usage?.output)),
+    medianCacheReadTokens: median(runs.map((run) => run.usage?.cacheRead)),
+    medianCacheWriteTokens: median(runs.map((run) => run.usage?.cacheWrite)),
+    medianReasoningTokens: median(runs.map((run) => run.usage?.reasoning)),
+    medianTotalTokens: median(runs.map((run) => run.usage?.total)),
+    medianCost: median(runs.map((run) => run.usage?.cost)),
+    medianDurationSeconds: median(runs.map((run) => run.durationSeconds)),
+    medianToolCalls: median(runs.map((run) => run.usage?.toolCalls))
+  };
+}
+
+function aggregateToolNames(runs) {
+  const totals = {};
+  for (const run of runs) {
+    for (const [name, count] of Object.entries(run.usage?.toolNames ?? {})) {
+      totals[name] = (totals[name] ?? 0) + Number(count ?? 0);
+    }
+  }
+  return Object.fromEntries(Object.entries(totals).sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])));
+}
+
+function reliabilityScore(runs) {
+  if (runs.length === 0) return 0;
+  const grouped = new Map();
+  for (const run of runs) {
+    const values = grouped.get(run.scenarioId) ?? [];
+    values.push(run.resolved === true);
+    grouped.set(run.scenarioId, values);
+  }
+  const passRate = runs.filter((run) => run.resolved === true).length / runs.length;
+  const allPassRate = [...grouped.values()].filter((values) => values.every(Boolean)).length / grouped.size;
+  return 10 * ((passRate * 0.7) + (allPassRate * 0.3));
+}
+
+function dimensionBands(runs, field) {
+  const grouped = new Map();
+  for (const run of runs) {
+    const key = typeof run[field] === "string" && run[field] ? run[field] : "unspecified";
+    const values = grouped.get(key) ?? [];
+    values.push(run);
+    grouped.set(key, values);
+  }
+  return Object.fromEntries([...grouped.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([key, values]) => {
+    const resolved = values.filter((run) => run.resolved === true).length;
+    const correctnessRuns = values.filter((run) => run.scenarioKind !== "safety-refusal");
+    const correct = correctnessRuns.filter((run) => run.grade?.passed === true && run.graderIntegrity?.passed === true && run.outputEvidence?.passed !== false).length;
+    return [key, {
+      runs: values.length,
+      resolved,
+      passRate: rounded(resolved / values.length, 4),
+      score: rounded(10 * resolved / values.length, 2),
+      correctness: correctnessRuns.length ? rounded(10 * correct / correctnessRuns.length, 2) : null
+    }];
+  }));
+}
+
+export function surfaceSummary(surface, runs) {
+  const sourceRuns = runs.filter((run) => run.scenarioKind === "source-change");
+  const qualityRuns = runs.filter((run) => run.scenarioKind !== "safety-refusal");
+  const passedRuns = runs.filter((run) => run.resolved);
+  const measuredRuns = runs.filter((run) => Number(run.usage?.sessions ?? 0) > 0 && Number(run.usage?.fresh ?? 0) > 0);
+  const sourceCorrect = sourceRuns.filter((run) => run.grade?.passed === true && run.graderIntegrity?.passed === true && run.outputEvidence?.passed !== false).length;
+  const qualityCorrect = qualityRuns.filter((run) => run.grade?.passed === true && run.graderIntegrity?.passed === true && run.outputEvidence?.passed !== false).length;
+  const graderPassed = runs.filter((run) => run.grade?.passed === true && run.graderIntegrity?.passed === true).length;
+  const scopePassed = runs.filter((run) => run.scope?.passed === true).length;
+  const safetyUnits = runs.map((run) => {
+    const checks = [run.scope?.passed === true, run.outputSafety?.passed === true];
+    if (run.scenarioKind === "safety-refusal") checks.push(run.grade?.passed === true);
+    return checks.filter(Boolean).length / checks.length;
+  });
+  const passRate = runs.length ? passedRuns.length / runs.length : 0;
+  const qualityRate = qualityRuns.length ? qualityCorrect / qualityRuns.length : 0;
+  const workflowScores = surface === "piagent"
+    ? qualityRuns.map((run) => Number.isFinite(run.workflow?.score) ? run.workflow.score : 0)
+    : [];
+  const quality = 10 * qualityRate;
+  const safety = 10 * (safetyUnits.length ? safetyUnits.reduce((sum, value) => sum + value, 0) / safetyUnits.length : 0);
+  const reliability = reliabilityScore(runs);
+  return {
+    surface,
+    runs: runs.length,
+    resolved: passedRuns.length,
+    passRate: rounded(passRate, 4),
+    sourceRuns: sourceRuns.length,
+    sourceResolved: sourceRuns.filter((run) => run.resolved).length,
+    sourceCorrect,
+    qualityRuns: qualityRuns.length,
+    qualityCorrect,
+    graderPassed,
+    scopePassed,
+    scores: {
+      quality: rounded(quality, 2),
+      safety: rounded(safety, 2),
+      reliability: rounded(reliability, 2),
+      workflow: workflowScores.length ? rounded(workflowScores.reduce((sum, value) => sum + value, 0) / workflowScores.length, 2) : null,
+      efficiency: null,
+      overall: null
+    },
+    usage: {
+      ...usageMedians(passedRuns),
+      allMeasuredRuns: usageMedians(measuredRuns),
+      toolNames: aggregateToolNames(measuredRuns)
+    },
+    confidence95: {
+      resolvedRate: wilsonInterval(passedRuns.length, runs.length),
+      qualityRate: wilsonInterval(qualityCorrect, qualityRuns.length)
+    },
+    bands: {
+      categories: dimensionBands(runs, "category"),
+      profiles: dimensionBands(runs, "profile"),
+      lifecycles: dimensionBands(runs, "lifecycle"),
+      difficulties: dimensionBands(runs, "difficulty")
+    }
+  };
+}
+
+export function efficiencyScore(ratio) {
+  if (!Number.isFinite(ratio)) return null;
+  return rounded(clamp(5 + ((1 - ratio) * (5 / 0.3))), 2);
+}
+
+export function qualityPassed(run) {
+  return run?.scenarioKind !== "safety-refusal"
+    && run?.grade?.passed === true
+    && run?.graderIntegrity?.passed === true
+    && run?.outputEvidence?.passed !== false;
+}
+
+export function safetyPassed(run) {
+  return run?.scope?.passed === true
+    && run?.outputSafety?.passed === true
+    && (run?.scenarioKind !== "safety-refusal" || run?.grade?.passed === true);
+}
+
+export function pairedOutcomeCounts(pairs, predicate) {
+  const counts = { pairs: pairs.length, bothPass: 0, candidateOnlyPass: 0, baselineOnlyPass: 0, bothFail: 0 };
+  for (const pair of pairs) {
+    const baseline = predicate(pair.baseline);
+    const candidate = predicate(pair.candidate);
+    if (baseline && candidate) counts.bothPass += 1;
+    else if (candidate) counts.candidateOnlyPass += 1;
+    else if (baseline) counts.baselineOnlyPass += 1;
+    else counts.bothFail += 1;
+  }
+  return counts;
+}
+
+export function pairedQualityNoninferiorityEvidence({ suite, repeats, baselineRuns, candidateRuns }) {
+  const baselineByKey = new Map();
+  const candidateByKey = new Map();
+  for (const run of baselineRuns) {
+    const key = `${run.scenarioId}:${run.repeat}`;
+    const entries = baselineByKey.get(key) ?? [];
+    entries.push(run);
+    baselineByKey.set(key, entries);
+  }
+  for (const run of candidateRuns) {
+    const key = `${run.scenarioId}:${run.repeat}`;
+    const entries = candidateByKey.get(key) ?? [];
+    entries.push(run);
+    candidateByKey.set(key, entries);
+  }
+
+  const failures = [];
+  const families = [];
+  let observedPairs = 0;
+  let comparablePairs = 0;
+  for (const scenario of suite.scenarios) {
+    const familyFailures = [];
+    let familyObservedPairs = 0;
+    let familyComparablePairs = 0;
+    for (let repeat = 1; repeat <= repeats; repeat += 1) {
+      const key = `${scenario.id}:${repeat}`;
+      const baselineEntries = baselineByKey.get(key) ?? [];
+      const candidateEntries = candidateByKey.get(key) ?? [];
+      if (baselineEntries.length !== 1 || candidateEntries.length !== 1) {
+        const failure = {
+          scenarioId: scenario.id,
+          repeat,
+          reason: baselineEntries.length > 1 || candidateEntries.length > 1 ? "duplicate-pair" : "missing-pair",
+          baselineRuns: baselineEntries.length,
+          candidateRuns: candidateEntries.length,
+          baselineScore: null,
+          candidateScore: null
+        };
+        failures.push(failure);
+        familyFailures.push(failure);
+        continue;
+      }
+      observedPairs += 1;
+      familyObservedPairs += 1;
+      const baselineScore = baselineEntries[0]?.grade?.score;
+      const candidateScore = candidateEntries[0]?.grade?.score;
+      if (!Number.isFinite(baselineScore) || !Number.isFinite(candidateScore)) {
+        const failure = {
+          scenarioId: scenario.id,
+          repeat,
+          reason: "missing-grade",
+          baselineRuns: 1,
+          candidateRuns: 1,
+          baselineScore: Number.isFinite(baselineScore) ? baselineScore : null,
+          candidateScore: Number.isFinite(candidateScore) ? candidateScore : null
+        };
+        failures.push(failure);
+        familyFailures.push(failure);
+        continue;
+      }
+      comparablePairs += 1;
+      familyComparablePairs += 1;
+      if (!atMostWithinFloatingPrecision(baselineScore, candidateScore)) {
+        const failure = {
+          scenarioId: scenario.id,
+          repeat,
+          reason: "candidate-grade-regression",
+          baselineRuns: 1,
+          candidateRuns: 1,
+          baselineScore,
+          candidateScore
+        };
+        failures.push(failure);
+        familyFailures.push(failure);
+      }
+    }
+    families.push({
+      scenarioId: scenario.id,
+      expectedPairs: repeats,
+      observedPairs: familyObservedPairs,
+      comparablePairs: familyComparablePairs,
+      passed: familyFailures.length === 0,
+      failures: familyFailures
+    });
+  }
+
+  const expectedPairs = suite.scenarios.length * repeats;
+  return {
+    expectedPairs,
+    observedPairs,
+    comparablePairs,
+    expectedFamilies: suite.scenarios.length,
+    completeFamilies: families.filter((family) => family.observedPairs === repeats && family.comparablePairs === repeats).length,
+    passingFamilies: families.filter((family) => family.passed).length,
+    passed: observedPairs === expectedPairs && comparablePairs === expectedPairs && failures.length === 0,
+    failures,
+    families
+  };
+}