@tangle-network/agent-eval 0.89.0 → 0.90.1

package/CHANGELOG.md

@@ -4,7 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
-## [0.86.0] — 2026-06-09 — fleet-rebuilt eval primitives
+## [0.90.0] — 2026-06-10 — infra perf-benchmark substrate (`/perf`)
+
+Domain-agnostic infra-performance benchmarking: a journeys × axes scenario matrix, record-integrity contracts over flat metric records, and a percentile ratchet. Complements the judge-panel `BenchmarkRunner` (root) — that one scores QUALITY via judges; `/perf` scores LATENCY / RELIABILITY. All additive — no existing export changed.
+
+### Added
+
+- **`JourneySpec` + `expandMatrix` + `scenarioKey` (`/perf` + root).** A journey is one measurable user path (`provision.cold`, `chat.ttft`) carrying its own data contract: `requiredFields` (must be non-null on a passing record), `minimums` (numeric floors, e.g. `event_count ≥ 1` for streaming), `phaseFields` (per-phase breakdown, reported separately), and `requiresLLM` (nightly vs per-PR scheduling). `expandMatrix` does the cartesian expansion over free-form `ScenarioAxes` (driver × region × …) with a `filter` for invalid combos; scenario keys are `journeyId|dim=value|…` with dims sorted, so the key is stable across axes-object insertion order.
+- **`checkRecordIntegrity` + `assertRecordIntegrity` (`/perf` + root).** A record claiming `pass === true` must actually carry its journey's required measurements — a "passing" run with a null `total_ms` is an integrity violation (`null-required-field` / `below-minimum`), not a pass. Failed records are exempt (an errored run legitimately has nulls); `resolveJourney` returning null skips the record. The assert variant throws listing every violation.
+- **`summarizeRecords` + `gatePerf` (`/perf` + root).** Percentile ratchet: fold flat records into per-scenario `PerfStat` (`p50` / `p90` / `n`, nearest-rank on sorted values), then gate a current `PerfBaseline` against a committed one. Null / non-numeric metric values are excluded from `n` and a zero-sample field is omitted — no fake zeros. Regressions trip when p50 OR p90 exceed `tolerancePct` (default 10) over baseline; strict improvements are reported with negative `overBy`; scenarios under `minSamples` (default 3) in current are surfaced in `missingScenarios` and never gated; baseline/current key drift lands in `missingScenarios` / `newScenarios`.
 
 One clean, canonical version of five generic patterns the fleet kept hand-rolling across 2–4 product agents each. All additive — no existing export changed.
 

package/dist/chunk-STGVSCDH.js

@@ -0,0 +1,202 @@
+// src/perf/integrity.ts
+function isMissing(value) {
+  return value === null || value === void 0;
+}
+function checkRecordIntegrity(records, resolveJourney) {
+  const violations = [];
+  for (const [recordIndex, record] of records.entries()) {
+    if (record.pass !== true) continue;
+    const journey = resolveJourney(record);
+    if (journey === null) continue;
+    for (const field of journey.requiredFields) {
+      if (isMissing(record[field])) {
+        violations.push({
+          recordIndex,
+          journeyId: journey.id,
+          field,
+          reason: "null-required-field",
+          detail: `required field '${field}' is ${record[field] === null ? "null" : "undefined"} on a passing '${journey.id}' record`
+        });
+      }
+    }
+    for (const field of journey.phaseFields ?? []) {
+      if (isMissing(record[field])) {
+        violations.push({
+          recordIndex,
+          journeyId: journey.id,
+          field,
+          reason: "null-required-field",
+          detail: `phase field '${field}' is ${record[field] === null ? "null" : "undefined"} on a passing '${journey.id}' record`
+        });
+      }
+    }
+    for (const { field, min } of journey.minimums ?? []) {
+      const value = record[field];
+      if (isMissing(value)) continue;
+      if (typeof value !== "number" || Number.isNaN(value)) {
+        violations.push({
+          recordIndex,
+          journeyId: journey.id,
+          field,
+          reason: "below-minimum",
+          detail: `field '${field}' has non-numeric value ${JSON.stringify(value)} on a passing '${journey.id}' record (minimum ${min})`
+        });
+        continue;
+      }
+      if (value < min) {
+        violations.push({
+          recordIndex,
+          journeyId: journey.id,
+          field,
+          reason: "below-minimum",
+          detail: `field '${field}' is ${value}, below minimum ${min} on a passing '${journey.id}' record`
+        });
+      }
+    }
+  }
+  return { succeeded: violations.length === 0, violations };
+}
+function assertRecordIntegrity(records, resolveJourney) {
+  const result = checkRecordIntegrity(records, resolveJourney);
+  if (result.succeeded) return;
+  const lines = result.violations.map(
+    (v) => `  [record ${v.recordIndex}] ${v.journeyId}.${v.field} (${v.reason}): ${v.detail}`
+  );
+  throw new Error(
+    `Record integrity check failed with ${result.violations.length} violation(s):
+${lines.join("\n")}`
+  );
+}
+
+// src/perf/journey.ts
+function scenarioKey(journeyId, axes) {
+  const parts = Object.keys(axes).sort().map((dim) => `${dim}=${axes[dim]}`);
+  return [journeyId, ...parts].join("|");
+}
+function expandMatrix(journeys, axes, filter) {
+  const dims = Object.keys(axes).sort();
+  let combos = [{}];
+  for (const dim of dims) {
+    const values = axes[dim];
+    const next = [];
+    for (const combo of combos) {
+      for (const value of values) {
+        next.push({ ...combo, [dim]: value });
+      }
+    }
+    combos = next;
+  }
+  const scenarios = [];
+  for (const journey of journeys) {
+    for (const combo of combos) {
+      if (filter && !filter(journey.id, combo)) continue;
+      scenarios.push({ key: scenarioKey(journey.id, combo), journey, axes: combo });
+    }
+  }
+  return scenarios;
+}
+
+// src/perf/ratchet.ts
+function nearestRank(sorted, p) {
+  if (sorted.length === 0) throw new Error("nearestRank requires at least one sample");
+  const rank = Math.max(1, Math.ceil(p / 100 * sorted.length));
+  return sorted[rank - 1];
+}
+function summarizeRecords(records, keyOf, metricFields) {
+  const samples = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    const key = keyOf(record);
+    if (key === null) continue;
+    let byField = samples.get(key);
+    if (!byField) {
+      byField = /* @__PURE__ */ new Map();
+      samples.set(key, byField);
+    }
+    for (const field of metricFields) {
+      const value = record[field];
+      if (typeof value !== "number" || Number.isNaN(value)) continue;
+      let values = byField.get(field);
+      if (!values) {
+        values = [];
+        byField.set(field, values);
+      }
+      values.push(value);
+    }
+  }
+  const scenarios = {};
+  for (const [key, byField] of samples) {
+    const stats = {};
+    for (const [field, values] of byField) {
+      if (values.length === 0) continue;
+      const sorted = [...values].sort((a, b) => a - b);
+      stats[field] = {
+        p50: nearestRank(sorted, 50),
+        p90: nearestRank(sorted, 90),
+        n: sorted.length
+      };
+    }
+    scenarios[key] = stats;
+  }
+  return { version: 1, scenarios };
+}
+function pctOver(currentValue, baselineValue) {
+  if (baselineValue === 0) {
+    return currentValue === 0 ? 0 : Number.POSITIVE_INFINITY;
+  }
+  return (currentValue - baselineValue) / baselineValue * 100;
+}
+function gatePerf(current, baseline, options) {
+  const tolerancePct = options?.tolerancePct ?? 10;
+  const minSamples = options?.minSamples ?? 3;
+  const regressions = [];
+  const improvements = [];
+  const missing = /* @__PURE__ */ new Set();
+  for (const [scenarioKey2, baselineStats] of Object.entries(baseline.scenarios)) {
+    const currentStats = current.scenarios[scenarioKey2];
+    if (!currentStats) {
+      missing.add(scenarioKey2);
+      continue;
+    }
+    for (const [field, baselineStat] of Object.entries(baselineStats)) {
+      const currentStat = currentStats[field];
+      if (!currentStat || currentStat.n < minSamples) {
+        missing.add(scenarioKey2);
+        continue;
+      }
+      const overBy = {
+        p50Pct: pctOver(currentStat.p50, baselineStat.p50),
+        p90Pct: pctOver(currentStat.p90, baselineStat.p90)
+      };
+      const entry = {
+        scenarioKey: scenarioKey2,
+        field,
+        baseline: baselineStat,
+        current: currentStat,
+        overBy
+      };
+      if (overBy.p50Pct > tolerancePct || overBy.p90Pct > tolerancePct) {
+        regressions.push(entry);
+      } else if (overBy.p50Pct < 0 && overBy.p90Pct < 0) {
+        improvements.push(entry);
+      }
+    }
+  }
+  const newScenarios = Object.keys(current.scenarios).filter((key) => !(key in baseline.scenarios));
+  return {
+    succeeded: regressions.length === 0,
+    regressions,
+    improvements,
+    missingScenarios: [...missing],
+    newScenarios
+  };
+}
+
+export {
+  checkRecordIntegrity,
+  assertRecordIntegrity,
+  scenarioKey,
+  expandMatrix,
+  summarizeRecords,
+  gatePerf
+};
+//# sourceMappingURL=chunk-STGVSCDH.js.map

package/dist/chunk-STGVSCDH.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/perf/integrity.ts","../src/perf/journey.ts","../src/perf/ratchet.ts"],"sourcesContent":["/**\n * Record-integrity contracts for perf metric records.\n *\n * A record that claims `pass === true` must actually carry the journey's\n * required measurements — a \"passing\" provision run with a null\n * `total_ms` is a lying record, not a pass. Failed records are exempt:\n * a run that errored mid-flight legitimately has nulls.\n */\n\nimport type { JourneySpec } from './journey'\n\nexport interface IntegrityViolation {\n  recordIndex: number\n  journeyId: string\n  field: string\n  reason: 'null-required-field' | 'below-minimum'\n  detail: string\n}\n\nexport interface IntegrityResult {\n  succeeded: boolean\n  violations: IntegrityViolation[]\n}\n\nfunction isMissing(value: unknown): boolean {\n  return value === null || value === undefined\n}\n\n/**\n * Validates flat metric records (Record<string, unknown> with a boolean\n * `pass` field) against their journey contract. Only records with\n * pass === true are checked — a failed record may legitimately have nulls.\n * resolveJourney maps a record to its JourneySpec (or null to skip).\n */\nexport function checkRecordIntegrity(\n  records: ReadonlyArray<Record<string, unknown>>,\n  resolveJourney: (record: Record<string, unknown>) => JourneySpec | null,\n): IntegrityResult {\n  const violations: IntegrityViolation[] = []\n  for (const [recordIndex, record] of records.entries()) {\n    if (record.pass !== true) continue\n    const journey = resolveJourney(record)\n    if (journey === null) continue\n    for (const field of journey.requiredFields) {\n      if (isMissing(record[field])) {\n        violations.push({\n          recordIndex,\n          journeyId: journey.id,\n          field,\n          reason: 'null-required-field',\n          detail: `required field '${field}' is ${record[field] === null ? 'null' : 'undefined'} on a passing '${journey.id}' record`,\n        })\n      }\n    }\n    for (const field of journey.phaseFields ?? []) {\n      if (isMissing(record[field])) {\n        violations.push({\n          recordIndex,\n          journeyId: journey.id,\n          field,\n          reason: 'null-required-field',\n          detail: `phase field '${field}' is ${record[field] === null ? 'null' : 'undefined'} on a passing '${journey.id}' record`,\n        })\n      }\n    }\n    for (const { field, min } of journey.minimums ?? []) {\n      const value = record[field]\n      if (isMissing(value)) continue // null-ness is the required/phase fields' contract\n      if (typeof value !== 'number' || Number.isNaN(value)) {\n        violations.push({\n          recordIndex,\n          journeyId: journey.id,\n          field,\n          reason: 'below-minimum',\n          detail: `field '${field}' has non-numeric value ${JSON.stringify(value)} on a passing '${journey.id}' record (minimum ${min})`,\n        })\n        continue\n      }\n      if (value < min) {\n        violations.push({\n          recordIndex,\n          journeyId: journey.id,\n          field,\n          reason: 'below-minimum',\n          detail: `field '${field}' is ${value}, below minimum ${min} on a passing '${journey.id}' record`,\n        })\n      }\n    }\n  }\n  return { succeeded: violations.length === 0, violations }\n}\n\n/** Throws an Error listing every violation when the result fails. */\nexport function assertRecordIntegrity(\n  records: ReadonlyArray<Record<string, unknown>>,\n  resolveJourney: (record: Record<string, unknown>) => JourneySpec | null,\n): void {\n  const result = checkRecordIntegrity(records, resolveJourney)\n  if (result.succeeded) return\n  const lines = result.violations.map(\n    (v) => `  [record ${v.recordIndex}] ${v.journeyId}.${v.field} (${v.reason}): ${v.detail}`,\n  )\n  throw new Error(\n    `Record integrity check failed with ${result.violations.length} violation(s):\\n${lines.join('\\n')}`,\n  )\n}\n","/**\n * Journey × axes matrix for infra performance benchmarks.\n *\n * A journey is one measurable user path (\"provision.cold\", \"chat.ttft\");\n * axes are free-form scenario dimensions (driver, region, image…). The\n * matrix expansion is pure bookkeeping — running the scenarios and\n * recording metrics is the caller's job. This module complements the\n * judge-panel `BenchmarkRunner` (src/benchmark.ts): that one scores\n * QUALITY via judges, this one structures LATENCY / RELIABILITY runs\n * over flat metric records.\n */\n\n/** One measurable user journey (e.g. \"provision.cold\", \"chat.ttft\"). */\nexport interface JourneySpec {\n  id: string\n  description: string\n  /** Needs a real LLM call — schedule nightly, not per-PR. */\n  requiresLLM: boolean\n  /**\n   * Fields that MUST be non-null on a passing record of this journey.\n   * A \"passing\" record missing one is an integrity violation, not a pass.\n   */\n  requiredFields: ReadonlyArray<string>\n  /** Numeric floors, e.g. {field: 'event_count', min: 1} for streaming. */\n  minimums?: ReadonlyArray<{ field: string; min: number }>\n  /** Per-phase breakdown fields expected non-null (subset of requiredFields semantics, reported separately). */\n  phaseFields?: ReadonlyArray<string>\n}\n\nexport interface ScenarioAxes {\n  /** e.g. driver: ['docker','firecracker'] — every key is a free-form dimension. */\n  [dimension: string]: ReadonlyArray<string>\n}\n\nexport interface PerfScenario {\n  /** `${journeyId}|${dim1}=${v1}|${dim2}=${v2}` (dims sorted). */\n  key: string\n  journey: JourneySpec\n  axes: Record<string, string>\n}\n\n/** Stable scenario key: journey id then `dim=value` pairs in sorted-dim order. */\nexport function scenarioKey(journeyId: string, axes: Record<string, string>): string {\n  const parts = Object.keys(axes)\n    .sort()\n    .map((dim) => `${dim}=${axes[dim]}`)\n  return [journeyId, ...parts].join('|')\n}\n\n/** Cartesian expansion; `filter` lets callers drop invalid combos (e.g. firecracker×resume). */\nexport function expandMatrix(\n  journeys: ReadonlyArray<JourneySpec>,\n  axes: ScenarioAxes,\n  filter?: (journeyId: string, combo: Record<string, string>) => boolean,\n): PerfScenario[] {\n  const dims = Object.keys(axes).sort()\n  let combos: Record<string, string>[] = [{}]\n  for (const dim of dims) {\n    const values = axes[dim] as ReadonlyArray<string>\n    const next: Record<string, string>[] = []\n    for (const combo of combos) {\n      for (const value of values) {\n        next.push({ ...combo, [dim]: value })\n      }\n    }\n    combos = next\n  }\n  const scenarios: PerfScenario[] = []\n  for (const journey of journeys) {\n    for (const combo of combos) {\n      if (filter && !filter(journey.id, combo)) continue\n      scenarios.push({ key: scenarioKey(journey.id, combo), journey, axes: combo })\n    }\n  }\n  return scenarios\n}\n","/**\n * Percentile ratchet over perf metric records.\n *\n * `summarizeRecords` folds flat records into per-scenario p50/p90 stats;\n * `gatePerf` compares a current summary against a committed baseline and\n * trips on regressions beyond tolerance. Percentiles use nearest-rank on\n * sorted values. Null / non-numeric metric values are excluded from the\n * stat (n reflects only real samples); a field with zero samples is\n * omitted entirely — no fake zeros.\n */\n\nexport interface PerfStat {\n  p50: number\n  p90: number\n  n: number\n}\n\nexport interface PerfBaseline {\n  version: 1\n  /** key → metric field → stat. */\n  scenarios: Record<string, Record<string, PerfStat>>\n}\n\nexport interface PerfRegression {\n  scenarioKey: string\n  field: string\n  baseline: PerfStat\n  current: PerfStat\n  /** percent over baseline p50 / p90, whichever tripped. */\n  overBy: { p50Pct: number; p90Pct: number }\n}\n\nexport interface PerfGateResult {\n  succeeded: boolean\n  regressions: PerfRegression[]\n  /** Negative overBy: strictly better than baseline on both percentiles. */\n  improvements: PerfRegression[]\n  /** In baseline but absent (or under-sampled, n < minSamples) in current. */\n  missingScenarios: string[]\n  /** In records but absent from baseline. */\n  newScenarios: string[]\n}\n\n/** Nearest-rank percentile over a non-empty ascending-sorted array (p in (0, 100]). */\nfunction nearestRank(sorted: ReadonlyArray<number>, p: number): number {\n  if (sorted.length === 0) throw new Error('nearestRank requires at least one sample')\n  const rank = Math.max(1, Math.ceil((p / 100) * sorted.length))\n  return sorted[rank - 1] as number\n}\n\nexport function summarizeRecords(\n  records: ReadonlyArray<Record<string, unknown>>,\n  keyOf: (record: Record<string, unknown>) => string | null,\n  metricFields: ReadonlyArray<string>,\n): PerfBaseline {\n  const samples = new Map<string, Map<string, number[]>>()\n  for (const record of records) {\n    const key = keyOf(record)\n    if (key === null) continue\n    let byField = samples.get(key)\n    if (!byField) {\n      byField = new Map()\n      samples.set(key, byField)\n    }\n    for (const field of metricFields) {\n      const value = record[field]\n      if (typeof value !== 'number' || Number.isNaN(value)) continue\n      let values = byField.get(field)\n      if (!values) {\n        values = []\n        byField.set(field, values)\n      }\n      values.push(value)\n    }\n  }\n  const scenarios: Record<string, Record<string, PerfStat>> = {}\n  for (const [key, byField] of samples) {\n    const stats: Record<string, PerfStat> = {}\n    for (const [field, values] of byField) {\n      if (values.length === 0) continue\n      const sorted = [...values].sort((a, b) => a - b)\n      stats[field] = {\n        p50: nearestRank(sorted, 50),\n        p90: nearestRank(sorted, 90),\n        n: sorted.length,\n      }\n    }\n    scenarios[key] = stats\n  }\n  return { version: 1, scenarios }\n}\n\n/** Percent over baseline; baseline 0 → 0% when equal, Infinity when current grew. */\nfunction pctOver(currentValue: number, baselineValue: number): number {\n  if (baselineValue === 0) {\n    return currentValue === 0 ? 0 : Number.POSITIVE_INFINITY\n  }\n  return ((currentValue - baselineValue) / baselineValue) * 100\n}\n\nexport function gatePerf(\n  current: PerfBaseline,\n  baseline: PerfBaseline,\n  options?: { tolerancePct?: number; minSamples?: number },\n): PerfGateResult {\n  const tolerancePct = options?.tolerancePct ?? 10\n  const minSamples = options?.minSamples ?? 3\n  const regressions: PerfRegression[] = []\n  const improvements: PerfRegression[] = []\n  const missing = new Set<string>()\n\n  for (const [scenarioKey, baselineStats] of Object.entries(baseline.scenarios)) {\n    const currentStats = current.scenarios[scenarioKey]\n    if (!currentStats) {\n      missing.add(scenarioKey)\n      continue\n    }\n    for (const [field, baselineStat] of Object.entries(baselineStats)) {\n      const currentStat = currentStats[field]\n      if (!currentStat || currentStat.n < minSamples) {\n        // Under-sampled current data cannot gate — surface it instead of\n        // pretending the scenario passed.\n        missing.add(scenarioKey)\n        continue\n      }\n      const overBy = {\n        p50Pct: pctOver(currentStat.p50, baselineStat.p50),\n        p90Pct: pctOver(currentStat.p90, baselineStat.p90),\n      }\n      const entry: PerfRegression = {\n        scenarioKey,\n        field,\n        baseline: baselineStat,\n        current: currentStat,\n        overBy,\n      }\n      if (overBy.p50Pct > tolerancePct || overBy.p90Pct > tolerancePct) {\n        regressions.push(entry)\n      } else if (overBy.p50Pct < 0 && overBy.p90Pct < 0) {\n        improvements.push(entry)\n      }\n    }\n  }\n\n  const newScenarios = Object.keys(current.scenarios).filter((key) => !(key in baseline.scenarios))\n\n  return {\n    succeeded: regressions.length === 0,\n    regressions,\n    improvements,\n    missingScenarios: [...missing],\n    newScenarios,\n  }\n}\n"],"mappings":";AAwBA,SAAS,UAAU,OAAyB;AAC1C,SAAO,UAAU,QAAQ,UAAU;AACrC;AAQO,SAAS,qBACd,SACA,gBACiB;AACjB,QAAM,aAAmC,CAAC;AAC1C,aAAW,CAAC,aAAa,MAAM,KAAK,QAAQ,QAAQ,GAAG;AACrD,QAAI,OAAO,SAAS,KAAM;AAC1B,UAAM,UAAU,eAAe,MAAM;AACrC,QAAI,YAAY,KAAM;AACtB,eAAW,SAAS,QAAQ,gBAAgB;AAC1C,UAAI,UAAU,OAAO,KAAK,CAAC,GAAG;AAC5B,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ,mBAAmB,KAAK,QAAQ,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,kBAAkB,QAAQ,EAAE;AAAA,QACnH,CAAC;AAAA,MACH;AAAA,IACF;AACA,eAAW,SAAS,QAAQ,eAAe,CAAC,GAAG;AAC7C,UAAI,UAAU,OAAO,KAAK,CAAC,GAAG;AAC5B,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ,gBAAgB,KAAK,QAAQ,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,kBAAkB,QAAQ,EAAE;AAAA,QAChH,CAAC;AAAA,MACH;AAAA,IACF;AACA,eAAW,EAAE,OAAO,IAAI,KAAK,QAAQ,YAAY,CAAC,GAAG;AACnD,YAAM,QAAQ,OAAO,KAAK;AAC1B,UAAI,UAAU,KAAK,EAAG;AACtB,UAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,GAAG;AACpD,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ,UAAU,KAAK,2BAA2B,KAAK,UAAU,KAAK,CAAC,kBAAkB,QAAQ,EAAE,qBAAqB,GAAG;AAAA,QAC7H,CAAC;AACD;AAAA,MACF;AACA,UAAI,QAAQ,KAAK;AACf,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ,UAAU,KAAK,QAAQ,KAAK,mBAAmB,GAAG,kBAAkB,QAAQ,EAAE;AAAA,QACxF,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO,EAAE,WAAW,WAAW,WAAW,GAAG,WAAW;AAC1D;AAGO,SAAS,sBACd,SACA,gBACM;AACN,QAAM,SAAS,qBAAqB,SAAS,cAAc;AAC3D,MAAI,OAAO,UAAW;AACtB,QAAM,QAAQ,OAAO,WAAW;AAAA,IAC9B,CAAC,MAAM,aAAa,EAAE,WAAW,KAAK,EAAE,SAAS,IAAI,EAAE,KAAK,KAAK,EAAE,MAAM,MAAM,EAAE,MAAM;AAAA,EACzF;AACA,QAAM,IAAI;AAAA,IACR,sCAAsC,OAAO,WAAW,MAAM;AAAA,EAAmB,MAAM,KAAK,IAAI,CAAC;AAAA,EACnG;AACF;;;AC/DO,SAAS,YAAY,WAAmB,MAAsC;AACnF,QAAM,QAAQ,OAAO,KAAK,IAAI,EAC3B,KAAK,EACL,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,KAAK,GAAG,CAAC,EAAE;AACrC,SAAO,CAAC,WAAW,GAAG,KAAK,EAAE,KAAK,GAAG;AACvC;AAGO,SAAS,aACd,UACA,MACA,QACgB;AAChB,QAAM,OAAO,OAAO,KAAK,IAAI,EAAE,KAAK;AACpC,MAAI,SAAmC,CAAC,CAAC,CAAC;AAC1C,aAAW,OAAO,MAAM;AACtB,UAAM,SAAS,KAAK,GAAG;AACvB,UAAM,OAAiC,CAAC;AACxC,eAAW,SAAS,QAAQ;AAC1B,iBAAW,SAAS,QAAQ;AAC1B,aAAK,KAAK,EAAE,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AACA,aAAS;AAAA,EACX;AACA,QAAM,YAA4B,CAAC;AACnC,aAAW,WAAW,UAAU;AAC9B,eAAW,SAAS,QAAQ;AAC1B,UAAI,UAAU,CAAC,OAAO,QAAQ,IAAI,KAAK,EAAG;AAC1C,gBAAU,KAAK,EAAE,KAAK,YAAY,QAAQ,IAAI,KAAK,GAAG,SAAS,MAAM,MAAM,CAAC;AAAA,IAC9E;AAAA,EACF;AACA,SAAO;AACT;;;AC/BA,SAAS,YAAY,QAA+B,GAAmB;AACrE,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,0CAA0C;AACnF,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,KAAM,IAAI,MAAO,OAAO,MAAM,CAAC;AAC7D,SAAO,OAAO,OAAO,CAAC;AACxB;AAEO,SAAS,iBACd,SACA,OACA,cACc;AACd,QAAM,UAAU,oBAAI,IAAmC;AACvD,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,MAAM,MAAM;AACxB,QAAI,QAAQ,KAAM;AAClB,QAAI,UAAU,QAAQ,IAAI,GAAG;AAC7B,QAAI,CAAC,SAAS;AACZ,gBAAU,oBAAI,IAAI;AAClB,cAAQ,IAAI,KAAK,OAAO;AAAA,IAC1B;AACA,eAAW,SAAS,cAAc;AAChC,YAAM,QAAQ,OAAO,KAAK;AAC1B,UAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,EAAG;AACtD,UAAI,SAAS,QAAQ,IAAI,KAAK;AAC9B,UAAI,CAAC,QAAQ;AACX,iBAAS,CAAC;AACV,gBAAQ,IAAI,OAAO,MAAM;AAAA,MAC3B;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,QAAM,YAAsD,CAAC;AAC7D,aAAW,CAAC,KAAK,OAAO,KAAK,SAAS;AACpC,UAAM,QAAkC,CAAC;AACzC,eAAW,CAAC,OAAO,MAAM,KAAK,SAAS;AACrC,UAAI,OAAO,WAAW,EAAG;AACzB,YAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,YAAM,KAAK,IAAI;AAAA,QACb,KAAK,YAAY,QAAQ,EAAE;AAAA,QAC3B,KAAK,YAAY,QAAQ,EAAE;AAAA,QAC3B,GAAG,OAAO;AAAA,MACZ;AAAA,IACF;AACA,cAAU,GAAG,IAAI;AAAA,EACnB;AACA,SAAO,EAAE,SAAS,GAAG,UAAU;AACjC;AAGA,SAAS,QAAQ,cAAsB,eAA+B;AACpE,MAAI,kBAAkB,GAAG;AACvB,WAAO,iBAAiB,IAAI,IAAI,OAAO;AAAA,EACzC;AACA,UAAS,eAAe,iBAAiB,gBAAiB;AAC5D;AAEO,SAAS,SACd,SACA,UACA,SACgB;AAChB,QAAM,eAAe,SAAS,gBAAgB;AAC9C,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAiC,CAAC;AACxC,QAAM,UAAU,oBAAI,IAAY;AAEhC,aAAW,CAACA,cAAa,aAAa,KAAK,OAAO,QAAQ,SAAS,SAAS,GAAG;AAC7E,UAAM,eAAe,QAAQ,UAAUA,YAAW;AAClD,QAAI,CAAC,cAAc;AACjB,cAAQ,IAAIA,YAAW;AACvB;AAAA,IACF;AACA,eAAW,CAAC,OAAO,YAAY,KAAK,OAAO,QAAQ,aAAa,GAAG;AACjE,YAAM,cAAc,aAAa,KAAK;AACtC,UAAI,CAAC,eAAe,YAAY,IAAI,YAAY;AAG9C,gBAAQ,IAAIA,YAAW;AACvB;AAAA,MACF;AACA,YAAM,SAAS;AAAA,QACb,QAAQ,QAAQ,YAAY,KAAK,aAAa,GAAG;AAAA,QACjD,QAAQ,QAAQ,YAAY,KAAK,aAAa,GAAG;AAAA,MACnD;AACA,YAAM,QAAwB;AAAA,QAC5B,aAAAA;AAAA,QACA;AAAA,QACA,UAAU;AAAA,QACV,SAAS;AAAA,QACT;AAAA,MACF;AACA,UAAI,OAAO,SAAS,gBAAgB,OAAO,SAAS,cAAc;AAChE,oBAAY,KAAK,KAAK;AAAA,MACxB,WAAW,OAAO,SAAS,KAAK,OAAO,SAAS,GAAG;AACjD,qBAAa,KAAK,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,KAAK,QAAQ,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,SAAS,UAAU;AAEhG,SAAO;AAAA,IACL,WAAW,YAAY,WAAW;AAAA,IAClC;AAAA,IACA;AAAA,IACA,kBAAkB,CAAC,GAAG,OAAO;AAAA,IAC7B;AAAA,EACF;AACF;","names":["scenarioKey"]}

package/dist/contract/index.js

@@ -1,3 +1,12 @@
+import {
+  fromClaudeCodeSession,
+  fromCodexSession,
+  fromKimiCodeSession,
+  fromOpenCodeSession,
+  fromPiSession,
+  fromPigraphSession,
+  parseCodeAgentJsonl
+} from "../chunk-S42AWHMP.js";
 import {
   createHostedClient
 } from "../chunk-ZZUXHH3R.js";
@@ -10,15 +19,6 @@ import {
   paretoSignificanceGate,
   runEval
 } from "../chunk-6SOJM3VR.js";
-import {
-  fromClaudeCodeSession,
-  fromCodexSession,
-  fromKimiCodeSession,
-  fromOpenCodeSession,
-  fromPiSession,
-  fromPigraphSession,
-  parseCodeAgentJsonl
-} from "../chunk-S42AWHMP.js";
 import {
   analyzeRuns
 } from "../chunk-Y47J2LJ3.js";

package/dist/fuzz.d.ts

@@ -167,6 +167,17 @@ interface CapsuleData<S> {
         /** Runs whose cost was unknown (`costOf` returned null) — counted apart,
          *  never folded into `costUsd` as a fabricated $0. */
         costUnknownRuns?: number;
+        /** Evaluations that threw (transport/backend failures). They consumed no
+         *  run budget and scored nothing — an infra axis, never folded into
+         *  robustness or reported as findings. */
+        evalErrors: number;
+        /** Present when the run stopped before its budget because consecutive
+         *  eval errors tripped the circuit breaker (a dead backend must not burn
+         *  the remaining budget). The capsule-so-far is complete and honest. */
+        stoppedEarly?: {
+            reason: 'eval-errors';
+            detail: string;
+        };
     };
 }
 /**
@@ -190,6 +201,11 @@ type ExploreEvent<S> = {
 } | {
     type: 'finding';
     finding: Finding<S>;
+} | {
+    type: 'eval-error';
+    cell: Cell;
+    scenarioId: string;
+    message: string;
 } | {
     type: 'round';
     runsUsed: number;
@@ -226,6 +242,9 @@ interface ExploreOptions<S> {
     minimize?: (scenario: S, evaluate: Evaluator<S>, cell: Cell) => Promise<S> | S;
     /** Max concurrent `evaluate` calls. Default 1. */
     concurrency?: number;
+    /** Stop the run after this many CONSECUTIVE eval errors (a dead backend must
+     *  not burn the remaining budget). Successes reset the streak. Default 5. */
+    maxConsecutiveEvalErrors?: number;
     /** Cooperative cancellation. */
     signal?: AbortSignal;
     /** Progress stream. */
@@ -315,6 +334,13 @@ interface BuildCapsuleInput<S> {
         costUsd: number;
         costUnknownRuns: number;
     };
+    /** Evaluations that threw — infra outcomes, never folded into robustness. */
+    evalErrors: number;
+    /** Set when the consecutive-error circuit breaker stopped the run early. */
+    stoppedEarly?: {
+        reason: 'eval-errors';
+        detail: string;
+    };
 }
 declare function buildCapsule<S>(input: BuildCapsuleInput<S>): CapsuleData<S>;
 interface RenderCapsuleOptions {
@@ -356,6 +382,9 @@ declare class BehaviorExplorer<S> {
     private readonly _findings;
     private runsUsed;
     private candidateFindings;
+    private evalErrors;
+    private consecutiveEvalErrors;
+    private stoppedEarly;
     private rngState;
     /** Accumulated KNOWN dollars — unknown-cost runs never inflate it. */
     private spentKnownUsd;

package/dist/fuzz.js

@@ -69,7 +69,9 @@ function buildCapsule(input) {
       candidateFindings: input.candidateFindings,
       verifiedFindings: input.findings.length,
       meanRobustness,
-      ...input.cost ? { costUsd: input.cost.costUsd, costUnknownRuns: input.cost.costUnknownRuns } : {}
+      ...input.cost ? { costUsd: input.cost.costUsd, costUnknownRuns: input.cost.costUnknownRuns } : {},
+      evalErrors: input.evalErrors,
+      ...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
     }
   };
 }
@@ -201,8 +203,10 @@ ${kpi("verified findings", String(s.verifiedFindings), s.verifiedFindings > 0 ?
 ${kpi("cells covered", `${s.cellsCovered}/${s.cellsTotal}`)}
 ${kpi("scenarios run", String(s.totalRuns))}
 ${cost}
+${s.evalErrors > 0 ? kpi("eval errors", String(s.evalErrors), "#e5b566") : ""}
 ${lift}
 </div>
+${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)} \u2014 coverage below reflects the completed portion only</div>` : ""}
 <h2>Coverage map</h2>
 ${heatmapHtml(capsule.coverage)}
 <h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` \xB7 ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ""}</h2>
@@ -317,6 +321,9 @@ var BehaviorExplorer = class {
   _findings = [];
   runsUsed = 0;
   candidateFindings = 0;
+  evalErrors = 0;
+  consecutiveEvalErrors = 0;
+  stoppedEarly;
   rngState;
   /** Accumulated KNOWN dollars — unknown-cost runs never inflate it. */
   spentKnownUsd = 0;
@@ -400,7 +407,7 @@ var BehaviorExplorer = class {
     const newFindings = [];
     let runsThisStep = 0;
     for (const alloc of allocations) {
-      if (this.runsUsed >= this.opts.budget || this.costExhausted() || this.opts.signal?.aborted)
+      if (this.runsUsed >= this.opts.budget || this.costExhausted() || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
         break;
       const cell = this.cellById.get(alloc.cellId);
       if (!cell) continue;
@@ -422,39 +429,60 @@ var BehaviorExplorer = class {
       await pMap(
         toEval,
         async (scenario) => {
-          if (this.runsUsed >= this.opts.budget || this.costExhausted() || this.opts.signal?.aborted)
+          if (this.runsUsed >= this.opts.budget || this.costExhausted() || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
             return;
-          const ev = await this.opts.evaluate(scenario, cell);
-          this.runsUsed++;
-          runsThisStep++;
-          this.recordRunCost(scenario, cell, ev);
-          const interest = this.objective.interest(ev, this.objectiveContext());
-          this.log.push({ cell, ev, interest, scenarioId: this.opts.scenarioId(scenario) });
-          this.opts.onProgress?.({ type: "evaluated", cell, scenario, evaluation: ev });
-          const bin = this.binId(cell, ev.descriptor);
-          const cur = this.archiveByBin.get(bin);
-          if (!cur || interest > cur.interest)
-            this.archiveByBin.set(bin, { binId: bin, cell, scenario, evaluation: ev, interest });
-          if (interest < this.threshold) return;
-          this.candidateFindings++;
-          if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell))
-            return;
-          if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell))
-            return;
-          const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
-          const finding = {
-            id: this.opts.scenarioId(scenario),
-            cell,
-            scenario,
-            minimized,
-            text: this.opts.scenarioText?.(minimized),
-            evaluation: ev,
-            interest,
-            objective: this.objective.kind
-          };
-          this._findings.push(finding);
-          newFindings.push(finding);
-          this.opts.onProgress?.({ type: "finding", finding });
+          try {
+            const ev = await this.opts.evaluate(scenario, cell);
+            this.runsUsed++;
+            runsThisStep++;
+            this.consecutiveEvalErrors = 0;
+            this.recordRunCost(scenario, cell, ev);
+            const interest = this.objective.interest(ev, this.objectiveContext());
+            this.log.push({ cell, ev, interest, scenarioId: this.opts.scenarioId(scenario) });
+            this.opts.onProgress?.({ type: "evaluated", cell, scenario, evaluation: ev });
+            const bin = this.binId(cell, ev.descriptor);
+            const cur = this.archiveByBin.get(bin);
+            if (!cur || interest > cur.interest)
+              this.archiveByBin.set(bin, { binId: bin, cell, scenario, evaluation: ev, interest });
+            if (interest < this.threshold) return;
+            this.candidateFindings++;
+            if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell))
+              return;
+            if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell))
+              return;
+            const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
+            const finding = {
+              id: this.opts.scenarioId(scenario),
+              cell,
+              scenario,
+              minimized,
+              text: this.opts.scenarioText?.(minimized),
+              evaluation: ev,
+              interest,
+              objective: this.objective.kind
+            };
+            this._findings.push(finding);
+            newFindings.push(finding);
+            this.opts.onProgress?.({ type: "finding", finding });
+          } catch (err) {
+            if (err instanceof RangeError) throw err;
+            this.evalErrors++;
+            this.consecutiveEvalErrors++;
+            const message = err instanceof Error ? err.message : String(err);
+            this.opts.onProgress?.({
+              type: "eval-error",
+              cell,
+              scenarioId: this.opts.scenarioId(scenario),
+              message
+            });
+            const limit = this.opts.maxConsecutiveEvalErrors ?? 5;
+            if (this.consecutiveEvalErrors >= limit) {
+              this.stoppedEarly = {
+                reason: "eval-errors",
+                detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`
+              };
+            }
+          }
         },
         this.opts.concurrency ?? 1,
         this.opts.signal
@@ -466,9 +494,9 @@ var BehaviorExplorer = class {
   /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
    *  or no progress is made. */
   async run() {
-    while (this.runsUsed < this.opts.budget && !this.costExhausted() && !this.opts.signal?.aborted) {
+    while (this.runsUsed < this.opts.budget && !this.costExhausted() && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
       const { runs } = await this.step();
-      if (runs === 0) break;
+      if (runs === 0 && this.stoppedEarly === void 0) break;
     }
     return this.capsule();
   }
@@ -489,7 +517,9 @@ var BehaviorExplorer = class {
       findings: this._findings,
       candidateFindings: this.candidateFindings,
       runsUsed: this.runsUsed,
-      cost: this.opts.costOf ? { costUsd: this.spentKnownUsd, costUnknownRuns: this.costUnknownRuns } : void 0
+      cost: this.opts.costOf ? { costUsd: this.spentKnownUsd, costUnknownRuns: this.costUnknownRuns } : void 0,
+      evalErrors: this.evalErrors,
+      stoppedEarly: this.stoppedEarly
     });
   }
 };