@tangle-network/agent-eval 0.89.0 → 0.90.0

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.89.0",
+    "version": "0.90.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/perf/index.d.ts

@@ -0,0 +1,123 @@
+/**
+ * Journey × axes matrix for infra performance benchmarks.
+ *
+ * A journey is one measurable user path ("provision.cold", "chat.ttft");
+ * axes are free-form scenario dimensions (driver, region, image…). The
+ * matrix expansion is pure bookkeeping — running the scenarios and
+ * recording metrics is the caller's job. This module complements the
+ * judge-panel `BenchmarkRunner` (src/benchmark.ts): that one scores
+ * QUALITY via judges, this one structures LATENCY / RELIABILITY runs
+ * over flat metric records.
+ */
+/** One measurable user journey (e.g. "provision.cold", "chat.ttft"). */
+interface JourneySpec {
+    id: string;
+    description: string;
+    /** Needs a real LLM call — schedule nightly, not per-PR. */
+    requiresLLM: boolean;
+    /**
+     * Fields that MUST be non-null on a passing record of this journey.
+     * A "passing" record missing one is an integrity violation, not a pass.
+     */
+    requiredFields: ReadonlyArray<string>;
+    /** Numeric floors, e.g. {field: 'event_count', min: 1} for streaming. */
+    minimums?: ReadonlyArray<{
+        field: string;
+        min: number;
+    }>;
+    /** Per-phase breakdown fields expected non-null (subset of requiredFields semantics, reported separately). */
+    phaseFields?: ReadonlyArray<string>;
+}
+interface ScenarioAxes {
+    /** e.g. driver: ['docker','firecracker'] — every key is a free-form dimension. */
+    [dimension: string]: ReadonlyArray<string>;
+}
+interface PerfScenario {
+    /** `${journeyId}|${dim1}=${v1}|${dim2}=${v2}` (dims sorted). */
+    key: string;
+    journey: JourneySpec;
+    axes: Record<string, string>;
+}
+/** Stable scenario key: journey id then `dim=value` pairs in sorted-dim order. */
+declare function scenarioKey(journeyId: string, axes: Record<string, string>): string;
+/** Cartesian expansion; `filter` lets callers drop invalid combos (e.g. firecracker×resume). */
+declare function expandMatrix(journeys: ReadonlyArray<JourneySpec>, axes: ScenarioAxes, filter?: (journeyId: string, combo: Record<string, string>) => boolean): PerfScenario[];
+
+/**
+ * Record-integrity contracts for perf metric records.
+ *
+ * A record that claims `pass === true` must actually carry the journey's
+ * required measurements — a "passing" provision run with a null
+ * `total_ms` is a lying record, not a pass. Failed records are exempt:
+ * a run that errored mid-flight legitimately has nulls.
+ */
+
+interface IntegrityViolation {
+    recordIndex: number;
+    journeyId: string;
+    field: string;
+    reason: 'null-required-field' | 'below-minimum';
+    detail: string;
+}
+interface IntegrityResult {
+    succeeded: boolean;
+    violations: IntegrityViolation[];
+}
+/**
+ * Validates flat metric records (Record<string, unknown> with a boolean
+ * `pass` field) against their journey contract. Only records with
+ * pass === true are checked — a failed record may legitimately have nulls.
+ * resolveJourney maps a record to its JourneySpec (or null to skip).
+ */
+declare function checkRecordIntegrity(records: ReadonlyArray<Record<string, unknown>>, resolveJourney: (record: Record<string, unknown>) => JourneySpec | null): IntegrityResult;
+/** Throws an Error listing every violation when the result fails. */
+declare function assertRecordIntegrity(records: ReadonlyArray<Record<string, unknown>>, resolveJourney: (record: Record<string, unknown>) => JourneySpec | null): void;
+
+/**
+ * Percentile ratchet over perf metric records.
+ *
+ * `summarizeRecords` folds flat records into per-scenario p50/p90 stats;
+ * `gatePerf` compares a current summary against a committed baseline and
+ * trips on regressions beyond tolerance. Percentiles use nearest-rank on
+ * sorted values. Null / non-numeric metric values are excluded from the
+ * stat (n reflects only real samples); a field with zero samples is
+ * omitted entirely — no fake zeros.
+ */
+interface PerfStat {
+    p50: number;
+    p90: number;
+    n: number;
+}
+interface PerfBaseline {
+    version: 1;
+    /** key → metric field → stat. */
+    scenarios: Record<string, Record<string, PerfStat>>;
+}
+interface PerfRegression {
+    scenarioKey: string;
+    field: string;
+    baseline: PerfStat;
+    current: PerfStat;
+    /** percent over baseline p50 / p90, whichever tripped. */
+    overBy: {
+        p50Pct: number;
+        p90Pct: number;
+    };
+}
+interface PerfGateResult {
+    succeeded: boolean;
+    regressions: PerfRegression[];
+    /** Negative overBy: strictly better than baseline on both percentiles. */
+    improvements: PerfRegression[];
+    /** In baseline but absent (or under-sampled, n < minSamples) in current. */
+    missingScenarios: string[];
+    /** In records but absent from baseline. */
+    newScenarios: string[];
+}
+declare function summarizeRecords(records: ReadonlyArray<Record<string, unknown>>, keyOf: (record: Record<string, unknown>) => string | null, metricFields: ReadonlyArray<string>): PerfBaseline;
+declare function gatePerf(current: PerfBaseline, baseline: PerfBaseline, options?: {
+    tolerancePct?: number;
+    minSamples?: number;
+}): PerfGateResult;
+
+export { type IntegrityResult, type IntegrityViolation, type JourneySpec, type PerfBaseline, type PerfGateResult, type PerfRegression, type PerfScenario, type PerfStat, type ScenarioAxes, assertRecordIntegrity, checkRecordIntegrity, expandMatrix, gatePerf, scenarioKey, summarizeRecords };

package/dist/perf/index.js

@@ -0,0 +1,18 @@
+import {
+  assertRecordIntegrity,
+  checkRecordIntegrity,
+  expandMatrix,
+  gatePerf,
+  scenarioKey,
+  summarizeRecords
+} from "../chunk-STGVSCDH.js";
+import "../chunk-PZ5AY32C.js";
+export {
+  assertRecordIntegrity,
+  checkRecordIntegrity,
+  expandMatrix,
+  gatePerf,
+  scenarioKey,
+  summarizeRecords
+};
+//# sourceMappingURL=index.js.map

package/dist/perf/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/rl.js

@@ -1,15 +1,15 @@
-import {
-  detectRewardHacking,
-  extractVerifiableReward,
-  extractVerifiableRewardsFromRecords,
-  filterDeterministicallyRewarded
-} from "./chunk-5HRORJQY.js";
 import {
   doublyRobust,
   inverseProbabilityWeighting,
   offPolicyEstimateAll,
   selfNormalizedImportanceWeighting
 } from "./chunk-4DIJWVUT.js";
+import {
+  detectRewardHacking,
+  extractVerifiableReward,
+  extractVerifiableRewardsFromRecords,
+  filterDeterministicallyRewarded
+} from "./chunk-5HRORJQY.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.89.0",
+  "version": "0.90.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -109,6 +109,11 @@
       "import": "./dist/matrix/index.js",
       "default": "./dist/matrix/index.js"
     },
+    "./perf": {
+      "types": "./dist/perf/index.d.ts",
+      "import": "./dist/perf/index.js",
+      "default": "./dist/perf/index.js"
+    },
     "./multishot": {
       "types": "./dist/multishot/index.d.ts",
       "import": "./dist/multishot/index.js",