@tangle-network/agent-eval 0.95.0 → 0.96.0

package/dist/{rl/off-policy.d.ts → off-policy-DiwuKKg7.d.ts}

@@ -36,7 +36,7 @@
  * complement with `replayCampaign` (exact replay where the request hashes
  * match) for high-confidence answers and OPE for the gap.
  */
-export interface OffPolicyTrajectory {
+interface OffPolicyTrajectory {
     /** Stable id, for traceability through the dataset. */
     runId: string;
     /** Reward observed under the behavior policy (the realized outcome). */
@@ -59,7 +59,7 @@ export interface OffPolicyTrajectory {
      */
     qHat?: number | null;
 }
-export interface OffPolicyEstimate {
+interface OffPolicyEstimate {
     /** Estimated value of the target policy. */
     value: number;
     /** Standard error of the estimate. */
@@ -74,7 +74,7 @@ export interface OffPolicyEstimate {
      */
     maxImportanceWeight: number;
 }
-export interface OffPolicyOptions {
+interface OffPolicyOptions {
     /**
      * Cap importance weights at this value (Ionides 2008 truncated IS) to
      * trade unbiasedness for variance reduction. Default `Infinity` (no cap).
@@ -92,13 +92,13 @@ export interface OffPolicyOptions {
  * of E[reward under target policy]. Variance scales with the spread of
  * target/behavior ratios.
  */
-export declare function inverseProbabilityWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+declare function inverseProbabilityWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
 /**
  * Self-Normalized Importance Sampling. Lower variance than vanilla IPS at
  * the cost of small bias (vanishing as N grows). The right default for
  * LLM-agent evaluation where overlap is often poor.
  */
-export declare function selfNormalizedImportanceWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+declare function selfNormalizedImportanceWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
 /**
  * Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).
  *
@@ -117,15 +117,16 @@ export declare function selfNormalizedImportanceWeighting(trajectories: OffPolic
  * report's `n` reflects the full set but `effectiveSampleSize` accounts
  * for the lost variance reduction).
  */
-export declare function doublyRobust(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+declare function doublyRobust(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
 /**
  * Convenience: run all three estimators and return them side-by-side.
  * The recommended diagnostic — agreement across estimators is a much
  * stronger signal than any single one.
  */
-export declare function offPolicyEstimateAll(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): {
+declare function offPolicyEstimateAll(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): {
     ips: OffPolicyEstimate;
     snips: OffPolicyEstimate;
     dr: OffPolicyEstimate;
 };
-//# sourceMappingURL=off-policy.d.ts.map
+
+export { type OffPolicyEstimate as O, type OffPolicyOptions as a, type OffPolicyTrajectory as b, doublyRobust as d, inverseProbabilityWeighting as i, offPolicyEstimateAll as o, selfNormalizedImportanceWeighting as s };

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.95.0",
+    "version": "0.96.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{meta-eval/outcome-store.d.ts → outcome-store-rnXLEqSn.d.ts}

@@ -10,7 +10,7 @@
  * The whole point of this module is to make the meta-eval correlation
  * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
  */
-export interface DeploymentOutcome {
+interface DeploymentOutcome {
     runId: string;
     capturedAt: number;
     /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
@@ -20,7 +20,7 @@ export interface DeploymentOutcome {
     /** Free-form provenance (source system, pipeline version). */
     source?: string;
 }
-export interface OutcomeFilter {
+interface OutcomeFilter {
     runIds?: string[];
     since?: number;
     until?: number;
@@ -30,24 +30,24 @@ export interface OutcomeFilter {
     };
     source?: string;
 }
-export interface OutcomeStore {
+interface OutcomeStore {
     append(outcome: DeploymentOutcome): Promise<void>;
     /** All outcomes attached to this run (a single run can have many — multiple
      *  capture windows over deployment time). */
     forRun(runId: string): Promise<DeploymentOutcome[]>;
     list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
 }
-export declare class InMemoryOutcomeStore implements OutcomeStore {
+declare class InMemoryOutcomeStore implements OutcomeStore {
     private items;
     append(outcome: DeploymentOutcome): Promise<void>;
     forRun(runId: string): Promise<DeploymentOutcome[]>;
     list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
 }
-export interface FileSystemOutcomeStoreOptions {
+interface FileSystemOutcomeStoreOptions {
     dir: string;
     maxBytes?: number;
 }
-export declare class FileSystemOutcomeStore implements OutcomeStore {
+declare class FileSystemOutcomeStore implements OutcomeStore {
     private dir;
     private maxBytes;
     private memo?;
@@ -59,4 +59,5 @@ export declare class FileSystemOutcomeStore implements OutcomeStore {
     forRun(runId: string): Promise<DeploymentOutcome[]>;
     list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
 }
-//# sourceMappingURL=outcome-store.d.ts.map
+
+export { type DeploymentOutcome as D, FileSystemOutcomeStore as F, InMemoryOutcomeStore as I, type OutcomeFilter as O, type FileSystemOutcomeStoreOptions as a, type OutcomeStore as b };

package/dist/{pareto.d.ts → pareto-E-pembql.d.ts}

@@ -13,14 +13,14 @@
  * or 'minimize' (lower=better). Candidates are any object; pass an
  * `objective(candidate)` accessor.
  */
-export type Direction = 'maximize' | 'minimize';
-export interface Objective<T> {
+type Direction = 'maximize' | 'minimize';
+interface Objective<T> {
     /** Stable label used in reports. */
     name: string;
     direction: Direction;
     value: (candidate: T) => number;
 }
-export interface ParetoResult<T> {
+interface ParetoResult<T> {
     frontier: T[];
     dominated: T[];
     /** Index map: frontier[i] dominates each of dominatedBy[i]. */
@@ -30,13 +30,13 @@ export interface ParetoResult<T> {
     }>;
 }
 /** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
-export declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
+declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
 /**
  * Compute the non-dominated frontier. Candidates with NaN/Infinity on any
  * objective are excluded (can't rank them). A candidate enters the frontier
  * iff no other candidate dominates it.
  */
-export declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
+declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
 /**
  * Weighted-sum scalarisation. Use as a tie-break / single-winner selector
  * when callers don't want to consume a frontier. Each objective contributes
@@ -46,7 +46,7 @@ export declare function paretoFrontier<T>(candidates: T[], objectives: Objective
  * Direction is honoured automatically — `minimize` axes have their values
  * inverted before scaling so "higher scalar = better" always holds.
  */
-export declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], options?: {
+declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], options?: {
     weights?: Partial<Record<string, number>>;
 }): Array<{
     candidate: T;
@@ -64,7 +64,7 @@ export declare function scalarScore<T>(candidates: T[], objectives: Objective<T>
  * input. Higher distance = more isolated = should be preferred when
  * preserving diversity.
  */
-export declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]): Array<{
     candidate: T;
     distance: number;
 }>;
@@ -73,8 +73,9 @@ export declare function crowdingDistance<T>(candidates: T[], objectives: Objecti
  * NSGA-II selection step. Returns the frontier sorted by descending crowding
  * distance so callers can `.slice(0, k)` to pick K diverse winners.
  */
-export declare function paretoFrontierWithCrowding<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+declare function paretoFrontierWithCrowding<T>(candidates: T[], objectives: Objective<T>[]): Array<{
     candidate: T;
     distance: number;
 }>;
-//# sourceMappingURL=pareto.d.ts.map
+
+export { type Direction as D, type Objective as O, type ParetoResult as P, paretoFrontierWithCrowding as a, crowdingDistance as c, dominates as d, paretoFrontier as p, scalarScore as s };

package/dist/perf/index.d.ts

@@ -1,17 +1,123 @@
 /**
- * @tangle-network/agent-eval/perf
+ * Journey × axes matrix for infra performance benchmarks.
  *
- * Domain-agnostic infra-performance benchmarking substrate: a journeys ×
- * axes scenario matrix, record-integrity contracts over flat metric
- * records, and a percentile ratchet (summarize → baseline → gate).
+ * A journey is one measurable user path ("provision.cold", "chat.ttft");
+ * axes are free-form scenario dimensions (driver, region, image…). The
+ * matrix expansion is pure bookkeeping — running the scenarios and
+ * recording metrics is the caller's job. This module complements the
+ * judge-panel `BenchmarkRunner` (src/benchmark.ts): that one scores
+ * QUALITY via judges, this one structures LATENCY / RELIABILITY runs
+ * over flat metric records.
+ */
+/** One measurable user journey (e.g. "provision.cold", "chat.ttft"). */
+interface JourneySpec {
+    id: string;
+    description: string;
+    /** Needs a real LLM call — schedule nightly, not per-PR. */
+    requiresLLM: boolean;
+    /**
+     * Fields that MUST be non-null on a passing record of this journey.
+     * A "passing" record missing one is an integrity violation, not a pass.
+     */
+    requiredFields: ReadonlyArray<string>;
+    /** Numeric floors, e.g. {field: 'event_count', min: 1} for streaming. */
+    minimums?: ReadonlyArray<{
+        field: string;
+        min: number;
+    }>;
+    /** Per-phase breakdown fields expected non-null (subset of requiredFields semantics, reported separately). */
+    phaseFields?: ReadonlyArray<string>;
+}
+interface ScenarioAxes {
+    /** e.g. driver: ['docker','firecracker'] — every key is a free-form dimension. */
+    [dimension: string]: ReadonlyArray<string>;
+}
+interface PerfScenario {
+    /** `${journeyId}|${dim1}=${v1}|${dim2}=${v2}` (dims sorted). */
+    key: string;
+    journey: JourneySpec;
+    axes: Record<string, string>;
+}
+/** Stable scenario key: journey id then `dim=value` pairs in sorted-dim order. */
+declare function scenarioKey(journeyId: string, axes: Record<string, string>): string;
+/** Cartesian expansion; `filter` lets callers drop invalid combos (e.g. firecracker×resume). */
+declare function expandMatrix(journeys: ReadonlyArray<JourneySpec>, axes: ScenarioAxes, filter?: (journeyId: string, combo: Record<string, string>) => boolean): PerfScenario[];
+
+/**
+ * Record-integrity contracts for perf metric records.
+ *
+ * A record that claims `pass === true` must actually carry the journey's
+ * required measurements — a "passing" provision run with a null
+ * `total_ms` is a lying record, not a pass. Failed records are exempt:
+ * a run that errored mid-flight legitimately has nulls.
+ */
+
+interface IntegrityViolation {
+    recordIndex: number;
+    journeyId: string;
+    field: string;
+    reason: 'null-required-field' | 'below-minimum';
+    detail: string;
+}
+interface IntegrityResult {
+    succeeded: boolean;
+    violations: IntegrityViolation[];
+}
+/**
+ * Validates flat metric records (Record<string, unknown> with a boolean
+ * `pass` field) against their journey contract. Only records with
+ * pass === true are checked — a failed record may legitimately have nulls.
+ * resolveJourney maps a record to its JourneySpec (or null to skip).
+ */
+declare function checkRecordIntegrity(records: ReadonlyArray<Record<string, unknown>>, resolveJourney: (record: Record<string, unknown>) => JourneySpec | null): IntegrityResult;
+/** Throws an Error listing every violation when the result fails. */
+declare function assertRecordIntegrity(records: ReadonlyArray<Record<string, unknown>>, resolveJourney: (record: Record<string, unknown>) => JourneySpec | null): void;
+
+/**
+ * Percentile ratchet over perf metric records.
  *
- * Complements the judge-panel `BenchmarkRunner` (root): that one scores
- * QUALITY; this one scores LATENCY / RELIABILITY over flat metric records.
+ * `summarizeRecords` folds flat records into per-scenario p50/p90 stats;
+ * `gatePerf` compares a current summary against a committed baseline and
+ * trips on regressions beyond tolerance. Percentiles use nearest-rank on
+ * sorted values. Null / non-numeric metric values are excluded from the
+ * stat (n reflects only real samples); a field with zero samples is
+ * omitted entirely — no fake zeros.
  */
-export type { IntegrityResult, IntegrityViolation } from './integrity';
-export { assertRecordIntegrity, checkRecordIntegrity } from './integrity';
-export type { JourneySpec, PerfScenario, ScenarioAxes } from './journey';
-export { expandMatrix, scenarioKey } from './journey';
-export type { PerfBaseline, PerfGateResult, PerfRegression, PerfStat } from './ratchet';
-export { gatePerf, summarizeRecords } from './ratchet';
-//# sourceMappingURL=index.d.ts.map
+interface PerfStat {
+    p50: number;
+    p90: number;
+    n: number;
+}
+interface PerfBaseline {
+    version: 1;
+    /** key → metric field → stat. */
+    scenarios: Record<string, Record<string, PerfStat>>;
+}
+interface PerfRegression {
+    scenarioKey: string;
+    field: string;
+    baseline: PerfStat;
+    current: PerfStat;
+    /** percent over baseline p50 / p90, whichever tripped. */
+    overBy: {
+        p50Pct: number;
+        p90Pct: number;
+    };
+}
+interface PerfGateResult {
+    succeeded: boolean;
+    regressions: PerfRegression[];
+    /** Negative overBy: strictly better than baseline on both percentiles. */
+    improvements: PerfRegression[];
+    /** In baseline but absent (or under-sampled, n < minSamples) in current. */
+    missingScenarios: string[];
+    /** In records but absent from baseline. */
+    newScenarios: string[];
+}
+declare function summarizeRecords(records: ReadonlyArray<Record<string, unknown>>, keyOf: (record: Record<string, unknown>) => string | null, metricFields: ReadonlyArray<string>): PerfBaseline;
+declare function gatePerf(current: PerfBaseline, baseline: PerfBaseline, options?: {
+    tolerancePct?: number;
+    minSamples?: number;
+}): PerfGateResult;
+
+export { type IntegrityResult, type IntegrityViolation, type JourneySpec, type PerfBaseline, type PerfGateResult, type PerfRegression, type PerfScenario, type PerfStat, type ScenarioAxes, assertRecordIntegrity, checkRecordIntegrity, expandMatrix, gatePerf, scenarioKey, summarizeRecords };

package/dist/pipelines/index.d.ts

@@ -1,8 +1,173 @@
-export * from './budget-breach';
-export * from './failure-cluster';
-export * from './first-divergence';
-export * from './judge-agreement';
-export * from './regression';
-export * from './stuck-loop';
-export * from './tool-waste';
-//# sourceMappingURL=index.d.ts.map
+import { h as BudgetSpec, R as Run, T as ToolSpan } from '../schema-m0gsnbt3.js';
+import { T as TraceStore, R as RunFilter } from '../store-BcFXE6LG.js';
+export { a as FailureCluster, F as FailureClusterReport, f as failureClusterView } from '../failure-cluster-DH9Flgcf.js';
+import { a as TrajectoryStep } from '../trajectory-2TkpSEVh.js';
+import { B as BaselineOptions, a as BaselineReport } from '../baseline-Bbid3WoO.js';
+export { c as computeToolUseMetrics } from '../baseline-Bbid3WoO.js';
+import { l as llmSpans } from '../query-B7GGjRox.js';
+
+/**
+ * BudgetBreachView — aggregates breach events across the corpus.
+ *
+ * Answers: which dimensions get hit most often? Which scenarios are
+ * underbudgeted? Which variants trigger the most breaches?
+ */
+
+interface BudgetBreachFinding {
+    runId: string;
+    scenarioId: string;
+    variantId?: string;
+    dimension: keyof BudgetSpec;
+    limit: number;
+    consumed: number;
+    excessRatio: number;
+    timestamp: number;
+}
+interface BudgetBreachReport {
+    findings: BudgetBreachFinding[];
+    byDimension: Record<string, number>;
+    byScenario: Record<string, number>;
+    byVariant: Record<string, number>;
+    totalRuns: number;
+    breachedRunRatio: number;
+}
+declare function budgetBreachView(store: TraceStore, options?: {
+    scenarioId?: string;
+    variantId?: string;
+}): Promise<BudgetBreachReport>;
+
+/**
+ * FirstDivergenceView — aligns two trajectories by step index, reports
+ * the first step where they differ.
+ *
+ * "Differ" is configurable — default is (kind, toolName if tool, model
+ * if llm). Use this view to attribute "why is variant B better?" to a
+ * specific step rather than an aggregate mean delta.
+ */
+
+interface DivergenceReport {
+    runA: string;
+    runB: string;
+    firstDivergenceIndex: number | null;
+    aStep?: TrajectoryStep;
+    bStep?: TrajectoryStep;
+    reason?: string;
+    /** Common prefix length (steps that matched). */
+    commonPrefixLen: number;
+}
+interface DivergenceOptions {
+    /** Returns true if two steps are considered equal. Default: kind + tool/model match. */
+    stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean;
+}
+declare function firstDivergenceView(store: TraceStore, runA: string, runB: string, options?: DivergenceOptions): Promise<DivergenceReport>;
+
+/**
+ * JudgeAgreementView — pairwise agreement between judges across the
+ * corpus, grouped by dimension.
+ *
+ * Output drives two workflows:
+ *   - Judge robustness audit: "does Claude agree with GPT at κ ≥ 0.6?"
+ *   - Calibration tracking: κ vs golden human labels over time (by
+ *     providing a `humanGoldenJudgeId`).
+ */
+
+interface JudgePair {
+    judgeA: string;
+    judgeB: string;
+    dimension: string;
+    /** Number of (targetSpanId, dimension) tuples both judges scored. */
+    commonItems: number;
+    pearson: number;
+    krippendorff: number;
+}
+interface JudgeAgreementReport {
+    pairs: JudgePair[];
+    dimensions: string[];
+    judgeIds: string[];
+}
+declare function judgeAgreementView(store: TraceStore): Promise<JudgeAgreementReport>;
+
+/**
+ * RegressionView — compares a candidate slice to a baseline slice on a
+ * named metric. Delegates the statistics (Welch's t-test, Cohen's d,
+ * IQR stability) to `baseline.ts`.
+ *
+ * This is the entry point for CI regression gates: "given runs tagged
+ * release=A and release=B, did any metric regress?"
+ */
+
+interface RegressionSpec {
+    metric: string;
+    higherIsBetter: boolean;
+    /** Extract a scalar from a run. Default extractors handle common metrics. */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface RegressionOptions extends BaselineOptions {
+    baseline: RunFilter;
+    candidate: RunFilter;
+}
+declare function regressionView(store: TraceStore, metrics: RegressionSpec[], options: RegressionOptions): Promise<BaselineReport>;
+
+/**
+ * StuckLoopView — detects when an agent calls the same tool with the
+ * same (or structurally similar) arguments ≥ N times in a short window.
+ *
+ * Rationale: agents that loop are the number-one production failure
+ * mode on long-horizon flows. The view returns (runId, toolName,
+ * argHash, occurrences, windowMs) for each detected loop plus a
+ * fraction of runs affected.
+ */
+
+interface StuckLoopFinding {
+    runId: string;
+    toolName: string;
+    argHash: string;
+    occurrences: number;
+    spanIds: string[];
+    /** Milliseconds between first and last call in the loop. */
+    windowMs: number;
+}
+interface StuckLoopReport {
+    findings: StuckLoopFinding[];
+    affectedRunRatio: number;
+    totalRuns: number;
+}
+interface StuckLoopOptions {
+    /** Minimum call count to flag a loop (default 3). */
+    minOccurrences?: number;
+    /** Filter to a specific runId; omit to scan the entire corpus. */
+    runId?: string;
+}
+declare function stuckLoopView(store: TraceStore, options?: StuckLoopOptions): Promise<StuckLoopReport>;
+
+/**
+ * ToolWasteView — fraction of tool calls whose results weren't used
+ * downstream. Without a "used" signal we fall back to structural
+ * proxies: error calls, duplicate calls, and tool calls followed by
+ * zero subsequent LLM spans are all considered waste.
+ *
+ * Consumers can pass a `usageOracle` that inspects a tool span and
+ * returns true iff the tool's result appears in a later LLM message,
+ * artifact, or state mutation — that's the canonical definition; the
+ * default heuristic is a reasonable fallback.
+ */
+
+interface ToolWasteFinding {
+    runId: string;
+    wastedCalls: number;
+    totalCalls: number;
+    wasteRate: number;
+}
+interface ToolWasteReport {
+    byRun: ToolWasteFinding[];
+    overallWasteRate: number;
+}
+interface ToolWasteOptions {
+    runId?: string;
+    usageOracle?: (tool: ToolSpan, later: {
+        llm: Awaited<ReturnType<typeof llmSpans>>;
+    }) => boolean;
+}
+declare function toolWasteView(store: TraceStore, options?: ToolWasteOptions): Promise<ToolWasteReport>;
+
+export { type BudgetBreachFinding, type BudgetBreachReport, type DivergenceOptions, type DivergenceReport, type JudgeAgreementReport, type JudgePair, type RegressionOptions, type RegressionSpec, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, budgetBreachView, firstDivergenceView, judgeAgreementView, regressionView, stuckLoopView, toolWasteView };