@tangle-network/agent-eval 0.93.0 → 0.95.0

package/dist/{llm-client-BeEcAokY.d.ts → llm-client.d.ts}

@@ -1,6 +1,3 @@
-import { A as AgentEvalError, C as CaptureIntegrityError } from './errors-CzMUYo7b.js';
-import { R as RawProviderSink, P as ProviderRedactor } from './raw-provider-sink-C46HDghv.js';
-
 /**
  * LLM client with graceful degrade.
  *
@@ -22,8 +19,9 @@ import { R as RawProviderSink, P as ProviderRedactor } from './raw-provider-sink
  * output (semantic concept judge, reviewer directives, critic scores). Primitives
  * that need free-form text use `callLlm` and parse output themselves.
  */
-
-interface LlmMessage {
+import { AgentEvalError, CaptureIntegrityError } from './errors';
+import { type ProviderRedactor, type RawProviderSink } from './trace/raw-provider-sink';
+export interface LlmMessage {
     role: 'system' | 'user' | 'assistant';
     /**
      * Either a plain text content string OR a multimodal content array
@@ -40,7 +38,7 @@ interface LlmMessage {
         };
     }>;
 }
-interface LlmCallRequest {
+export interface LlmCallRequest {
     model: string;
     messages: LlmMessage[];
     /** Optional JSON-mode response format (response_format: json_object). */
@@ -55,14 +53,14 @@ interface LlmCallRequest {
     /** Per-call timeout, default 300s. */
     timeoutMs?: number;
 }
-interface LlmUsage {
+export interface LlmUsage {
     promptTokens: number;
     completionTokens: number;
     totalTokens: number;
     /** Proxies populate this when prompt caching is on. */
     cachedPromptTokens?: number;
 }
-interface LlmCallResult {
+export interface LlmCallResult {
     /** The text content of the first choice. Empty string if none. */
     content: string;
     usage: LlmUsage;
@@ -75,16 +73,32 @@ interface LlmCallResult {
     model: string;
     /** Wall-clock duration of the HTTP call (last attempt, if retried). */
     durationMs: number;
+    /**
+     * `finish_reason` echoed from the first choice (`stop`, `length`,
+     * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.
+     * Exposed so a free-form `callLlm` caller CAN detect a truncated answer
+     * (`length`) instead of treating a cut-off completion as complete. Note:
+     * `callLlm` does not itself reject on it — acting on this signal is the
+     * caller's responsibility (in-repo free-form drivers do not yet enforce it).
+     */
+    finishReason?: string | null;
+    /**
+     * True when `content.trim()` is empty. An empty completion is a silent zero
+     * for free-form `callLlm` callers; this flag is the signal a caller can
+     * inspect to fail loud rather than proceed on an empty string. `callLlm`
+     * surfaces it but does not throw on it.
+     */
+    contentEmpty?: boolean;
     /** Raw response body. */
     raw: Record<string, unknown>;
 }
-declare class LlmCallError extends AgentEvalError {
+export declare class LlmCallError extends AgentEvalError {
     readonly status: number;
     readonly body: string;
     readonly model: string;
     constructor(message: string, status: number, body: string, model: string);
 }
-interface LlmClientOptions {
+export interface LlmClientOptions {
     /** Base URL (without trailing slash). Must end at the `/v1` prefix. */
     baseUrl?: string;
     /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
@@ -97,6 +111,21 @@ interface LlmClientOptions {
     };
     /** Default timeout in ms. Per-call can override. */
     defaultTimeoutMs?: number;
+    /**
+     * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
+     * each attempt's per-attempt timeout controller, so aborting it cancels
+     * the in-flight fetch. A caller abort is FATAL: it is not retried even
+     * though an AbortError otherwise matches the transient patterns.
+     */
+    signal?: AbortSignal;
+    /**
+     * Cross-attempt wall-clock budget in ms, measured from the first attempt.
+     * Before launching each attempt the loop checks the remaining budget and
+     * stops retrying once it is exhausted, rather than waiting the full
+     * per-attempt timeout on every retry. Bounds total time independent of
+     * `maxRetries` × `timeoutMs`.
+     */
+    deadlineMs?: number;
     /** Max retry attempts on retriable errors. Default 3 (1 initial + 2 retries). */
     maxRetries?: number;
     /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
@@ -134,38 +163,39 @@ interface LlmClientOptions {
  * treated identically whether it surfaces in the HTTP client or a
  * TCloud-backed judge.
  */
-declare function isTransientLlmError(err: unknown): boolean;
+export declare function isTransientLlmError(err: unknown): boolean;
 /** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */
-declare function backoffMs(attempt: number): number;
+export declare function backoffMs(attempt: number): number;
 /**
  * Strip a ```json / ``` code fence if the model emitted one.
  * Idempotent for naked JSON. Some models (claude-code via router, certain
  * deepseek models) wrap output even under json_object.
  */
-declare function stripFencedJson(raw: string): string;
+export declare function stripFencedJson(raw: string): string;
+export declare function extractJsonPayload(raw: string): string;
 /**
  * Low-level call. Returns raw content + usage + cost. Retries on transient
  * failures; does NOT degrade schema here — callers that want graceful
  * degrade use `callLlmJson`.
  */
-declare function callLlm(req: LlmCallRequest, opts?: LlmClientOptions): Promise<LlmCallResult>;
+export declare function callLlm(req: LlmCallRequest, opts?: LlmClientOptions): Promise<LlmCallResult>;
 /**
  * Structured-output call. Returns parsed JSON plus the raw result envelope.
  * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —
  * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept
  * the `response_format.json_schema` shape but DO accept `json_object`.
  */
-declare function callLlmJson<T = unknown>(req: LlmCallRequest, opts?: LlmClientOptions): Promise<{
+export declare function callLlmJson<T = unknown>(req: LlmCallRequest, opts?: LlmClientOptions): Promise<{
     value: T;
     result: LlmCallResult;
 }>;
-type LlmRouteAssertionReason = 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider';
-declare class LlmRouteAssertionError extends CaptureIntegrityError {
+export type LlmRouteAssertionReason = 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider';
+export declare class LlmRouteAssertionError extends CaptureIntegrityError {
     readonly reason: LlmRouteAssertionReason;
     readonly baseUrl: string;
     constructor(message: string, reason: LlmRouteAssertionReason, baseUrl: string);
 }
-interface LlmRouteRequirements {
+export interface LlmRouteRequirements {
     /**
      * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to
      * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using
@@ -197,7 +227,7 @@ interface LlmRouteRequirements {
  * Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call
  * from constructors and CI gates.
  */
-declare function assertLlmRoute(opts: LlmClientOptions, req?: LlmRouteRequirements): void;
+export declare function assertLlmRoute(opts: LlmClientOptions, req?: LlmRouteRequirements): void;
 /**
  * Probe whether a model is reachable. Returns latency + null error on
  * success; `ok=false` + error message on any failure (HTTP, timeout,
@@ -209,7 +239,7 @@ declare function assertLlmRoute(opts: LlmClientOptions, req?: LlmRouteRequiremen
  * for short prompts, so don't tighten this further. We don't validate
  * content; HTTP 200 means reachable.
  */
-declare function probeLlm(model: string, opts?: LlmClientOptions & {
+export declare function probeLlm(model: string, opts?: LlmClientOptions & {
     timeoutMs?: number;
 }): Promise<{
     ok: boolean;
@@ -221,7 +251,7 @@ declare function probeLlm(model: string, opts?: LlmClientOptions & {
  * Thin wrapper around the free functions; exists for callers that want
  * to inject a single configured instance into multiple primitives.
  */
-declare class LlmClient {
+export declare class LlmClient {
     private readonly opts;
     constructor(opts?: LlmClientOptions);
     call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult>;
@@ -230,5 +260,4 @@ declare class LlmClient {
         result: LlmCallResult;
     }>;
 }
-
-export { type LlmClientOptions as L, type LlmRouteRequirements as a, type LlmCallRequest as b, type LlmCallResult as c, LlmCallError as d, LlmClient as e, type LlmMessage as f, LlmRouteAssertionError as g, type LlmUsage as h, assertLlmRoute as i, backoffMs as j, callLlm as k, callLlmJson as l, isTransientLlmError as m, probeLlm as p, stripFencedJson as s };
+//# sourceMappingURL=llm-client.d.ts.map

package/dist/llm-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-client.d.ts","sourceRoot":"","sources":["../src/llm-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAA;AAChE,OAAO,EAEL,KAAK,gBAAgB,EAGrB,KAAK,eAAe,EACrB,MAAM,2BAA2B,CAAA;AAIlC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAA;IACrC;;;OAGG;IACH,OAAO,EACH,MAAM,GACN,KAAK,CACD;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAC9B;QAAE,IAAI,EAAE,WAAW,CAAC;QAAC,SAAS,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAA;SAAE,CAAA;KAAE,CACtF,CAAA;CACN;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,UAAU,EAAE,CAAA;IACtB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,oFAAoF;IACpF,UAAU,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAA;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,YAAY,EAAE,MAAM,CAAA;IACpB,gBAAgB,EAAE,MAAM,CAAA;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,uDAAuD;IACvD,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,QAAQ,CAAA;IACf;;;OAGG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAA;IACb,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAA;IAClB;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,yBAAyB;IACzB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC7B;AAED,qBAAa,YAAa,SAAQ,cAAc;aAG5B,MAAM,EAAE,MAAM;aACd,IAAI,EAAE,MAAM;aACZ,KAAK,EAAE,MAAM;gBAH7B,OAAO,EAAE,MAAM,EACC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM;CAIhC;AAED,MAAM,WAAW,gBAAgB;IAC/B,uEAAuE;IACvE,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,wFAAwF;IACxF,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,yGAAyG;IACzG,UAAU,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC5C,oDAAoD;IACpD,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,iFAAiF;IACjF,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,qGAAqG;IACrG,KAAK,CAAC,EAAE,OAAO,KAAK,CAAA;IACpB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,eAAe,CAAA;IACzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,gFAAgF;IAChF,YAAY,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAClD,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,gBAAgB,CAAA;CAC5B;AA2CD;;;;;;;;;;;GAWG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAEzD;AA6BD,uFAAuF;AACvF,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEjD;AAoFD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAInD;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAwBtD;AAsCD;;;;GAIG;AACH,wBAAsB,OAAO,CAC3B,GAAG,EAAE,cAAc,EACnB,IAAI,GAAE,gBAAqB,GAC1B,OAAO,CAAC,aAAa,CAAC,CA8OxB;AA6BD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,CAAC,GAAG,OAAO,EAC3C,GAAG,EAAE,cAAc,EACnB,IAAI,GAAE,gBAAqB,GAC1B,OAAO,CAAC;IAAE,KAAK,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,CAAC,CAe9C;AAiBD,MAAM,MAAM,uBAAuB,GAC/B,sBAAsB,GACtB,kBAAkB,GAClB,sBAAsB,GACtB,SAAS,GACT,gBAAgB,CAAA;AAEpB,qBAAa,sBAAuB,SAAQ,qBAAqB;aAG7C,MAAM,EAAE,uBAAuB;aAC/B,OAAO,EAAE,MAAM;gBAF/B,OAAO,EAAE,MAAM,EACC,MAAM,EAAE,uBAAuB,EAC/B,OAAO,EAAE,MAAM;CAIlC;AAED,MAAM,WAAW,oBAAoB;IACnC;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAChC;;;OAGG;IACH,eAAe,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAA;IACxC,8DAA8D;IAC9D,eAAe,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAA;IACxC,uDAAuD;IACvD,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,gBAAgB,EAAE,GAAG,GAAE,oBAAyB,GAAG,IAAI,CAiD3F;AAWD;;;;;;;;;;GAUG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,gBAAgB,GAAG;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAoBnE;AAED;;;;GAIG;AACH,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,GAAE,gBAAqB;IAExD,IAAI,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC;IAIzE,QAAQ,CAAC,CAAC,GAAG,OAAO,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,CAAC,EAAE,gBAAgB,GACrB,OAAO,CAAC;QAAE,KAAK,EAAE,CAAC,CAAC;QAAC,MAAM,EAAE,aAAa,CAAA;KAAE,CAAC;CAGhD"}

package/dist/llm-client.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=llm-client.test.d.ts.map

package/dist/llm-client.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-client.test.d.ts","sourceRoot":"","sources":["../src/llm-client.test.ts"],"names":[],"mappings":""}

package/dist/locked-jsonl-appender.d.ts

@@ -0,0 +1,19 @@
+/**
+ * LockedJsonlAppender — mutex-serialized JSONL append helper for arbitrary
+ * payloads. The reference-replay store does the same thing for typed
+ * `ReferenceReplayRun` rows; this is the generic version used by
+ * `MutationTelemetry`, `TrialTelemetry`, and any other consumer that wants
+ * append-only durable telemetry without rolling its own lock.
+ *
+ * Locks are per absolute file path (process-local). Cross-process
+ * concurrency is NOT addressed — that's an fcntl/flock problem.
+ */
+export declare class LockedJsonlAppender {
+    readonly path: string;
+    private readonly mutex;
+    constructor(path: string);
+    append(entry: unknown): Promise<void>;
+}
+/** Reset all internal mutex state — tests only. */
+export declare function resetLockedAppendersForTesting(): void;
+//# sourceMappingURL=locked-jsonl-appender.d.ts.map

package/dist/locked-jsonl-appender.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"locked-jsonl-appender.d.ts","sourceRoot":"","sources":["../src/locked-jsonl-appender.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAiBH,qBAAa,mBAAmB;aAEF,IAAI,EAAE,MAAM;IADxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAO;gBACD,IAAI,EAAE,MAAM;IAOlC,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;CAM5C;AAED,mDAAmD;AACnD,wBAAgB,8BAA8B,IAAI,IAAI,CAErD"}

package/dist/matrix/aggregation.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Per-axis aggregation of cell runs into `AxisSummary` rows.
+ *
+ * Pure: consumes the final `cells: [{cell, runs}]` array and returns the
+ * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost
+ * and duration always count — the budget was spent regardless.
+ */
+import type { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult } from './types';
+interface Row<Output> {
+    cell: MatrixCell;
+    result: CellResult<Output>;
+}
+export declare function summariseRows<Output>(rows: Row<Output>[], axisName: string, axisValue: string): AxisSummary;
+export declare function buildByAxis<Output>(cells: MatrixResult<Output>['cells'], axes: MatrixAxis<unknown>[], aggregateBy: string[]): Record<string, Record<string, AxisSummary>>;
+export {};
+//# sourceMappingURL=aggregation.d.ts.map

package/dist/matrix/aggregation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aggregation.d.ts","sourceRoot":"","sources":["../../src/matrix/aggregation.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAE5F,UAAU,GAAG,CAAC,MAAM;IAClB,IAAI,EAAE,UAAU,CAAA;IAChB,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;CAC3B;AAqBD,wBAAgB,aAAa,CAAC,MAAM,EAClC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,EACnB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAChB,WAAW,CAyCb;AA2BD,wBAAgB,WAAW,CAAC,MAAM,EAChC,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EACpC,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,EAC3B,WAAW,EAAE,MAAM,EAAE,GACpB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAe7C"}

package/dist/matrix/index.d.ts

@@ -1,34 +1,16 @@
-import { M as MatrixResult, a as MatrixAxis, A as AxisSummary, b as MatrixCell, C as CellResult, R as RunAgentMatrixOptions } from '../types-mn5Aqk7x.js';
-export { D as DefaultVerdict } from '../verdict-C9MlYujm.js';
-
 /**
- * Per-axis aggregation of cell runs into `AxisSummary` rows.
+ * @tangle-network/agent-eval/matrix
  *
- * Pure: consumes the final `cells: [{cell, runs}]` array and returns the
- * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost
- * and duration always count — the budget was spent regardless.
- */
-
-interface Row<Output> {
-    cell: MatrixCell;
-    result: CellResult<Output>;
-}
-declare function summariseRows<Output>(rows: Row<Output>[], axisName: string, axisValue: string): AxisSummary;
-declare function buildByAxis<Output>(cells: MatrixResult<Output>['cells'], axes: MatrixAxis<unknown>[], aggregateBy: string[]): Record<string, Record<string, AxisSummary>>;
-
-/**
- * N-axis cartesian runner.
- *
- * Expansion order: cartesian over `axes` in declared order, then `reps` as the
- * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned
- * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder
- * the output.
+ * N-axis cartesian runner over substrate types. The matrix is a runner +
+ * aggregator: cells carry caller-supplied values (AgentProfile from
+ * agent-interface, Driver / Validator from agent-runtime, rubric records, anything), and the
+ * runner returns per-axis pass / score / cost / duration summaries.
  *
- * Scheduling is a sliding window of in-flight promises capped at
- * `maxConcurrency`. The window stops admitting new cells when the cost
- * ceiling trips or the abort signal fires; in-flight cells finish.
+ * No "Spec" / "Wrapper" / "Config" types around substrate. The only
+ * matrix-owned types are orchestration types: `MatrixAxis<V>`, `MatrixCell`,
+ * `CellResult`, `AxisSummary`, `MatrixResult`.
  */
-
-declare function runAgentMatrix<Output>(opts: RunAgentMatrixOptions<Output>): Promise<MatrixResult<Output>>;
-
-export { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult, RunAgentMatrixOptions, buildByAxis, runAgentMatrix, summariseRows };
+export { buildByAxis, summariseRows } from './aggregation';
+export { runAgentMatrix } from './runner';
+export type { AxisSummary, CellResult, DefaultVerdict, MatrixAxis, MatrixCell, MatrixResult, RunAgentMatrixOptions, } from './types';
+//# sourceMappingURL=index.d.ts.map

package/dist/matrix/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matrix/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,YAAY,EACV,WAAW,EACX,UAAU,EACV,cAAc,EACd,UAAU,EACV,UAAU,EACV,YAAY,EACZ,qBAAqB,GACtB,MAAM,SAAS,CAAA"}

package/dist/matrix/runner.d.ts

@@ -0,0 +1,15 @@
+/**
+ * N-axis cartesian runner.
+ *
+ * Expansion order: cartesian over `axes` in declared order, then `reps` as the
+ * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned
+ * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder
+ * the output.
+ *
+ * Scheduling is a sliding window of in-flight promises capped at
+ * `maxConcurrency`. The window stops admitting new cells when the cost
+ * ceiling trips or the abort signal fires; in-flight cells finish.
+ */
+import type { MatrixResult, RunAgentMatrixOptions } from './types';
+export declare function runAgentMatrix<Output>(opts: RunAgentMatrixOptions<Output>): Promise<MatrixResult<Output>>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/matrix/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/matrix/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAIV,YAAY,EACZ,qBAAqB,EACtB,MAAM,SAAS,CAAA;AA8DhB,wBAAsB,cAAc,CAAC,MAAM,EACzC,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC,GAClC,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAwJ/B"}

package/dist/{types-mn5Aqk7x.d.ts → matrix/types.d.ts}

@@ -1,20 +1,17 @@
-import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
-
 /**
- * @experimental
- *
  * N-axis cartesian matrix over substrate types — types module.
  *
  * The matrix is a runner + aggregator. It iterates the cartesian product of
- * caller-provided axes (any value type — `AgentProfile` from sandbox, `Driver`
- * / `Validator` from agent-runtime, rubric records, thinking levels, anything)
+ * caller-provided axes (any value type — `AgentProfile` from agent-interface,
+ * `Driver` / `Validator` from agent-runtime, rubric records, thinking levels, anything)
  * and aggregates per-axis pass/score/cost summaries. Substrate types are
  * imported at the boundary by JSDoc only; the matrix never wraps them.
  */
-
+import type { DefaultVerdict } from '../verdict';
+export type { DefaultVerdict } from '../verdict';
 /** One axis = one dimension to iterate. `V` is the value type — pass any
  *  substrate type (AgentProfile, Driver, Validator, rubric record). */
-interface MatrixAxis<V> {
+export interface MatrixAxis<V> {
     /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
     name: string;
     /** Stable id per value. Used as the bucket key in aggregation. */
@@ -27,7 +24,7 @@ interface MatrixAxis<V> {
     label?: (value: V, id: string) => string;
 }
 /** A cell carries one picked value from each axis, keyed by axis name. */
-interface MatrixCell {
+export interface MatrixCell {
     axes: Record<string, {
         id: string;
         value: unknown;
@@ -37,7 +34,7 @@ interface MatrixCell {
     /** Stable sort key — preserves cartesian order across concurrent execution. */
     ordinal: number;
 }
-interface CellResult<Output> {
+export interface CellResult<Output> {
     output: Output;
     verdict: DefaultVerdict;
     costUsd: number;
@@ -50,7 +47,7 @@ interface CellResult<Output> {
         kind: string;
     };
 }
-interface AxisSummary {
+export interface AxisSummary {
     axisName: string;
     axisValue: string;
     cells: number;
@@ -61,7 +58,7 @@ interface AxisSummary {
     totalCostUsd: number;
     meanDurationMs: number;
 }
-interface MatrixResult<Output> {
+export interface MatrixResult<Output> {
     cells: Array<{
         cell: MatrixCell;
         runs: CellResult<Output>[];
@@ -83,7 +80,7 @@ interface MatrixResult<Output> {
     /** Stable id-like string generated at the end of the run. */
     matrixId: string;
 }
-interface RunAgentMatrixOptions<Output> {
+export interface RunAgentMatrixOptions<Output> {
     axes: MatrixAxis<unknown>[];
     /** User-supplied cell executor. May throw; the matrix captures throws as
      *  `CellResult.error` and continues. */
@@ -106,5 +103,4 @@ interface RunAgentMatrixOptions<Output> {
      *  and suppresses scheduling of new ones. */
     signal?: AbortSignal;
 }
-
-export type { AxisSummary as A, CellResult as C, MatrixResult as M, RunAgentMatrixOptions as R, MatrixAxis as a, MatrixCell as b };
+//# sourceMappingURL=types.d.ts.map

package/dist/matrix/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/matrix/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD;uEACuE;AACvE,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,2DAA2D;IAC3D,IAAI,EAAE,MAAM,CAAA;IACZ,kEAAkE;IAClE,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,CAAC,CAAA;KAAE,CAAC,CAAA;IACvC;4DACwD;IACxD,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,KAAK,MAAM,CAAA;CACzC;AAED,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACpD,gEAAgE;IAChE,GAAG,EAAE,MAAM,CAAA;IACX,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,UAAU,CAAC,MAAM;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,cAAc,CAAA;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;6CACyC;IACzC,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CAC1C;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,YAAY,CAAC,MAAM;IAClC,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAA;KAAE,CAAC,CAAA;IAC9D;mEAC+D;IAC/D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAA;IACnD,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAA;QAClB,YAAY,EAAE,MAAM,CAAA;QACpB;+CACuC;QACvC,YAAY,EAAE,MAAM,CAAA;QACpB,eAAe,EAAE,MAAM,CAAA;QACvB,gBAAgB,EAAE,MAAM,CAAA;QACxB,YAAY,EAAE,MAAM,CAAA;QACpB,UAAU,EAAE,MAAM,CAAA;KACnB,CAAA;IACD,6DAA6D;IAC7D,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,qBAAqB,CAAC,MAAM;IAC3C,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAA;IAC3B;4CACwC;IACxC,OAAO,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAA;IAC1D,sCAAsC;IACtC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,2DAA2D;IAC3D,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,SAAS,CAAC,KAAK,OAAO,CAAA;IAC/D,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB;;sDAEkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,IAAI,CAAA;IACvE;iDAC6C;IAC7C,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB"}

package/dist/meta-eval/calibration.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Calibration curve — binned "if eval says X, what does reality show?"
+ *
+ * Companion to correlationStudy. Raw correlation is a single number;
+ * the calibration curve shows *where* the eval is well-calibrated vs
+ * overconfident / underconfident. Buckets the eval metric, computes
+ * mean outcome per bucket, reports expected-calibration-error (ECE).
+ */
+import type { TraceStore } from '../trace/store';
+import type { EvalMetricSpec } from './correlation-study';
+import type { OutcomeStore } from './outcome-store';
+export interface CalibrationBin {
+    lower: number;
+    upper: number;
+    n: number;
+    evalMean: number;
+    outcomeMean: number;
+    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
+    gap: number;
+}
+export interface CalibrationReport {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    bins: CalibrationBin[];
+    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
+    ece: number;
+    /** Max bin gap — upper bound on miscalibration. */
+    maxGap: number;
+}
+export interface CalibrationOptions {
+    bins?: number;
+    /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
+    binning?: 'equal-width' | 'equal-frequency';
+    /** Clip eval values to [lo, hi] before binning. */
+    range?: {
+        lo: number;
+        hi: number;
+    };
+}
+export interface CalibrationPair {
+    evalScore: number;
+    outcome: number;
+}
+export declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
+export declare function calibrationFromPairs(inputPairs: CalibrationPair[], evalMetric: string, outcomeMetric: string, options?: CalibrationOptions): CalibrationReport | null;
+//# sourceMappingURL=calibration.d.ts.map

package/dist/meta-eval/calibration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibration.d.ts","sourceRoot":"","sources":["../../src/meta-eval/calibration.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACzD,OAAO,KAAK,EAAqB,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAEtE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,CAAC,EAAE,MAAM,CAAA;IACT,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,wEAAwE;IACxE,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,CAAC,EAAE,MAAM,CAAA;IACT,IAAI,EAAE,cAAc,EAAE,CAAA;IACtB,6EAA6E;IAC7E,GAAG,EAAE,MAAM,CAAA;IACX,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,wEAAwE;IACxE,OAAO,CAAC,EAAE,aAAa,GAAG,iBAAiB,CAAA;IAC3C,mDAAmD;IACnD,KAAK,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAA;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,UAAU,EACtB,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,cAAc,EAC1B,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CA8BnC;AAED,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,eAAe,EAAE,EAC7B,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,GAAE,kBAAuB,GAC/B,iBAAiB,GAAG,IAAI,CAsC1B"}

package/dist/meta-eval/correlation-study.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Correlation study — "does our eval score predict real-world outcomes?"
+ *
+ * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
+ * joins on runId, computes Pearson + Spearman + bootstrap CI for every
+ * (evalMetric, outcomeMetric) pair the caller declares.
+ *
+ * Without this number the framework is ornamental. With it and r > 0.6
+ * the framework is a moat — no other agent-eval tool publishes one.
+ */
+import type { Run } from '../trace/schema';
+import type { TraceStore } from '../trace/store';
+import type { OutcomeFilter, OutcomeStore } from './outcome-store';
+export interface EvalMetricSpec {
+    id: string;
+    /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+export interface OutcomePair {
+    evalMetric: string;
+    outcomeMetric: string;
+}
+export interface CorrelationResult {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    /** 95% bootstrap CI for Pearson. */
+    pearsonCi95: {
+        lower: number;
+        upper: number;
+    };
+    /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
+    verdict: 'strong' | 'moderate' | 'weak';
+}
+export interface CorrelationStudyResult {
+    pairs: CorrelationResult[];
+    joinedSamples: number;
+    skippedRuns: number;
+}
+export interface CorrelationStudyOptions {
+    /** Only join outcomes captured within this window after run.startedAt. */
+    maxCaptureLagMs?: number;
+    /** Restrict to a subset of outcomes (cohort, region, source). */
+    outcomeFilter?: OutcomeFilter;
+    /** Which outcome per run to use when multiple exist. Default 'latest'. */
+    reduction?: 'latest' | 'mean' | 'max';
+    /** Bootstrap iterations for the CI. Default 500. */
+    bootstrapIterations?: number;
+}
+export declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
+//# sourceMappingURL=correlation-study.d.ts.map

package/dist/meta-eval/correlation-study.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"correlation-study.d.ts","sourceRoot":"","sources":["../../src/meta-eval/correlation-study.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,KAAK,EAAqB,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAErF,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAA;IACV,yFAAyF;IACzF,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;CAClE;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,CAAC,EAAE,MAAM,CAAA;IACT,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,oCAAoC;IACpC,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC7C,oEAAoE;IACpE,OAAO,EAAE,QAAQ,GAAG,UAAU,GAAG,MAAM,CAAA;CACxC;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,iBAAiB,EAAE,CAAA;IAC1B,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,uBAAuB;IACtC,0EAA0E;IAC1E,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,iEAAiE;IACjE,aAAa,CAAC,EAAE,aAAa,CAAA;IAC7B,0EAA0E;IAC1E,SAAS,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAA;IACrC,oDAAoD;IACpD,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,UAAU,EACtB,YAAY,EAAE,YAAY,EAC1B,WAAW,EAAE,cAAc,EAAE,EAC7B,kBAAkB,EAAE,MAAM,EAAE,EAC5B,OAAO,GAAE,uBAA4B,GACpC,OAAO,CAAC,sBAAsB,CAAC,CA0EjC"}