@tangle-network/agent-eval 0.85.0 → 0.89.0

package/dist/{rubric-predictive-validity-CnEl9Jc8.d.ts → rubric-predictive-validity-Cy_W-hWZ.d.ts}

@@ -1,4 +1,4 @@
-import { R as RunRecord } from './run-record-De9VarXR.js';
+import { R as RunRecord } from './run-record-e7vj1uZQ.js';
 import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
 
 /**

package/dist/{run-campaign-4Y5V5CN3.js → run-campaign-RDGAM5KJ.js}

@@ -1,10 +1,10 @@
 import {
   runCampaign
-} from "./chunk-ZZ2HOPME.js";
-import "./chunk-IDVBLYCY.js";
+} from "./chunk-TWS7AZEY.js";
+import "./chunk-LMZQ2Z4U.js";
 import "./chunk-3BFEG2F6.js";
 import "./chunk-PZ5AY32C.js";
 export {
   runCampaign
 };
-//# sourceMappingURL=run-campaign-4Y5V5CN3.js.map
+//# sourceMappingURL=run-campaign-RDGAM5KJ.js.map

package/dist/{run-improvement-loop-CNqQckTj.d.ts → run-improvement-loop-5z_l5zDz.d.ts}

@@ -1,5 +1,5 @@
-import { L as LlmClientOptions } from './llm-client-CuUg2Mn3.js';
-import { I as ImprovementDriver, S as Scenario, g as CampaignResult, k as GateResult, D as DispatchFn, a as JudgeConfig, L as LabeledScenarioStore, h as CampaignTraceWriter, m as GenerationRecord, M as MutableSurface, P as ParetoParent, G as Gate } from './types-D7lLRYe9.js';
+import { L as LlmClientOptions } from './llm-client-BeEcAokY.js';
+import { I as ImprovementDriver, S as Scenario, g as CampaignResult, k as GateResult, D as DispatchFn, a as JudgeConfig, L as LabeledScenarioStore, h as CampaignTraceWriter, m as GenerationRecord, M as MutableSurface, P as ParetoParent, G as Gate } from './types-BU-7W85F.js';
 
 /**
  * @experimental

package/dist/{run-record-De9VarXR.d.ts → run-record-e7vj1uZQ.d.ts}

@@ -1,4 +1,4 @@
-import { V as ValidationError } from './errors-Dwqw-T_m.js';
+import { V as ValidationError } from './errors-CzMUYo7b.js';
 import { F as FailureClass } from './schema-m0gsnbt3.js';
 
 type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';

package/dist/{runtime-trajectory-BLRiaifm.d.ts → runtime-trajectory-BDgfGZSr.d.ts}

@@ -1,4 +1,4 @@
-import { a as RunSplitTag } from './run-record-De9VarXR.js';
+import { a as RunSplitTag } from './run-record-e7vj1uZQ.js';
 
 interface RuntimeTrajectoryHookEvent {
     id: string;

package/dist/{semantic-concept-judge-DIEgr_6v.d.ts → semantic-concept-judge-Dn8Z6KEG.d.ts}

@@ -1,11 +1,10 @@
 import { AxAIService } from '@ax-llm/ax';
-import { c as TraceAnalystKindSpec } from './kind-factory-CVecZZG_.js';
-import { b as AnalystRegistryOptions, a as AnalystRegistry } from './registry-DrEQ3Luj.js';
 import { z } from 'zod';
-import { c as AnalystFinding, A as Analyst, a as AnalystContext } from './types-Cu3u_x59.js';
+import { c as AnalystFinding, A as Analyst, a as AnalystContext } from './types-2VVIL04s.js';
+import { T as TraceAnalystKindSpec } from './kind-factory-5b7xXXOr.js';
 import { a as TraceAnalystSpan } from './store-C1YxJDEK.js';
-import { L as LlmClientOptions } from './llm-client-CuUg2Mn3.js';
-import { S as Severity } from './multi-layer-verifier-DlWCXuxL.js';
+import { L as LlmClientOptions } from './llm-client-BeEcAokY.js';
+import { S as Severity } from './multi-layer-verifier-DUZXrPDA.js';
 
 interface CreateAnalystAiConfig {
     /** OpenAI-compatible API key forwarded as `Authorization: Bearer`.
@@ -33,31 +32,6 @@ interface CreateAnalystAiConfig {
  */
 declare function createAnalystAi(config: CreateAnalystAiConfig): AxAIService;
 
-/**
- * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers
- * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`.
- *
- * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no
- * model and is model-agnostic by construction). The agentic RLM kinds are
- * registered only when an `ai` service is supplied — so a caller with no LLM
- * still gets the full behavioral/efficiency diagnosis, and the substrate's
- * "any model (including no model)" guarantee holds at the suite level.
- */
-
-interface DefaultAnalystRegistryOptions {
-    /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
-    ai?: AxAIService;
-    /** Model for the agentic kinds (falls back to the ai service default). */
-    model?: string;
-    /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
-    kinds?: readonly TraceAnalystKindSpec[];
-    /** Set false to omit the deterministic behavioral analyst (default: include). */
-    includeBehavioral?: boolean;
-    /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */
-    registry?: AnalystRegistryOptions;
-}
-declare function buildDefaultAnalystRegistry(opts?: DefaultAnalystRegistryOptions): AnalystRegistry;
-
 /**
  * Typed `FindingSubject` — the canonical grammar every analyst kind emits.
  *
@@ -647,4 +621,4 @@ declare function runSemanticConceptJudge(input: SemanticConceptJudgeInput, optio
  */
 declare function createSemanticConceptJudge(options?: SemanticConceptJudgeOptions): (input: SemanticConceptJudgeInput) => Promise<SemanticConceptJudgeResult>;
 
-export { type ConceptFinding as A, type BehavioralMetrics as B, type CreateAnalystAiConfig as C, DEFAULT_TRACE_ANALYST_KINDS as D, type ConceptSpec as E, FAILURE_MODE_KIND_SPEC as F, type ConceptWeightStrategy as G, DEFAULT_COMPLEXITY_WEIGHTS as H, IMPROVEMENT_KIND_SPEC as I, SEMANTIC_CONCEPT_JUDGE_VERSION as J, KIND_EXPECTED_SUBJECTS as K, type SemanticConceptJudgeResult as L, type SuboptimalCode as M, type SuboptimalSignal as N, computeTraceMetrics as O, type PersistedFinding as P, createSemanticConceptJudge as Q, runSemanticConceptJudge as R, type SemanticConceptJudgeOptions as S, type SemanticConceptJudgeInput as a, type DefaultAnalystRegistryOptions as b, type DiffPolicy as c, FINDING_SUBJECT_GRAMMAR_PROMPT as d, FINDING_SUBJECT_KINDS as e, type FindingSubject as f, type FindingSubjectKind as g, FindingSubjectStringSchema as h, type FindingsDiff as i, FindingsStore as j, KNOWLEDGE_GAP_KIND_SPEC as k, KNOWLEDGE_POISONING_KIND_SPEC as l, SKILL_USAGE_ANALYST as m, SkillUsageAnalyst as n, type SkillUsageRecord as o, type SkillUsageReport as p, type SkillUsageScanConfig as q, buildDefaultAnalystRegistry as r, buildSkillUsageReport as s, createAnalystAi as t, defaultIsMaterial as u, diffFindings as v, emitSkillUsageFindings as w, parseFindingSubject as x, renderFindingSubject as y, type ConceptComplexity as z };
+export { type ConceptWeightStrategy as A, type BehavioralMetrics as B, type CreateAnalystAiConfig as C, DEFAULT_TRACE_ANALYST_KINDS as D, DEFAULT_COMPLEXITY_WEIGHTS as E, FAILURE_MODE_KIND_SPEC as F, SEMANTIC_CONCEPT_JUDGE_VERSION as G, type SemanticConceptJudgeResult as H, IMPROVEMENT_KIND_SPEC as I, type SuboptimalCode as J, KIND_EXPECTED_SUBJECTS as K, type SuboptimalSignal as L, computeTraceMetrics as M, createSemanticConceptJudge as N, runSemanticConceptJudge as O, type PersistedFinding as P, type SemanticConceptJudgeOptions as S, type SemanticConceptJudgeInput as a, type DiffPolicy as b, FINDING_SUBJECT_GRAMMAR_PROMPT as c, FINDING_SUBJECT_KINDS as d, type FindingSubject as e, type FindingSubjectKind as f, FindingSubjectStringSchema as g, type FindingsDiff as h, FindingsStore as i, KNOWLEDGE_GAP_KIND_SPEC as j, KNOWLEDGE_POISONING_KIND_SPEC as k, SKILL_USAGE_ANALYST as l, SkillUsageAnalyst as m, type SkillUsageRecord as n, type SkillUsageReport as o, type SkillUsageScanConfig as p, buildSkillUsageReport as q, createAnalystAi as r, defaultIsMaterial as s, diffFindings as t, emitSkillUsageFindings as u, parseFindingSubject as v, renderFindingSubject as w, type ConceptComplexity as x, type ConceptFinding as y, type ConceptSpec as z };

package/dist/series-convergence-D5OWMBg6.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Series convergence — detects whether a sequence of scalar measurements
+ * is stabilizing, drifting, or noisy.
+ *
+ * Lifted from ADC convergence.ts. The per-turn `ConvergenceTracker` is
+ * about progress *within* a single run; this module is about drift
+ * *across* runs (e.g. "are my nightly eval scores stabilizing?").
+ *
+ * Three signals:
+ *   - stabilized: last K values have low variance (< epsilon) — done
+ *   - drifting:   recent trend is monotonic and beyond noise — regressing or improving
+ *   - noisy:      neither — keep iterating, but flag as untrustworthy for gating
+ */
+interface SeriesConvergenceOptions {
+    /** Window size for "recent" analysis (default 5). */
+    window?: number;
+    /** Coefficient-of-variation threshold below which the window is stabilized (default 0.05 = 5%). */
+    stableCv?: number;
+    /** Minimum monotone run length to call drift (default 3). */
+    driftRun?: number;
+}
+interface SeriesConvergenceResult {
+    state: 'stabilized' | 'drifting-up' | 'drifting-down' | 'noisy' | 'insufficient-data';
+    windowMean: number;
+    windowCv: number;
+    /** Longest monotonic run at the tail of the series (positive for up, negative for down). */
+    tailRun: number;
+    /** True when n ≥ window AND windowCv ≤ stableCv. */
+    stable: boolean;
+}
+declare function analyzeSeries(values: number[], options?: SeriesConvergenceOptions): SeriesConvergenceResult;
+
+export { type SeriesConvergenceOptions as S, type SeriesConvergenceResult as a, analyzeSeries as b };

package/dist/{statistics-CnC1FMbx.d.ts → statistics-C7PozGrZ.d.ts}

@@ -1,4 +1,4 @@
-import { C as ContinuousAgreementOptions, a as ContinuousAgreement } from './judge-calibration-DilmB3Ml.js';
+import { d as ContinuousAgreementOptions, C as ContinuousAgreement } from './judge-calibration-0p2QcWNE.js';
 import { J as JudgeScore } from './types-Croy5h7V.js';
 
 /** Identity: dimensions already follow "higher = better" by prompt convention
@@ -245,5 +245,74 @@ interface PairedBootstrapOptions {
  * gain is real at the confidence level. Throws on unequal sample sizes.
  */
 declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
+interface EProcessOptions {
+    /** Type-I error budget. The process decides when wealth ≥ 1/alpha
+     *  (Ville's inequality). Default 0.05. */
+    alpha?: number;
+    /** Truncation bound on the predictable bet λ ∈ [0, maxBet]. Must satisfy
+     *  maxBet < 1/nullMean so every wealth factor stays strictly positive.
+     *  Default 0.5. */
+    maxBet?: number;
+    /** The null boundary m₀ for H0: E[x] ≤ m₀ on x ∈ [0,1]. Default 0.5
+     *  (the paired-delta encoding x = (d+1)/2 maps "no effect" to 1/2).
+     *  A pre-registered minEffect shifts this — see `sequentialPairedGate`. */
+    nullMean?: number;
+}
+interface EProcessStep {
+    /** Current wealth W_n — the e-value against H0 after n observations. */
+    wealth: number;
+    /** Observations consumed so far. */
+    n: number;
+    /** True from the first n where W_n ≥ 1/alpha onward (sticky). */
+    decided: boolean;
+}
+interface EProcessState extends EProcessStep {
+    alpha: number;
+    maxBet: number;
+    nullMean: number;
+    /** The decision boundary 1/alpha. */
+    threshold: number;
+    /** Observation count at the first threshold crossing; undefined until decided. */
+    decidedAtN?: number;
+}
+interface EProcess {
+    /** Consume one observation x ∈ [0,1]. Throws on non-finite / out-of-range
+     *  input — a silent clamp would corrupt the type-I guarantee. */
+    update(x: number): EProcessStep;
+    state(): EProcessState;
+}
+/**
+ * Betting test-martingale for bounded observations — the e-process core of
+ * anytime-valid sequential testing (Waudby-Smith & Ramdas, "Estimating means
+ * of bounded random variables by betting", JRSS-B 2024).
+ *
+ * Observations x_i ∈ [0,1]; H0: E[x] ≤ m₀ (`nullMean`, default 1/2). Wealth
+ *
+ *   W_t = Π_{i≤t} (1 + λ_i (x_i − m₀)),  W_0 = 1
+ *
+ * with the truncated GROW-style plug-in bet computed from PRIOR observations:
+ *
+ *   λ_i = clamp((μ̂_{i−1} − m₀) / (σ̂²_{i−1} + (μ̂_{i−1} − m₀)²), 0, maxBet)
+ *
+ * where μ̂/σ̂² are the shrunk running estimates μ̂_t = (1/2 + Σx_i)/(t+1),
+ * σ̂²_t = (1/4 + Σ(x_i − μ̂_i)²)/(t+1).
+ *
+ * PREDICTABILITY INVARIANT (load-bearing): λ_i is a function of x_1..x_{i−1}
+ * ONLY — it may never see x_i. With λ_i ≥ 0 predictable, each factor has
+ * E[1 + λ_i(x_i − m₀) | past] ≤ 1 under H0, so W is a nonnegative
+ * supermartingale and Ville's inequality gives P(∃t: W_t ≥ 1/α) ≤ α — the
+ * type-I guarantee holds at ANY data-dependent stopping time. λ_1 is always 0
+ * (no prior evidence), so the first observation never moves wealth.
+ *
+ * `decided` latches at the first crossing W_t ≥ 1/α and never un-latches;
+ * wealth keeps updating after the crossing (the e-process remains valid), but
+ * the decision time is the first crossing.
+ */
+declare function eProcess(opts?: EProcessOptions): EProcess;
+/** Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not
+ *  cryptographic. Exported so e-process shuffles and bootstrap resampling
+ *  share ONE PRNG implementation; a seed is REQUIRED (unseeded randomness in
+ *  gate verdicts is non-reproducible by construction). */
+declare function mulberry32(seed: number): () => number;
 
-export { type CliffsMagnitude as C, type PairedBootstrapOptions as P, type WeightedCompositeInput as W, type PairedBootstrapResult as a, benjaminiHochberg as b, type CorpusAgreementOptions as c, type CorpusAgreementPerDimension as d, type CorpusAgreementReport as e, type CorpusScoreRecord as f, type WeightedCompositeResult as g, bonferroni as h, cliffsDelta as i, cohensD as j, confidenceInterval as k, corpusInterRaterAgreement as l, corpusInterRaterAgreementFromJudgeScores as m, interRaterReliability as n, interpretCliffs as o, pairedBootstrap as p, mannWhitneyU as q, normalizeScores as r, pairedMde as s, pairedTTest as t, partialCredit as u, requiredSampleSize as v, wilcoxonSignedRank as w, weightedComposite as x, weightedMean as y };
+export { partialCredit as A, requiredSampleSize as B, type CorpusAgreementReport as C, weightedComposite as D, type EProcessState as E, weightedMean as F, type PairedBootstrapOptions as P, type WeightedCompositeInput as W, type PairedBootstrapResult as a, benjaminiHochberg as b, type CliffsMagnitude as c, type CorpusAgreementOptions as d, type CorpusAgreementPerDimension as e, type CorpusScoreRecord as f, type EProcess as g, type EProcessOptions as h, type EProcessStep as i, type WeightedCompositeResult as j, bonferroni as k, cliffsDelta as l, cohensD as m, confidenceInterval as n, corpusInterRaterAgreement as o, pairedBootstrap as p, corpusInterRaterAgreementFromJudgeScores as q, eProcess as r, interRaterReliability as s, interpretCliffs as t, mannWhitneyU as u, mulberry32 as v, wilcoxonSignedRank as w, normalizeScores as x, pairedMde as y, pairedTTest as z };

package/dist/{summary-report-Db0dDSWP.d.ts → summary-report-DGmUucwQ.d.ts}

@@ -1,4 +1,4 @@
-import { R as RunRecord } from './run-record-De9VarXR.js';
+import { R as RunRecord } from './run-record-e7vj1uZQ.js';
 import { F as FailureClusterReport } from './failure-cluster-CL7IVgkJ.js';
 
 /**

package/dist/traces.d.ts

@@ -1,9 +1,9 @@
-import { N as NotFoundError, R as ReplayError } from './errors-Dwqw-T_m.js';
+import { N as NotFoundError, R as ReplayError } from './errors-CzMUYo7b.js';
 import { P as ProviderRedactor, R as RawProviderSink, d as RawProviderEvent } from './raw-provider-sink-C46HDghv.js';
 export { F as FileSystemRawProviderSink, a as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, b as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, c as RawProviderDirection, e as RawProviderSinkFilter, f as defaultProviderRedactor, p as providerFromBaseUrl } from './raw-provider-sink-C46HDghv.js';
 import { a as RunCompleteHookContext, R as RunCompleteHook } from './emitter-DEZwY14K.js';
 export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-DEZwY14K.js';
-export { b as RunIntegrityError, R as RunIntegrityExpectations, c as RunIntegrityIssue, d as RunIntegrityIssueCode, a as RunIntegrityReport, e as assertRunCaptured, t as throwIfRunIncomplete } from './integrity-CJzrpUua.js';
+export { b as RunIntegrityError, R as RunIntegrityExpectations, c as RunIntegrityIssue, d as RunIntegrityIssueCode, a as RunIntegrityReport, e as assertRunCaptured, t as throwIfRunIncomplete } from './integrity-VJ9A7aST.js';
 import { T as TraceStore } from './store-CKUAgsJz.js';
 export { E as EventFilter, F as FileSystemTraceStore, a as FileSystemTraceStoreOptions, I as InMemoryTraceStore, R as RunFilter, S as SpanFilter } from './store-CKUAgsJz.js';
 export { a as aggregateLlm, b as argHash, g as groupBy, j as judgeSpans, l as llmSpans, r as runFailureClass, c as runsForScenario, t as toolSpans } from './query-CqTxMwDw.js';
@@ -14,7 +14,7 @@ import { A as AnalyzeTracesOptions, b as AnalyzeTracesResult } from './analyst-C
 export { a as AnalyzeTracesInput, c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
 import { h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, T as TraceAnalysisStore, g as TraceAnalystFilters, b as DatasetOverview, Q as QueryTracesPage, l as ViewTraceResult, V as ViewSpansResult, c as SearchTraceResult, S as SearchSpanResult } from './store-C1YxJDEK.js';
 export { D as DEFAULT_TRACE_ANALYST_BUDGETS, E as ErrorCluster, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, f as TraceAnalystByteBudgets, a as TraceAnalystSpan, j as TraceAnalystTraceSummary, k as ViewTraceOversized } from './store-C1YxJDEK.js';
-import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from './run-record-De9VarXR.js';
+import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from './run-record-e7vj1uZQ.js';
 import { AxFunction } from '@ax-llm/ax';
 
 /**
@@ -581,6 +581,52 @@ declare function traceAnalystFunctionGroup(opts: BuildTraceAnalystToolsOpts): {
     functions: AxFunction[];
 };
 
+/**
+ * Token-usage extraction from chat-completions responses and SSE streams.
+ *
+ * `captureFetchToRawSink` records the raw provider triple but deliberately does
+ * not interpret token usage — cost is a per-consumer axis. Three consumers each
+ * re-implement the same `usage` parser on top of the captured response (the
+ * OpenAI `prompt_tokens`/`completion_tokens` shape, the Anthropic
+ * `input_tokens`/`output_tokens` shape, and the camelCase variants), plus an
+ * SSE accumulator that sums the per-chunk `usage` deltas. This is the one
+ * canonical version.
+ *
+ * Both functions return `null` (not a silent `{ input: 0, output: 0 }`) when no
+ * usage is present, so a caller can tell "no usage reported" from "zero tokens".
+ */
+interface ExtractedUsage {
+    input: number;
+    output: number;
+    /** Cached prompt tokens, when the provider reports them. */
+    cached?: number;
+}
+/**
+ * Pull `{ input, output, cached? }` from a parsed chat-completions response
+ * body. Accepts a top-level `usage` object (the common case) or a body that IS
+ * the usage object. Returns null when neither an input nor an output count is
+ * present — callers must inspect for null rather than treat it as zero cost.
+ */
+declare function extractUsage(body: unknown): ExtractedUsage | null;
+/**
+ * Sum token usage across an SSE response body. Each `data:` line is parsed and
+ * fed to `extractUsage`; the per-chunk counts are accumulated. The `[DONE]`
+ * sentinel and non-JSON lines are skipped. Returns null when no chunk carried
+ * usage — distinguishing a usage-less stream from a genuine zero.
+ *
+ * Providers report SSE usage in two ways: a single terminal chunk with the full
+ * totals, or incremental per-chunk deltas. Summing is correct for the delta
+ * case and harmless for the terminal case (one non-null chunk ⇒ the total).
+ */
+declare function extractUsageFromSse(text: string): ExtractedUsage | null;
+/**
+ * Extract usage from an HTTP `Response` without consuming the caller's body:
+ * clones, reads the text, and tries the JSON parser first, then the SSE
+ * accumulator. Best-effort — returns null on any read/parse miss so a usage tee
+ * never takes down the underlying call.
+ */
+declare function extractUsageFromResponse(response: Response): Promise<ExtractedUsage | null>;
+
 /**
  * `captureFetchToRawSink` — wrap a `fetch` so every request / response / error
  * against a provider is recorded into a `RawProviderSink` as the canonical
@@ -617,6 +663,14 @@ interface CaptureFetchOptions {
     /** When true, a sink-write failure propagates to the caller. Default false
      *  — capture is best-effort so a sink failure never kills the LLM call. */
     failClosed?: boolean;
+    /**
+     * Invoked with the token usage parsed off each successful response (JSON or
+     * SSE), keyed by the captured context. Lets a caller fold usage → cost without
+     * re-cloning the response themselves. Not called when the response carries no
+     * usage. Best-effort: a throw here is swallowed (it never kills the LLM call)
+     * unless `failClosed` is set.
+     */
+    onUsage?: (usage: ExtractedUsage, ctx: CaptureFetchContext) => void;
 }
 declare function captureFetchToRawSink(fetch: typeof globalThis.fetch, sink: RawProviderSink, ctx: CaptureFetchContext, opts?: CaptureFetchOptions): typeof globalThis.fetch;
 
@@ -890,4 +944,4 @@ declare function iterateRawCalls(sink: RawProviderSink, filter?: {
     spanId?: string;
 }): AsyncGenerator<ReplayCacheEntry>;
 
-export { AnalyzeTracesOptions, AnalyzeTracesResult, type CaptureFetchContext, type CaptureFetchOptions, DatasetOverview, type ExportableSpan, type FlattenOtlpOptions, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, ProviderRedactor, QueryTracesPage, RawProviderEvent, RawProviderSink, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, Run, RunCompleteHook, RunCompleteHookContext, SearchSpanResult, SearchTraceResult, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, type TraceAggregate, TraceAnalysisStore, TraceAnalystFilters, type TraceAnalystHookOptions, TraceAnalystSpanKind, TraceAnalystSpanStatus, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracesToOtlpResult, ViewSpansResult, ViewTraceResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };
+export { AnalyzeTracesOptions, AnalyzeTracesResult, type CaptureFetchContext, type CaptureFetchOptions, DatasetOverview, type ExportableSpan, type ExtractedUsage, type FlattenOtlpOptions, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, ProviderRedactor, QueryTracesPage, RawProviderEvent, RawProviderSink, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, Run, RunCompleteHook, RunCompleteHookContext, SearchSpanResult, SearchTraceResult, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, type TraceAggregate, TraceAnalysisStore, TraceAnalystFilters, type TraceAnalystHookOptions, TraceAnalystSpanKind, TraceAnalystSpanStatus, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracesToOtlpResult, ViewSpansResult, ViewTraceResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };

package/dist/traces.js

@@ -15,6 +15,9 @@ import {
   describeTraceInsightScope,
   domainEvidencePattern,
   exportRunAsOtlp,
+  extractUsage,
+  extractUsageFromResponse,
+  extractUsageFromSse,
   flattenOtlpExportToNdjson,
   inferDomainKeywords,
   iterateRawCalls,
@@ -25,7 +28,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-XQL22JDG.js";
+} from "./chunk-QG2OVF2D.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -52,16 +55,18 @@ import {
   isToolSpan
 } from "./chunk-5BKGXME7.js";
 import {
-  DEFAULT_TRACE_ANALYST_BUDGETS,
-  OtlpFileTraceStore,
-  SpanNotFoundError,
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  analyzeTraces
+} from "./chunk-UHMJT4T7.js";
+import {
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  OtlpFileTraceStore,
+  SpanNotFoundError,
   TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TraceFileMissingError,
   TraceNotFoundError,
-  analyzeTraces,
   asNumber,
   asString,
   buildTraceAnalystTools,
@@ -73,7 +78,7 @@ import {
   readOtlpStatus,
   stringField,
   traceAnalystFunctionGroup
-} from "./chunk-VUINJM5M.js";
+} from "./chunk-QAY5UIJO.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -138,6 +143,9 @@ export {
   domainEvidencePattern,
   exportRunAsOtlp,
   extractOtlpAttributes,
+  extractUsage,
+  extractUsageFromResponse,
+  extractUsageFromSse,
   firstNumberAttr,
   firstStringAttr,
   flattenOtlpExportToNdjson,

package/dist/{types-Cu3u_x59.d.ts → types-2VVIL04s.d.ts}

@@ -1,7 +1,7 @@
-import { R as RunRecord } from './run-record-De9VarXR.js';
+import { R as RunRecord } from './run-record-e7vj1uZQ.js';
 import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
 import { a as JudgeInput } from './types-Croy5h7V.js';
-import { b as LlmCallRequest, c as LlmCallResult } from './llm-client-CuUg2Mn3.js';
+import { b as LlmCallRequest, c as LlmCallResult } from './llm-client-BeEcAokY.js';
 
 /**
  * ChatClient — the single LLM abstraction analysts call.

package/dist/{types-D7lLRYe9.d.ts → types-BU-7W85F.d.ts}

@@ -1,4 +1,4 @@
-import { b as RunTokenUsage } from './run-record-De9VarXR.js';
+import { b as RunTokenUsage } from './run-record-e7vj1uZQ.js';
 
 /**
  * @experimental
@@ -93,10 +93,30 @@ interface JudgeConfig<TArtifact, TScenario extends Scenario = Scenario> {
     }): JudgeScore | Promise<JudgeScore>;
     appliesTo?: (scenario: TScenario) => boolean;
 }
+/** The canonical judge verdict shape — one declaration, shared by campaign
+ *  judges and the multishot judge runner (which re-exports this type).
+ *
+ *  Scale is PRODUCER-DEFINED: campaign convention is [0,1]; the legacy
+ *  multishot runner emits 0-10. Cross-scale comparison must go through
+ *  `detectScale` (src/campaign/gates/statistical-heldout.ts, used by
+ *  promotion-policy) — never renormalize a producer's values in place, as
+ *  downstream thresholds (`composite >= 5` in multishot/matrix.ts, live-soak
+ *  `>= 7` gates) key on the producer's native scale. */
 interface JudgeScore {
     dimensions: Record<string, number>;
     composite: number;
     notes: string;
+    /** Set when the judge itself failed (call error, unparseable output).
+     *  `composite`/`dimensions` carry no signal — aggregators MUST exclude
+     *  failed scores from means instead of folding them into zeros. */
+    failed?: true;
+    /** Ensemble extras (populated by `ensembleJudge`): max per-dimension
+     *  spread across surviving judges — the inter-rater signal. */
+    maxDisagreement?: number;
+    /** Ensemble extras: judge identities whose verdict failed. */
+    failedJudges?: string[];
+    /** Ensemble extras: each surviving judge's per-dimension scores. */
+    perJudge?: Record<string, Record<string, number>>;
 }
 /** @experimental A tier-4 code surface — a candidate change to the agent's
  *  IMPLEMENTATION, not its prompt. Produced by autoresearch (reads codebase +

package/dist/{types-CqPax19X.d.ts → types-mn5Aqk7x.d.ts}

@@ -1,4 +1,4 @@
-import { D as DefaultVerdict } from './verdict-CeEgtjyI.js';
+import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
 
 /**
  * @experimental

package/dist/{verdict-CeEgtjyI.d.ts → verdict-C9MlYujm.d.ts}

@@ -17,6 +17,9 @@
  * Minimal verdict shape — `valid` + `score` are required; `scores` +
  * `notes` are optional surface. Validators that need richer shapes
  * parameterise `Validator<Output, MyVerdict>` with their own type.
+ *
+ * Need structured extras? Extend DefaultVerdict with typed fields — never
+ * serialize extras into `notes`.
  */
 interface DefaultVerdict {
     /** Whether the output meets the validator's pass criteria. */

package/dist/wire/index.d.ts

@@ -1,4 +1,4 @@
-import { F as FeedbackTrajectoryStore } from '../feedback-trajectory-B3rErRsh.js';
+import { F as FeedbackTrajectoryStore } from '../feedback-trajectory-D9OVLrg9.js';
 import { T as TraceStore } from '../store-CKUAgsJz.js';
 import { z } from 'zod';
 import { OpenAPIObject } from 'openapi3-ts/oas31';
@@ -8,8 +8,8 @@ import { Hono } from 'hono';
 import '../control-runtime-DuFBYg7A.js';
 import '../emitter-DEZwY14K.js';
 import '../schema-m0gsnbt3.js';
-import '../dataset-B2kL-fSM.js';
-import '../errors-Dwqw-T_m.js';
+import '../dataset-BbGkaN2I.js';
+import '../errors-CzMUYo7b.js';
 
 declare const RubricDimensionSchema: z.ZodObject<{
     id: z.ZodString;

package/dist/workflow/index.d.ts

@@ -1,25 +1,26 @@
 import { W as WorkflowTopology } from '../harness-optimizer-EnEnQPsr.js';
-import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from '../run-record-De9VarXR.js';
-import { c as AnalystFinding, h as AnalystSeverity, E as EvidenceRef } from '../types-Cu3u_x59.js';
-import { F as FailureClusterInsight } from '../insight-report-3ADTfClO.js';
-import { a as VerificationReport, L as LayerResult } from '../multi-layer-verifier-DlWCXuxL.js';
+import { a as RunSplitTag, b as RunTokenUsage, R as RunRecord } from '../run-record-e7vj1uZQ.js';
+import { c as AnalystFinding, h as AnalystSeverity, E as EvidenceRef } from '../types-2VVIL04s.js';
+import { F as FailureClusterInsight } from '../insight-report-BBwvOh6x.js';
+import { a as VerificationReport, L as LayerResult } from '../multi-layer-verifier-DUZXrPDA.js';
 import { F as FailureClusterReport } from '../failure-cluster-CL7IVgkJ.js';
 import { R as RedactionRule, a as RedactionReport } from '../redact-B40YG2M_.js';
-import { D as DatasetSplit } from '../dataset-B2kL-fSM.js';
-import { a as FeedbackTrajectory } from '../feedback-trajectory-B3rErRsh.js';
-import { a as PairedBootstrapResult } from '../statistics-CnC1FMbx.js';
+import { D as DatasetSplit } from '../dataset-BbGkaN2I.js';
+import { a as FeedbackTrajectory } from '../feedback-trajectory-D9OVLrg9.js';
+import { a as PairedBootstrapResult } from '../statistics-C7PozGrZ.js';
 import '../pareto-E-pembql.js';
 import '../run-critic-BAIjX99r.js';
 import '../schema-m0gsnbt3.js';
 import '../store-CKUAgsJz.js';
-import '../errors-Dwqw-T_m.js';
+import '../errors-CzMUYo7b.js';
 import '../store-C1YxJDEK.js';
 import '../types-Croy5h7V.js';
 import '@tangle-network/tcloud';
-import '../llm-client-CuUg2Mn3.js';
+import '../llm-client-BeEcAokY.js';
 import '../raw-provider-sink-C46HDghv.js';
-import '../summary-report-Db0dDSWP.js';
-import '../judge-calibration-DilmB3Ml.js';
+import '../summary-report-DGmUucwQ.js';
+import '../judge-calibration-0p2QcWNE.js';
+import '../verdict-C9MlYujm.js';
 import '../control-runtime-DuFBYg7A.js';
 import '../emitter-DEZwY14K.js';
 

package/dist/workflow/index.js

@@ -1,6 +1,6 @@
 import {
   pairedBootstrap
-} from "../chunk-IDVBLYCY.js";
+} from "../chunk-LMZQ2Z4U.js";
 import {
   DEFAULT_REDACTION_RULES,
   redactString

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.85.0",
+  "version": "0.89.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -39,6 +39,16 @@
       "import": "./dist/rl.js",
       "default": "./dist/rl.js"
     },
+    "./diagnose": {
+      "types": "./dist/diagnose.d.ts",
+      "import": "./dist/diagnose.js",
+      "default": "./dist/diagnose.js"
+    },
+    "./fuzz": {
+      "types": "./dist/fuzz.d.ts",
+      "import": "./dist/fuzz.js",
+      "default": "./dist/fuzz.js"
+    },
     "./traces": {
       "types": "./dist/traces.d.ts",
       "import": "./dist/traces.js",
@@ -169,6 +179,19 @@
   "publishConfig": {
     "access": "public"
   },
+  "scripts": {
+    "build": "tsup && pnpm openapi",
+    "dev": "tsup --watch",
+    "prepare": "husky",
+    "prepublishOnly": "pnpm build",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check src",
+    "format": "biome format --write src",
+    "openapi": "node dist/cli.js openapi --out dist/openapi.json",
+    "verify:package": "node scripts/verify-package-exports.mjs"
+  },
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^8.5.0",
     "@ax-llm/ax": "^19.0.25",
@@ -196,6 +219,16 @@
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
   },
+  "pnpm": {
+    "minimumReleaseAge": 4320,
+    "minimumReleaseAgeExclude": [
+      "@tangle-network/sandbox"
+    ],
+    "overrides": {
+      "postcss@<8.5.10": "^8.5.10",
+      "ws@>=8.0.0 <8.20.1": "^8.20.1"
+    }
+  },
   "engines": {
     "node": ">=20"
   },
@@ -208,15 +241,5 @@
     ]
   },
   "license": "MIT",
-  "scripts": {
-    "build": "tsup && pnpm openapi",
-    "dev": "tsup --watch",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit",
-    "lint": "biome check src",
-    "format": "biome format --write src",
-    "openapi": "node dist/cli.js openapi --out dist/openapi.json",
-    "verify:package": "node scripts/verify-package-exports.mjs"
-  }
-}
+  "packageManager": "pnpm@10.22.0"
+}