@tangle-network/agent-eval 0.23.1 → 0.24.0

package/dist/multi-layer-verifier-LkP3LVKj.d.ts

@@ -0,0 +1,141 @@
+/**
+ * Multi-layer verifier — ordered pipeline of verification layers.
+ *
+ * Different contract from {@link JudgeRunner} (which runs parallel
+ * specs against a sandbox). MultiLayerVerifier is a DAG of layers
+ * (install → typecheck → build → lint → serve → semantic → …) with
+ * dependency-based skip, per-layer findings, soft-fail semantics, and
+ * an aggregated `blendedScore` across all passed layers.
+ *
+ * Use when you want:
+ *   - ordered stages where a failing upstream stage skips downstream ones
+ *   - each stage produces rich `findings` (severity + message + evidence)
+ *   - a single composite score across stages with per-stage weights
+ *   - soft-fail stages whose failure doesn't abort the pipeline
+ *
+ * Use {@link JudgeRunner} when you want:
+ *   - N independent judges running in parallel against the same artifact
+ *   - no inter-judge dependencies
+ *   - boolean `passed` per judge + overall
+ *
+ * Both primitives compose — JudgeRunner can be invoked as a single
+ * layer inside a MultiLayerVerifier if that suits the caller.
+ */
+type LayerStatus = 'pass' | 'fail' | 'skipped' | 'error' | 'timeout';
+type Severity = 'critical' | 'major' | 'minor' | 'info';
+interface Finding {
+    severity: Severity;
+    message: string;
+    evidence?: string;
+    /** Optional layer name the finding belongs to (set by the verifier if omitted). */
+    layer?: string;
+    /**
+     * Free-form structured payload — used by `multiToolchainLayer` to attach
+     * `{ adapter: 'pnpm' }`, by judges to attach evidence pointers, etc.
+     * Renderers MAY interrogate; agent-eval primitives never assume shape.
+     */
+    detail?: Record<string, unknown>;
+}
+interface LayerResult {
+    layer: string;
+    status: LayerStatus;
+    /** 0..1 score, optional — layers that don't produce a numeric score omit. */
+    score?: number;
+    durationMs: number;
+    findings: Finding[];
+    /** Short human-readable summary (one line). */
+    reason?: string;
+    /**
+     * Numeric layer-level diagnostics: error counts, warning counts,
+     * cyclomatic complexity, total adapter wall-time, etc. Keyed by
+     * diagnostic name; null = "diagnostic not applicable / not measured."
+     * Renderers that know the keys can display them; ones that don't,
+     * ignore. Free-form on purpose — consumers type the value shape in
+     * their own namespace. Added in 0.10.
+     */
+    diagnostics?: Record<string, number | null>;
+    /** Any rich per-layer detail — rendered as-is by consumers that know the layer. */
+    detail?: Record<string, unknown>;
+}
+interface VerifyContext<Env = unknown> {
+    /** Per-run opaque context the caller provides. Layers destructure what they need. */
+    env: Env;
+    /** Previously-computed results from layers that already ran. */
+    prior: Record<string, LayerResult>;
+    /** Signal — if aborted, layers MUST bail within reasonable wall. */
+    signal: AbortSignal;
+}
+interface Layer<Env = unknown> {
+    name: string;
+    /** Stages that must have `status: 'pass'` before this layer runs. */
+    dependsOn?: string[];
+    /**
+     * Weight in the composite `blendedScore`. Default 1.0. Layers with weight 0
+     * contribute findings but not score.
+     */
+    weight?: number;
+    /**
+     * If true, a `fail` status contributes to `blendedScore` (as 0) instead of
+     * being dropped — use for layers whose failure is a real signal. Default:
+     * fail drops from numerator + denominator, matching VB's existing semantics.
+     */
+    failContributesToScore?: boolean;
+    /** Optional per-layer wall-cap in ms. Honored by the verifier (AbortSignal). */
+    capMs?: number;
+    run: (ctx: VerifyContext<Env>) => Promise<LayerResult> | LayerResult;
+}
+interface VerifyOptions<Env = unknown> {
+    env: Env;
+    /**
+     * Overall wall cap. Default: sum of layer capMs, or Infinity if any layer
+     * omits a cap. The verifier short-circuits remaining layers on overall cap.
+     */
+    overallCapMs?: number;
+    /** Called with each layer result as it completes. */
+    onLayer?: (result: LayerResult) => void;
+}
+interface VerificationReport {
+    layers: LayerResult[];
+    passCount: number;
+    failCount: number;
+    skippedCount: number;
+    errorCount: number;
+    /** True iff at least one scored layer ran AND every scored layer passed. */
+    allPass: boolean;
+    /**
+     * Weighted mean of `score` across contributing layers. 0 when no layers
+     * contributed. See {@link Layer.failContributesToScore} for fail semantics.
+     */
+    blendedScore: number;
+    durationMs: number;
+    startedAt: string;
+    finishedAt: string;
+}
+/**
+ * Grade a semantic-concept-style judge result into a single layer status.
+ *
+ * Pass when overall score >= threshold AND no critical-severity concept gap.
+ * Fail otherwise. Use inside a `Layer.run` when wrapping a concept judge.
+ *
+ * Generalized from VerticalBench H3 fix: `failingConcepts.length === 0` was
+ * too strict — a single concept at 6/10 failed the entire layer despite
+ * overall score being >= 0.7. Now we trust the judge's own `severity` field:
+ * `critical` findings veto; `major`/`minor` reduce the score but don't veto.
+ */
+declare function gradeSemanticStatus(input: {
+    score: number;
+    findings: Array<{
+        severity: Severity;
+        present?: boolean;
+        score?: number;
+    }>;
+    available: boolean;
+    threshold?: number;
+}): LayerStatus;
+declare class MultiLayerVerifier<Env = unknown> {
+    private readonly layers;
+    constructor(layers: Layer<Env>[]);
+    run(opts: VerifyOptions<Env>): Promise<VerificationReport>;
+}
+
+export { type Finding as F, type Layer as L, MultiLayerVerifier as M, type Severity as S, type VerificationReport as V, type LayerResult as a, type VerifyContext as b, type LayerStatus as c, type VerifyOptions as d, gradeSemanticStatus as g };

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.23.1",
+    "version": "0.24.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/optimization.d.ts

@@ -1,8 +1,11 @@
-export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './eval-campaign-Ds5QljIh.js';
-export { A as ActionableSideInfo, a as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, G as GenerationReport, I as InMemoryTrialCache, M as MultiShotGateConfig, b as MultiShotGateResult, c as MultiShotMutateAdapter, d as MultiShotOptimizationConfig, e as MultiShotOptimizationResult, f as MultiShotRun, g as MultiShotRunInput, h as MultiShotRunner, i as MultiShotScore, j as MultiShotScorer, k as MultiShotSplit, l as MultiShotTrace, m as MultiShotTrialResult, n as MultiShotVariant, o as MutateAdapter, P as PromptEvolutionConfig, p as PromptEvolutionEvent, q as PromptEvolutionResult, R as ReflectionContext, r as ReflectionProposal, S as ScenarioAggregate, s as ScoreAdapter, T as TrialCache, t as TrialResult, u as TrialTrace, V as VariantAggregate, v as buildReflectionPrompt, w as defaultMultiShotObjectives, x as parseReflectionResponse, y as runMultiShotOptimization, z as runPromptEvolution, B as trialTraceFromMultiShotTrial } from './summary-report-Ce1r4EYo.js';
-export { F as FeedbackArtifactType, a as FeedbackAttempt, b as FeedbackLabel, c as FeedbackLabelKind, d as FeedbackLabelSource, e as FeedbackOptimizerRow, f as FeedbackOutcome, g as FeedbackReplayAdapter, h as FeedbackReplayResult, i as FeedbackSeverity, j as FeedbackSplitPolicy, k as FeedbackTask, l as FeedbackTrajectory, m as FeedbackTrajectoryFilter, n as FeedbackTrajectoryStore, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-c43WGtTX.js';
-import './run-record-DNiOMBrZ.js';
-import './integrity-Cr5YodSY.js';
-import './store-u47QaJ9G.js';
-import './emitter-B2XqDKFU.js';
-import './dataset-B9qvlm_o.js';
+export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './researcher-CUOiGcGv.js';
+export { F as FeedbackArtifactType, a as FeedbackAttempt, b as FeedbackLabel, c as FeedbackLabelKind, d as FeedbackLabelSource, e as FeedbackOptimizerRow, f as FeedbackOutcome, g as FeedbackReplayAdapter, h as FeedbackReplayResult, i as FeedbackSeverity, j as FeedbackSplitPolicy, k as FeedbackTask, l as FeedbackTrajectory, m as FeedbackTrajectoryFilter, n as FeedbackTrajectoryStore, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-DfFdrraJ.js';
+export { A as ActionableSideInfo, a as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, G as GenerationReport, I as InMemoryTrialCache, M as MultiShotGateConfig, b as MultiShotGateResult, c as MultiShotMutateAdapter, d as MultiShotOptimizationConfig, e as MultiShotOptimizationResult, f as MultiShotRun, g as MultiShotRunInput, h as MultiShotRunner, i as MultiShotScore, j as MultiShotScorer, k as MultiShotSplit, l as MultiShotTrace, m as MultiShotTrialResult, n as MultiShotVariant, o as MutateAdapter, P as PromptEvolutionConfig, p as PromptEvolutionEvent, q as PromptEvolutionResult, R as ReflectionContext, r as ReflectionProposal, S as ScenarioAggregate, s as ScoreAdapter, T as TrialCache, t as TrialResult, u as TrialTrace, V as VariantAggregate, v as buildReflectionPrompt, w as defaultMultiShotObjectives, x as parseReflectionResponse, y as runMultiShotOptimization, z as runPromptEvolution, B as trialTraceFromMultiShotTrial } from './summary-report-BXGs_9V0.js';
+import './errors-BZ9sTdz7.js';
+import './integrity-DK2EBVZC.js';
+import './store-Db2Bv8Cf.js';
+import './run-record-CqzahIbx.js';
+import './emitter-DP_cSSiw.js';
+import './control-runtime-BuJHoLg0.js';
+import './dataset-CiK_3LDr.js';
+import './failure-cluster-C2EGSDiT.js';

package/dist/optimization.js

@@ -25,17 +25,19 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-VQQSPGSM.js";
-import "./chunk-QBW3YBTR.js";
+} from "./chunk-VRJVTXRV.js";
+import "./chunk-NLMNWKVM.js";
 import {
   runEvalCampaign
-} from "./chunk-EXGR4XEM.js";
-import "./chunk-KAO3Q65R.js";
-import "./chunk-IOXMGMHQ.js";
-import "./chunk-QUKKGHTZ.js";
-import "./chunk-SQQLHODJ.js";
-import "./chunk-5IIQKMD5.js";
-import "./chunk-6M774GY6.js";
+} from "./chunk-SESZDQPX.js";
+import "./chunk-4S4BM3QQ.js";
+import "./chunk-2A5XJB43.js";
+import "./chunk-I4MBDTY5.js";
+import "./chunk-KTGTIOFD.js";
+import "./chunk-PC4UYEBM.js";
+import "./chunk-TVVP3ZZQ.js";
+import "./chunk-4F5DQN55.js";
+import "./chunk-NG236HPC.js";
 import "./chunk-PZ5AY32C.js";
 export {
   CallbackResearcher,

package/dist/outcome-store-D6KWmYvj.d.ts

@@ -0,0 +1,63 @@
+/**
+ * OutcomeStore — deployment outcomes attached to Run IDs.
+ *
+ * Outcomes arrive asynchronously from production telemetry after the
+ * eval run completed: user ratings, retention flags, conversion events,
+ * revenue, support-ticket rate, anything a product team can measure.
+ * The store is a peer to TraceStore — separate lifecycle, same runId
+ * foreign key.
+ *
+ * The whole point of this module is to make the meta-eval correlation
+ * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
+ */
+interface DeploymentOutcome {
+    runId: string;
+    capturedAt: number;
+    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
+    metrics: Record<string, number>;
+    /** Dimensions for stratified analysis — cohort, region, user_segment. */
+    labels?: Record<string, string>;
+    /** Free-form provenance (source system, pipeline version). */
+    source?: string;
+}
+interface OutcomeFilter {
+    runIds?: string[];
+    since?: number;
+    until?: number;
+    label?: {
+        key: string;
+        value: string;
+    };
+    source?: string;
+}
+interface OutcomeStore {
+    append(outcome: DeploymentOutcome): Promise<void>;
+    /** All outcomes attached to this run (a single run can have many — multiple
+     *  capture windows over deployment time). */
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+declare class InMemoryOutcomeStore implements OutcomeStore {
+    private items;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+interface FileSystemOutcomeStoreOptions {
+    dir: string;
+    maxBytes?: number;
+}
+declare class FileSystemOutcomeStore implements OutcomeStore {
+    private dir;
+    private maxBytes;
+    private memo?;
+    private loaded;
+    constructor(options: FileSystemOutcomeStoreOptions);
+    private ensureDir;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    private load;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+
+export { type DeploymentOutcome as D, FileSystemOutcomeStore as F, InMemoryOutcomeStore as I, type OutcomeStore as O, type OutcomeFilter as a, type FileSystemOutcomeStoreOptions as b };

package/dist/pipelines/index.d.ts

@@ -0,0 +1,172 @@
+import { g as BudgetSpec, T as TraceStore, h as RunFilter, R as Run, a as ToolSpan } from '../store-Db2Bv8Cf.js';
+export { a as FailureCluster, F as FailureClusterReport, f as failureClusterView } from '../failure-cluster-C2EGSDiT.js';
+import { a as TrajectoryStep } from '../trajectory-CnoBo-JY.js';
+import { B as BaselineOptions, a as BaselineReport } from '../baseline-4R5deP0N.js';
+export { c as computeToolUseMetrics } from '../baseline-4R5deP0N.js';
+import { l as llmSpans } from '../query-DODUYdPg.js';
+
+/**
+ * BudgetBreachView — aggregates breach events across the corpus.
+ *
+ * Answers: which dimensions get hit most often? Which scenarios are
+ * underbudgeted? Which variants trigger the most breaches?
+ */
+
+interface BudgetBreachFinding {
+    runId: string;
+    scenarioId: string;
+    variantId?: string;
+    dimension: keyof BudgetSpec;
+    limit: number;
+    consumed: number;
+    excessRatio: number;
+    timestamp: number;
+}
+interface BudgetBreachReport {
+    findings: BudgetBreachFinding[];
+    byDimension: Record<string, number>;
+    byScenario: Record<string, number>;
+    byVariant: Record<string, number>;
+    totalRuns: number;
+    breachedRunRatio: number;
+}
+declare function budgetBreachView(store: TraceStore, options?: {
+    scenarioId?: string;
+    variantId?: string;
+}): Promise<BudgetBreachReport>;
+
+/**
+ * FirstDivergenceView — aligns two trajectories by step index, reports
+ * the first step where they differ.
+ *
+ * "Differ" is configurable — default is (kind, toolName if tool, model
+ * if llm). Use this view to attribute "why is variant B better?" to a
+ * specific step rather than an aggregate mean delta.
+ */
+
+interface DivergenceReport {
+    runA: string;
+    runB: string;
+    firstDivergenceIndex: number | null;
+    aStep?: TrajectoryStep;
+    bStep?: TrajectoryStep;
+    reason?: string;
+    /** Common prefix length (steps that matched). */
+    commonPrefixLen: number;
+}
+interface DivergenceOptions {
+    /** Returns true if two steps are considered equal. Default: kind + tool/model match. */
+    stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean;
+}
+declare function firstDivergenceView(store: TraceStore, runA: string, runB: string, options?: DivergenceOptions): Promise<DivergenceReport>;
+
+/**
+ * JudgeAgreementView — pairwise agreement between judges across the
+ * corpus, grouped by dimension.
+ *
+ * Output drives two workflows:
+ *   - Judge robustness audit: "does Claude agree with GPT at κ ≥ 0.6?"
+ *   - Calibration tracking: κ vs golden human labels over time (by
+ *     providing a `humanGoldenJudgeId`).
+ */
+
+interface JudgePair {
+    judgeA: string;
+    judgeB: string;
+    dimension: string;
+    /** Number of (targetSpanId, dimension) tuples both judges scored. */
+    commonItems: number;
+    pearson: number;
+    krippendorff: number;
+}
+interface JudgeAgreementReport {
+    pairs: JudgePair[];
+    dimensions: string[];
+    judgeIds: string[];
+}
+declare function judgeAgreementView(store: TraceStore): Promise<JudgeAgreementReport>;
+
+/**
+ * RegressionView — compares a candidate slice to a baseline slice on a
+ * named metric. Delegates the statistics (Welch's t-test, Cohen's d,
+ * IQR stability) to `baseline.ts`.
+ *
+ * This is the entry point for CI regression gates: "given runs tagged
+ * release=A and release=B, did any metric regress?"
+ */
+
+interface RegressionSpec {
+    metric: string;
+    higherIsBetter: boolean;
+    /** Extract a scalar from a run. Default extractors handle common metrics. */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface RegressionOptions extends BaselineOptions {
+    baseline: RunFilter;
+    candidate: RunFilter;
+}
+declare function regressionView(store: TraceStore, metrics: RegressionSpec[], options: RegressionOptions): Promise<BaselineReport>;
+
+/**
+ * StuckLoopView — detects when an agent calls the same tool with the
+ * same (or structurally similar) arguments ≥ N times in a short window.
+ *
+ * Rationale: agents that loop are the number-one production failure
+ * mode on long-horizon flows. The view returns (runId, toolName,
+ * argHash, occurrences, windowMs) for each detected loop plus a
+ * fraction of runs affected.
+ */
+
+interface StuckLoopFinding {
+    runId: string;
+    toolName: string;
+    argHash: string;
+    occurrences: number;
+    spanIds: string[];
+    /** Milliseconds between first and last call in the loop. */
+    windowMs: number;
+}
+interface StuckLoopReport {
+    findings: StuckLoopFinding[];
+    affectedRunRatio: number;
+    totalRuns: number;
+}
+interface StuckLoopOptions {
+    /** Minimum call count to flag a loop (default 3). */
+    minOccurrences?: number;
+    /** Filter to a specific runId; omit to scan the entire corpus. */
+    runId?: string;
+}
+declare function stuckLoopView(store: TraceStore, options?: StuckLoopOptions): Promise<StuckLoopReport>;
+
+/**
+ * ToolWasteView — fraction of tool calls whose results weren't used
+ * downstream. Without a "used" signal we fall back to structural
+ * proxies: error calls, duplicate calls, and tool calls followed by
+ * zero subsequent LLM spans are all considered waste.
+ *
+ * Consumers can pass a `usageOracle` that inspects a tool span and
+ * returns true iff the tool's result appears in a later LLM message,
+ * artifact, or state mutation — that's the canonical definition; the
+ * default heuristic is a reasonable fallback.
+ */
+
+interface ToolWasteFinding {
+    runId: string;
+    wastedCalls: number;
+    totalCalls: number;
+    wasteRate: number;
+}
+interface ToolWasteReport {
+    byRun: ToolWasteFinding[];
+    overallWasteRate: number;
+}
+interface ToolWasteOptions {
+    runId?: string;
+    usageOracle?: (tool: ToolSpan, later: {
+        llm: Awaited<ReturnType<typeof llmSpans>>;
+    }) => boolean;
+}
+declare function toolWasteView(store: TraceStore, options?: ToolWasteOptions): Promise<ToolWasteReport>;
+
+export { type BudgetBreachFinding, type BudgetBreachReport, type DivergenceOptions, type DivergenceReport, type JudgeAgreementReport, type JudgePair, type RegressionOptions, type RegressionSpec, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, budgetBreachView, firstDivergenceView, judgeAgreementView, regressionView, stuckLoopView, toolWasteView };