@tangle-network/agent-eval 0.23.1 → 0.24.0

package/dist/{eval-campaign-Ds5QljIh.d.ts → researcher-CUOiGcGv.d.ts}

@@ -1,8 +1,9 @@
-import { C as GateDecision, F as ResearchReportOptions, H as ResearchReport } from './summary-report-Ce1r4EYo.js';
-import { R as RunRecord, a as RunSplitTag, b as RunTokenUsage, c as RunJudgeMetadata } from './run-record-DNiOMBrZ.js';
-import { R as RawProviderSink, P as ProviderRedactor, a as RunIntegrityExpectations, b as RunIntegrityReport } from './integrity-Cr5YodSY.js';
-import { T as TraceEmitter, R as RunCompleteHook } from './emitter-B2XqDKFU.js';
-import { T as TraceStore } from './store-u47QaJ9G.js';
+import { A as AgentEvalError, C as CaptureIntegrityError } from './errors-BZ9sTdz7.js';
+import { R as RawProviderSink, P as ProviderRedactor, a as RunIntegrityExpectations, b as RunIntegrityReport } from './integrity-DK2EBVZC.js';
+import { a as RunSplitTag, b as RunTokenUsage, c as RunJudgeMetadata, R as RunRecord } from './run-record-CqzahIbx.js';
+import { W as ResearchReportOptions, N as ResearchReport, a3 as GateDecision } from './summary-report-BXGs_9V0.js';
+import { T as TraceEmitter, R as RunCompleteHook } from './emitter-DP_cSSiw.js';
+import { T as TraceStore } from './store-Db2Bv8Cf.js';
 
 /**
  * LLM client with graceful degrade.
@@ -81,7 +82,7 @@ interface LlmCallResult {
     /** Raw response body. */
     raw: Record<string, unknown>;
 }
-declare class LlmCallError extends Error {
+declare class LlmCallError extends AgentEvalError {
     readonly status: number;
     readonly body: string;
     readonly model: string;
@@ -147,10 +148,11 @@ declare function callLlmJson<T = unknown>(req: LlmCallRequest, opts?: LlmClientO
     value: T;
     result: LlmCallResult;
 }>;
-declare class LlmRouteAssertionError extends Error {
-    readonly code: 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider';
+type LlmRouteAssertionReason = 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider';
+declare class LlmRouteAssertionError extends CaptureIntegrityError {
+    readonly reason: LlmRouteAssertionReason;
     readonly baseUrl: string;
-    constructor(message: string, code: 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider', baseUrl: string);
+    constructor(message: string, reason: LlmRouteAssertionReason, baseUrl: string);
 }
 interface LlmRouteRequirements {
     /**
@@ -218,143 +220,6 @@ declare class LlmClient {
     }>;
 }
 
-/**
- * Researcher interface — stable hook for an external autonomous-research
- * agent to drive the meta-loop.
- *
- * Implementations live downstream (typically in a private repo that
- * runs the actual LLM). This package ships only the contract + a
- * `NoopResearcher` so consumers can wire the surface without being
- * forced to implement every method up front.
- *
- * The four methods mirror the four stages of the paper "Two Loops,
- * Three Roles":
- *
- *   inspectFailures   — given the observed runs, what failure modes
- *                       are present? (data → diagnosis)
- *   proposeChange     — given diagnosed failure modes, what
- *                       structural changes should we try?
- *                       (diagnosis → plan delta)
- *   applyChange       — fold the proposed deltas into a concrete
- *                       experiment plan against an existing baseline.
- *                       (plan delta → executable plan)
- *   evaluateChange    — run the plan, return runs + the gate verdict.
- *                       (executable plan → verdict)
- *
- * Composition is the discipline: a Researcher implementation MUST
- * keep these four steps separate and inspectable. Conflating
- * "diagnose + propose + run" into a single LLM call defeats the
- * point of the framework — you can't audit which step lied.
- *
- * THIS INTERFACE IS STABLE. Breaking changes require a new module
- * (e.g. `Researcher2`) so existing implementations keep working.
- */
-
-/** A diagnosed failure mode with the run-IDs that exhibit it. */
-interface FailureMode {
-    /** Short machine-readable code. Must be stable across runs of the
-     *  same researcher to enable longitudinal tracking. */
-    code: string;
-    /** Human-readable description for the paper / dashboard. */
-    description: string;
-    evidence: {
-        /** Run IDs (from `RunRecord.runId`) where this failure mode was
-         *  observed. */
-        runIds: string[];
-        /** Number of run samples that informed the diagnosis. */
-        samples: number;
-    };
-}
-/** A single steering change the researcher wants to try. */
-interface SteeringChange {
-    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
-    /** Implementation-specific payload. Researcher implementations
-     *  define the schema — keep this `unknown` here to avoid coupling
-     *  the public interface to any one researcher's internal model. */
-    payload: unknown;
-    /** Why the researcher proposed this change. Goes into the audit
-     *  trail next to the failure-mode evidence. */
-    rationale: string;
-    /** Optional self-reported expected delta on the headline metric. */
-    expectedDelta?: number;
-}
-/** A single experiment plan, mapped onto the search/holdout splits. */
-interface ExperimentPlan {
-    baselineCandidateId: string;
-    proposedCandidateId: string;
-    changes: SteeringChange[];
-    /** USD ceiling for the entire experiment. The runner must stop
-     *  before exceeding this and report a partial result. */
-    evaluationBudgetUsd: number;
-    /** Item IDs (your dataset keys) for the search vs holdout splits. */
-    splits: {
-        search: string[];
-        holdout: string[];
-    };
-}
-/** Result of running a plan: every run, plus the gate verdict. */
-interface ExperimentResult {
-    plan: ExperimentPlan;
-    runs: RunRecord[];
-    gateDecision: GateDecision;
-}
-/**
- * The researcher loop. Stable, four-step, inspectable.
- *
- *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
- *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
- *   └──────────┘                   └──────────┘                └────┬─────┘
- *                                                                   │
- *                                                                   ▼
- *                              ┌────────────────┐  applyChange ┌────────┐
- *                              │ ExperimentPlan │ ◀────────────│  base  │
- *                              └────────┬───────┘              └────────┘
- *                                       │
- *                       evaluateChange  ▼
- *                              ┌────────────────┐
- *                              │ ExperimentResult│
- *                              └────────────────┘
- */
-interface Researcher {
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-interface CallbackResearcherOptions {
-    inspectFailures: Researcher['inspectFailures'];
-    proposeChange: Researcher['proposeChange'];
-    applyChange: Researcher['applyChange'];
-    evaluateChange: Researcher['evaluateChange'];
-}
-/**
- * Minimal concrete researcher for tests, scripts, and small integrations.
- * Larger autonomous researchers can still implement `Researcher` directly.
- */
-declare class CallbackResearcher implements Researcher {
-    private readonly callbacks;
-    constructor(callbacks: CallbackResearcherOptions);
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-/**
- * No-op researcher — fails loud on every method. Use as a placeholder
- * in code paths that wire the interface but don't have an implementation
- * yet. Importantly, this does NOT silently succeed: a no-op researcher
- * that returned empty arrays would muffle the loop's signal that
- * nobody implemented the brain.
- */
-declare class NoopResearcher implements Researcher {
-    private readonly hint;
-    constructor(hint?: string);
-    inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(_changes: SteeringChange[], _baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-
 /**
  * EvalCampaign — opinionated matrix runner that wires the four
  * capture-integrity directives by construction.
@@ -570,4 +435,141 @@ interface EvalCampaignResult {
 }
 declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
 
+/**
+ * Researcher interface — stable hook for an external autonomous-research
+ * agent to drive the meta-loop.
+ *
+ * Implementations live downstream (typically in a private repo that
+ * runs the actual LLM). This package ships only the contract + a
+ * `NoopResearcher` so consumers can wire the surface without being
+ * forced to implement every method up front.
+ *
+ * The four methods mirror the four stages of the paper "Two Loops,
+ * Three Roles":
+ *
+ *   inspectFailures   — given the observed runs, what failure modes
+ *                       are present? (data → diagnosis)
+ *   proposeChange     — given diagnosed failure modes, what
+ *                       structural changes should we try?
+ *                       (diagnosis → plan delta)
+ *   applyChange       — fold the proposed deltas into a concrete
+ *                       experiment plan against an existing baseline.
+ *                       (plan delta → executable plan)
+ *   evaluateChange    — run the plan, return runs + the gate verdict.
+ *                       (executable plan → verdict)
+ *
+ * Composition is the discipline: a Researcher implementation MUST
+ * keep these four steps separate and inspectable. Conflating
+ * "diagnose + propose + run" into a single LLM call defeats the
+ * point of the framework — you can't audit which step lied.
+ *
+ * THIS INTERFACE IS STABLE. Breaking changes require a new module
+ * (e.g. `Researcher2`) so existing implementations keep working.
+ */
+
+/** A diagnosed failure mode with the run-IDs that exhibit it. */
+interface FailureMode {
+    /** Short machine-readable code. Must be stable across runs of the
+     *  same researcher to enable longitudinal tracking. */
+    code: string;
+    /** Human-readable description for the paper / dashboard. */
+    description: string;
+    evidence: {
+        /** Run IDs (from `RunRecord.runId`) where this failure mode was
+         *  observed. */
+        runIds: string[];
+        /** Number of run samples that informed the diagnosis. */
+        samples: number;
+    };
+}
+/** A single steering change the researcher wants to try. */
+interface SteeringChange {
+    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
+    /** Implementation-specific payload. Researcher implementations
+     *  define the schema — keep this `unknown` here to avoid coupling
+     *  the public interface to any one researcher's internal model. */
+    payload: unknown;
+    /** Why the researcher proposed this change. Goes into the audit
+     *  trail next to the failure-mode evidence. */
+    rationale: string;
+    /** Optional self-reported expected delta on the headline metric. */
+    expectedDelta?: number;
+}
+/** A single experiment plan, mapped onto the search/holdout splits. */
+interface ExperimentPlan {
+    baselineCandidateId: string;
+    proposedCandidateId: string;
+    changes: SteeringChange[];
+    /** USD ceiling for the entire experiment. The runner must stop
+     *  before exceeding this and report a partial result. */
+    evaluationBudgetUsd: number;
+    /** Item IDs (your dataset keys) for the search vs holdout splits. */
+    splits: {
+        search: string[];
+        holdout: string[];
+    };
+}
+/** Result of running a plan: every run, plus the gate verdict. */
+interface ExperimentResult {
+    plan: ExperimentPlan;
+    runs: RunRecord[];
+    gateDecision: GateDecision;
+}
+/**
+ * The researcher loop. Stable, four-step, inspectable.
+ *
+ *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
+ *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
+ *   └──────────┘                   └──────────┘                └────┬─────┘
+ *                                                                   │
+ *                                                                   ▼
+ *                              ┌────────────────┐  applyChange ┌────────┐
+ *                              │ ExperimentPlan │ ◀────────────│  base  │
+ *                              └────────┬───────┘              └────────┘
+ *                                       │
+ *                       evaluateChange  ▼
+ *                              ┌────────────────┐
+ *                              │ ExperimentResult│
+ *                              └────────────────┘
+ */
+interface Researcher {
+    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+interface CallbackResearcherOptions {
+    inspectFailures: Researcher['inspectFailures'];
+    proposeChange: Researcher['proposeChange'];
+    applyChange: Researcher['applyChange'];
+    evaluateChange: Researcher['evaluateChange'];
+}
+/**
+ * Minimal concrete researcher for tests, scripts, and small integrations.
+ * Larger autonomous researchers can still implement `Researcher` directly.
+ */
+declare class CallbackResearcher implements Researcher {
+    private readonly callbacks;
+    constructor(callbacks: CallbackResearcherOptions);
+    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+/**
+ * No-op researcher — fails loud on every method. Use as a placeholder
+ * in code paths that wire the interface but don't have an implementation
+ * yet. Importantly, this does NOT silently succeed: a no-op researcher
+ * that returned empty arrays would muffle the loop's signal that
+ * nobody implemented the brain.
+ */
+declare class NoopResearcher implements Researcher {
+    private readonly hint;
+    constructor(hint?: string);
+    inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(_changes: SteeringChange[], _baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+
 export { CallbackResearcher as C, type EvalCampaignOptions as E, type FailedRun as F, type LlmClientOptions as L, NoopResearcher as N, type Researcher as R, type SteeringChange as S, type CallbackResearcherOptions as a, type CampaignFactoryParams as b, type CampaignIntegrityPolicy as c, type CampaignRunContext as d, type CampaignRunOutcome as e, type CampaignRunner as f, type CampaignScenario as g, type CampaignVariant as h, type EvalCampaignResult as i, type ExperimentPlan as j, type ExperimentResult as k, type FailureMode as l, LlmCallError as m, type LlmCallRequest as n, type LlmCallResult as o, LlmClient as p, type LlmMessage as q, runEvalCampaign as r, LlmRouteAssertionError as s, type LlmRouteRequirements as t, type LlmUsage as u, assertLlmRoute as v, callLlm as w, callLlmJson as x, probeLlm as y, stripFencedJson as z };