@tangle-network/agent-eval 0.21.0 → 0.23.0

package/dist/integrity-Cr5YodSY.d.ts

@@ -0,0 +1,210 @@
+import { T as TraceStore } from './store-u47QaJ9G.js';
+
+/**
+ * RawProviderSink — first-class persistence for the actual HTTP-level
+ * request/response bodies of every LLM provider call.
+ *
+ * Why this is a separate sink from the structured `LlmSpan`:
+ *
+ *   - `LlmSpan` records the *intent* — model name, messages, output text,
+ *     usage. It's what dashboards read; it's NOT enough for forensics.
+ *   - When a downstream consumer reports "the verifier used the wrong route"
+ *     or "tokens look right but reasoning was missing," the only way to
+ *     answer is the raw HTTP body. Span fields can lie (a proxy can echo
+ *     a different `model` value than what actually answered); the raw
+ *     response is ground truth.
+ *
+ * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
+ * matrix runner / BuilderSession sets it up automatically) and every
+ * request, response, and error is recorded — including retries, with the
+ * attempt index attached so a flaky call's full event chain is recoverable.
+ *
+ * Redaction is enforced at sink time. The default redactor strips
+ * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
+ * payload field whose key matches `apiKey | api_key | bearer | password |
+ * secret | token` (case-insensitive). Override via the sink constructor or
+ * the per-call `redactor`. The `redactedFields` array on the persisted
+ * event lets a reviewer see what was stripped without exposing the values.
+ */
+type RawProviderDirection = 'request' | 'response' | 'error';
+interface RawProviderEvent {
+    /** Stable id. Generated by the sink if omitted. */
+    eventId: string;
+    /** Trace context populated by `LlmClient` when the call is wrapped in a span. */
+    runId?: string;
+    spanId?: string;
+    /**
+     * Logical provider name. Free-form so callers can use whatever id matches
+     * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
+     * omitted, derived from `baseUrl` in `LlmClientOptions`.
+     */
+    provider: string;
+    model: string;
+    /** Endpoint path, e.g. `'/v1/chat/completions'`. */
+    endpoint: string;
+    /** Base URL used for the call (already-normalised — no trailing slash). */
+    baseUrl: string;
+    /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
+    attemptIndex: number;
+    direction: RawProviderDirection;
+    /** Unix ms. */
+    timestamp: number;
+    /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
+    durationMs?: number;
+    statusCode?: number;
+    requestHeaders?: Record<string, string>;
+    requestBody?: unknown;
+    responseHeaders?: Record<string, string>;
+    responseBody?: unknown;
+    /** Set on `direction: 'error'` events. */
+    errorMessage?: string;
+    /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
+    redactedFields: string[];
+}
+interface RawProviderSinkFilter {
+    runId?: string;
+    spanId?: string;
+    direction?: RawProviderDirection;
+    attemptIndex?: number;
+}
+interface RawProviderSink {
+    record(event: RawProviderEvent): Promise<void>;
+    /** Optional listing — implementations that durably persist (file, db) should support this. */
+    list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+    /** Optional teardown for backed implementations. */
+    close?(): Promise<void>;
+}
+type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
+/**
+ * Default redactor — strips well-known auth headers and any body field whose
+ * key matches the credential pattern. Records every redacted path on
+ * `event.redactedFields` so a downstream reviewer can see what was removed.
+ */
+declare function defaultProviderRedactor(event: RawProviderEvent): RawProviderEvent;
+interface InMemoryRawProviderSinkOptions {
+    redactor?: ProviderRedactor;
+}
+declare class InMemoryRawProviderSink implements RawProviderSink {
+    private events;
+    private redactor;
+    constructor(opts?: InMemoryRawProviderSinkOptions);
+    record(event: RawProviderEvent): Promise<void>;
+    list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+    size(): number;
+}
+declare class NoopRawProviderSink implements RawProviderSink {
+    record(): Promise<void>;
+    /**
+     * Returns an empty array. Implemented so `assertRunCaptured` does not
+     * trip the `no_raw_sink` issue when a caller explicitly opts out of
+     * capture by passing this sink — opt-out is a deliberate choice, not a
+     * misconfiguration.
+     */
+    list(): Promise<RawProviderEvent[]>;
+}
+interface FileSystemRawProviderSinkOptions {
+    /** Directory the NDJSON file is written into. Created if missing. */
+    dir: string;
+    /** File name; default `'raw-provider-events.ndjson'`. */
+    fileName?: string;
+    /** Bytes after which the writer rolls over to a new file (default 32 MiB). */
+    rollAtBytes?: number;
+    redactor?: ProviderRedactor;
+}
+declare class FileSystemRawProviderSink implements RawProviderSink {
+    private dir;
+    private fileName;
+    private rollAtBytes;
+    private redactor;
+    private bytesWritten;
+    private rollIndex;
+    private initPromise;
+    constructor(opts: FileSystemRawProviderSinkOptions);
+    private ensureInit;
+    private currentPath;
+    record(event: RawProviderEvent): Promise<void>;
+    list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+}
+/**
+ * Best-effort provider id from a base URL. Falls back to the URL host when
+ * none of the well-known patterns match.
+ */
+declare function providerFromBaseUrl(baseUrl: string): string;
+
+/**
+ * Run-completion integrity check — at end of run, verify the expected event
+ * types were actually captured. The point is the launch-review failure mode:
+ * a run *appears* successful but the raw provider events were never written,
+ * so a downstream reviewer can't reconstruct what happened.
+ *
+ * Pattern:
+ *
+ *   const report = await assertRunCaptured(store, runId, {
+ *     llmSpansMin: 1,
+ *     judgeSpansMin: 1,
+ *     rawSink: providerSink,                  // must have ≥ 1 event for this run
+ *     requireRawCoverageOfLlmSpans: true,     // every llm span has matching raw events
+ *   })
+ *   if (!report.ok) throwIfRunIncomplete(report)  // or mark run failed and continue
+ *
+ * The function is read-only on the store and returns a structured report;
+ * the caller chooses the failure mode (throw, mark run failed, log warning).
+ * `throwIfRunIncomplete` is the convenient strict mode.
+ */
+
+interface RunIntegrityExpectations {
+    /** Minimum LLM span count. Default 0 (no requirement). */
+    llmSpansMin?: number;
+    /** Minimum judge span count. Default 0. */
+    judgeSpansMin?: number;
+    /** Minimum tool span count. Default 0. */
+    toolSpansMin?: number;
+    /**
+     * Raw provider sink to consult for capture verification. When present,
+     * the check requires at least one raw event for the run.
+     */
+    rawSink?: RawProviderSink;
+    /** Minimum raw provider event count. Default 0; ignored when `rawSink` absent. */
+    rawProviderEventsMin?: number;
+    /**
+     * Every LLM span must have at least one matching raw `request` event
+     * (matched by spanId). Catches the common bug where the structured span
+     * was emitted but the raw HTTP capture was wired to a different sink.
+     */
+    requireRawCoverageOfLlmSpans?: boolean;
+    /** Run outcome must be set (not null/undefined). Default false. */
+    requireOutcome?: boolean;
+}
+type RunIntegrityIssueCode = 'no_run' | 'missing_llm_spans' | 'missing_judge_spans' | 'missing_tool_spans' | 'missing_raw_events' | 'no_raw_sink' | 'orphan_llm_span' | 'missing_outcome';
+interface RunIntegrityIssue {
+    code: RunIntegrityIssueCode;
+    message: string;
+    detail?: Record<string, unknown>;
+}
+interface RunIntegrityReport {
+    ok: boolean;
+    runId: string;
+    llmSpanCount: number;
+    judgeSpanCount: number;
+    toolSpanCount: number;
+    rawProviderEventCount: number;
+    /**
+     * Coverage of LLM spans by raw provider events keyed on spanId.
+     * `total` is the number of LLM spans; `covered` is the count with at
+     * least one matching `request` raw event.
+     */
+    rawSpanCoverage: {
+        covered: number;
+        total: number;
+    };
+    issues: RunIntegrityIssue[];
+}
+declare class RunIntegrityError extends Error {
+    readonly report: RunIntegrityReport;
+    constructor(report: RunIntegrityReport);
+}
+declare function assertRunCaptured(store: TraceStore, runId: string, expectations?: RunIntegrityExpectations): Promise<RunIntegrityReport>;
+/** Strict mode: throws `RunIntegrityError` when the report isn't ok. */
+declare function throwIfRunIncomplete(report: RunIntegrityReport): void;
+
+export { FileSystemRawProviderSink as F, InMemoryRawProviderSink as I, NoopRawProviderSink as N, type ProviderRedactor as P, type RawProviderSink as R, type RunIntegrityExpectations as a, type RunIntegrityReport as b, type FileSystemRawProviderSinkOptions as c, type InMemoryRawProviderSinkOptions as d, type RawProviderDirection as e, type RawProviderEvent as f, type RawProviderSinkFilter as g, RunIntegrityError as h, type RunIntegrityIssue as i, type RunIntegrityIssueCode as j, assertRunCaptured as k, defaultProviderRedactor as l, providerFromBaseUrl as p, throwIfRunIncomplete as t };

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.21.0",
+    "version": "0.23.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/optimization.d.ts

@@ -1,146 +1,8 @@
-import { G as GateDecision } from './multi-shot-optimization-Bvtz294B.js';
-export { A as ActionableSideInfo, b as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, e as GenerationReport, I as InMemoryTrialCache, h as MultiShotGateConfig, i as MultiShotGateResult, j as MultiShotMutateAdapter, k as MultiShotOptimizationConfig, l as MultiShotOptimizationResult, m as MultiShotRun, n as MultiShotRunInput, o as MultiShotRunner, p as MultiShotScore, q as MultiShotScorer, r as MultiShotSplit, s as MultiShotTrace, t as MultiShotTrialResult, u as MultiShotVariant, M as MutateAdapter, v as PromptEvolutionConfig, w as PromptEvolutionEvent, x as PromptEvolutionResult, R as ReflectionContext, y as ReflectionProposal, S as ScenarioAggregate, z as ScoreAdapter, T as TrialCache, a as TrialResult, B as TrialTrace, V as VariantAggregate, C as buildReflectionPrompt, J as defaultMultiShotObjectives, Q as parseReflectionResponse, U as runMultiShotOptimization, W as runPromptEvolution, Y as trialTraceFromMultiShotTrial } from './multi-shot-optimization-Bvtz294B.js';
-import { a as RunRecord } from './run-record-CX_jcAyr.js';
-export { n as FeedbackArtifactType, o as FeedbackAttempt, F as FeedbackLabel, p as FeedbackLabelKind, q as FeedbackLabelSource, r as FeedbackOptimizerRow, s as FeedbackOutcome, t as FeedbackReplayAdapter, u as FeedbackReplayResult, v as FeedbackSeverity, w as FeedbackSplitPolicy, x as FeedbackTask, b as FeedbackTrajectory, y as FeedbackTrajectoryFilter, a as FeedbackTrajectoryStore, z as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, A as ProposedSideEffect, D as assignFeedbackSplit, E as controlRunToFeedbackTrajectory, G as createFeedbackTrajectory, H as feedbackTrajectoriesToDatasetScenarios, J as feedbackTrajectoriesToOptimizerRows, K as feedbackTrajectoryToDatasetScenario, L as feedbackTrajectoryToOptimizerRow, N as parseFeedbackTrajectoriesJsonl, O as renderPreferenceMemoryMarkdown, Q as replayFeedbackTrajectories, R as replayFeedbackTrajectory, U as serializeFeedbackTrajectoriesJsonl, Y as summarizePreferenceMemory, Z as withAssignedFeedbackSplit } from './feedback-trajectory-CB0A32o3.js';
-import './dataset-B9qvlm_o.js';
-import './emitter-B2XqDKFU.js';
+export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './eval-campaign-Ds5QljIh.js';
+export { A as ActionableSideInfo, a as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, G as GenerationReport, I as InMemoryTrialCache, M as MultiShotGateConfig, b as MultiShotGateResult, c as MultiShotMutateAdapter, d as MultiShotOptimizationConfig, e as MultiShotOptimizationResult, f as MultiShotRun, g as MultiShotRunInput, h as MultiShotRunner, i as MultiShotScore, j as MultiShotScorer, k as MultiShotSplit, l as MultiShotTrace, m as MultiShotTrialResult, n as MultiShotVariant, o as MutateAdapter, P as PromptEvolutionConfig, p as PromptEvolutionEvent, q as PromptEvolutionResult, R as ReflectionContext, r as ReflectionProposal, S as ScenarioAggregate, s as ScoreAdapter, T as TrialCache, t as TrialResult, u as TrialTrace, V as VariantAggregate, v as buildReflectionPrompt, w as defaultMultiShotObjectives, x as parseReflectionResponse, y as runMultiShotOptimization, z as runPromptEvolution, B as trialTraceFromMultiShotTrial } from './summary-report-Ce1r4EYo.js';
+export { F as FeedbackArtifactType, a as FeedbackAttempt, b as FeedbackLabel, c as FeedbackLabelKind, d as FeedbackLabelSource, e as FeedbackOptimizerRow, f as FeedbackOutcome, g as FeedbackReplayAdapter, h as FeedbackReplayResult, i as FeedbackSeverity, j as FeedbackSplitPolicy, k as FeedbackTask, l as FeedbackTrajectory, m as FeedbackTrajectoryFilter, n as FeedbackTrajectoryStore, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-c43WGtTX.js';
+import './run-record-DNiOMBrZ.js';
+import './integrity-Cr5YodSY.js';
 import './store-u47QaJ9G.js';
-
-/**
- * Researcher interface — stable hook for an external autonomous-research
- * agent to drive the meta-loop.
- *
- * Implementations live downstream (typically in a private repo that
- * runs the actual LLM). This package ships only the contract + a
- * `NoopResearcher` so consumers can wire the surface without being
- * forced to implement every method up front.
- *
- * The four methods mirror the four stages of the paper "Two Loops,
- * Three Roles":
- *
- *   inspectFailures   — given the observed runs, what failure modes
- *                       are present? (data → diagnosis)
- *   proposeChange     — given diagnosed failure modes, what
- *                       structural changes should we try?
- *                       (diagnosis → plan delta)
- *   applyChange       — fold the proposed deltas into a concrete
- *                       experiment plan against an existing baseline.
- *                       (plan delta → executable plan)
- *   evaluateChange    — run the plan, return runs + the gate verdict.
- *                       (executable plan → verdict)
- *
- * Composition is the discipline: a Researcher implementation MUST
- * keep these four steps separate and inspectable. Conflating
- * "diagnose + propose + run" into a single LLM call defeats the
- * point of the framework — you can't audit which step lied.
- *
- * THIS INTERFACE IS STABLE. Breaking changes require a new module
- * (e.g. `Researcher2`) so existing implementations keep working.
- */
-
-/** A diagnosed failure mode with the run-IDs that exhibit it. */
-interface FailureMode {
-    /** Short machine-readable code. Must be stable across runs of the
-     *  same researcher to enable longitudinal tracking. */
-    code: string;
-    /** Human-readable description for the paper / dashboard. */
-    description: string;
-    evidence: {
-        /** Run IDs (from `RunRecord.runId`) where this failure mode was
-         *  observed. */
-        runIds: string[];
-        /** Number of run samples that informed the diagnosis. */
-        samples: number;
-    };
-}
-/** A single steering change the researcher wants to try. */
-interface SteeringChange {
-    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
-    /** Implementation-specific payload. Researcher implementations
-     *  define the schema — keep this `unknown` here to avoid coupling
-     *  the public interface to any one researcher's internal model. */
-    payload: unknown;
-    /** Why the researcher proposed this change. Goes into the audit
-     *  trail next to the failure-mode evidence. */
-    rationale: string;
-    /** Optional self-reported expected delta on the headline metric. */
-    expectedDelta?: number;
-}
-/** A single experiment plan, mapped onto the search/holdout splits. */
-interface ExperimentPlan {
-    baselineCandidateId: string;
-    proposedCandidateId: string;
-    changes: SteeringChange[];
-    /** USD ceiling for the entire experiment. The runner must stop
-     *  before exceeding this and report a partial result. */
-    evaluationBudgetUsd: number;
-    /** Item IDs (your dataset keys) for the search vs holdout splits. */
-    splits: {
-        search: string[];
-        holdout: string[];
-    };
-}
-/** Result of running a plan: every run, plus the gate verdict. */
-interface ExperimentResult {
-    plan: ExperimentPlan;
-    runs: RunRecord[];
-    gateDecision: GateDecision;
-}
-/**
- * The researcher loop. Stable, four-step, inspectable.
- *
- *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
- *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
- *   └──────────┘                   └──────────┘                └────┬─────┘
- *                                                                   │
- *                                                                   ▼
- *                              ┌────────────────┐  applyChange ┌────────┐
- *                              │ ExperimentPlan │ ◀────────────│  base  │
- *                              └────────┬───────┘              └────────┘
- *                                       │
- *                       evaluateChange  ▼
- *                              ┌────────────────┐
- *                              │ ExperimentResult│
- *                              └────────────────┘
- */
-interface Researcher {
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-interface CallbackResearcherOptions {
-    inspectFailures: Researcher['inspectFailures'];
-    proposeChange: Researcher['proposeChange'];
-    applyChange: Researcher['applyChange'];
-    evaluateChange: Researcher['evaluateChange'];
-}
-/**
- * Minimal concrete researcher for tests, scripts, and small integrations.
- * Larger autonomous researchers can still implement `Researcher` directly.
- */
-declare class CallbackResearcher implements Researcher {
-    private readonly callbacks;
-    constructor(callbacks: CallbackResearcherOptions);
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-/**
- * No-op researcher — fails loud on every method. Use as a placeholder
- * in code paths that wire the interface but don't have an implementation
- * yet. Importantly, this does NOT silently succeed: a no-op researcher
- * that returned empty arrays would muffle the loop's signal that
- * nobody implemented the brain.
- */
-declare class NoopResearcher implements Researcher {
-    private readonly hint;
-    constructor(hint?: string);
-    inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(_changes: SteeringChange[], _baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-
-export { CallbackResearcher, type CallbackResearcherOptions, type ExperimentPlan, type ExperimentResult, type FailureMode, NoopResearcher, type Researcher, type SteeringChange };
+import './emitter-B2XqDKFU.js';
+import './dataset-B9qvlm_o.js';

package/dist/optimization.js

@@ -25,9 +25,17 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-HRZELXCR.js";
-import "./chunk-YUFXO3TU.js";
-import "./chunk-KRR4VMH7.js";
+} from "./chunk-VQQSPGSM.js";
+import "./chunk-QBW3YBTR.js";
+import {
+  runEvalCampaign
+} from "./chunk-EXGR4XEM.js";
+import "./chunk-KAO3Q65R.js";
+import "./chunk-IOXMGMHQ.js";
+import "./chunk-QUKKGHTZ.js";
+import "./chunk-SQQLHODJ.js";
+import "./chunk-5IIQKMD5.js";
+import "./chunk-6M774GY6.js";
 import "./chunk-PZ5AY32C.js";
 export {
   CallbackResearcher,
@@ -50,6 +58,7 @@ export {
   renderPreferenceMemoryMarkdown,
   replayFeedbackTrajectories,
   replayFeedbackTrajectory,
+  runEvalCampaign,
   runMultiShotOptimization,
   runPromptEvolution,
   serializeFeedbackTrajectoriesJsonl,