@tangle-network/agent-eval 0.117.0 → 0.118.0

package/dist/researcher-C8XyxQsu.d.ts

@@ -1,387 +0,0 @@
-import { a as RunSplitTag, c as RunTokenUsage, d as RunJudgeMetadata, J as JudgeScoresRecord, A as AgentProfileCell, e as AgentProfileCellInput, R as RunRecord } from './run-record-BDH49H2E.js';
-import { a as LlmClientOptions, b as LlmRouteRequirements } from './llm-client-qoDd18Qz.js';
-import { h as ResearchReportOptions, d as ResearchReport, m as GateDecision } from './summary-report-C5bKFfm-.js';
-import { T as TraceEmitter, R as RunCompleteHook } from './emitter-CjD7vUwv.js';
-import { R as RunIntegrityExpectations, a as RunIntegrityReport } from './integrity-DqlBiLyK.js';
-import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
-import { F as FailureClass } from './schema-B3Q3l9Z_.js';
-import { T as TraceStore } from './store-DGqD0Pyo.js';
-
-/**
- * EvalCampaign — opinionated matrix runner that wires the four
- * capture-integrity directives by construction.
- *
- * The canonical benchmark shape — matrix runner → for each
- * (variant, scenario, seed) → start a TraceEmitter → call LLMs → end the
- * run → analyze — has a bug class at the integration boundary: raw
- * events not captured, route silently wrong, integrity not asserted,
- * analyst never run. The directives in `SKILL.md § Capture integrity`
- * are the mitigations.
- *
- * `EvalCampaign` is the structural fix — consumers don't wire the
- * integrity surface themselves; the campaign owns it. Specifically:
- *
- *   - calls `assertLlmRoute` once at preflight before any work runs
- *   - constructs a per-run `TraceStore` and `RawProviderSink` via factories
- *   - constructs the `TraceEmitter` with `onRunComplete: [analyst hook]`
- *   - hands the runner an `LlmClientOptions` pre-wired with the sink and
- *     trace context — the runner can't accidentally call an LLM without
- *     capturing the raw HTTP envelope
- *   - calls `assertRunCaptured` after every `endRun` and routes failures
- *     through a configurable policy (`throw` / `mark_failed` / `log`)
- *   - assembles per-run `RunRecord`s and runs `researchReport` at the end
- *     so the campaign artifact is launch-decision-grade by default
- *   - embeds the campaign fingerprint (a SHA-256 over the canonicalised
- *     run set) and optional `preregistrationHash` in the report
- *
- * The runner contract is intentionally narrow: produce a `CampaignRunOutcome`
- * given a fully-wired `CampaignRunContext`. Everything orchestration-shaped
- * lives in the campaign. This is the inversion-of-control point — consumers
- * stop writing matrix runners and start writing scenario-runners.
- *
- * Out of scope for v1 (tracked in `docs/research-report-methodology.md`):
- *
- *   - Distributed/cluster execution (concurrency is local async)
- *   - Adaptive sampling / sequential interim looks
- *   - Resume from partial state across crashes
- *   - LLM-call retry beyond what `LlmClient` already does
- */
-
-interface CampaignVariant<V> {
-    id: string;
-    payload: V;
-}
-interface CampaignScenario {
-    scenarioId: string;
-    /** Free-form metadata propagated to runs and reports. */
-    tags?: Record<string, string>;
-}
-interface CampaignRunContext<V> {
-    /** Stable run id. The campaign generates this; the runner does not. */
-    runId: string;
-    /** Logical experiment id (campaignId by default; overridable per-run via opts). */
-    experimentId: string;
-    variant: V;
-    variantId: string;
-    scenarioId: string;
-    scenarioTags: Record<string, string>;
-    seed: number;
-    splitTag: RunSplitTag;
-    /**
-     * The TraceEmitter for this run, with `onRunComplete` hooks pre-wired
-     * (analyst auto-execution if configured, plus integrity check). The
-     * runner MUST call `emitter.startRun` before doing any work and either
-     * `emitter.endRun` or `emitter.abortRun` before returning.
-     */
-    emitter: TraceEmitter;
-    store: TraceStore;
-    rawSink: RawProviderSink;
-    /**
-     * Pre-wired LLM client options — `rawSink` and `traceContext` are populated
-     * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The
-     * runner can spread additional fields if needed.
-     */
-    llmOpts: LlmClientOptions;
-}
-interface CampaignRunOutcome {
-    /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */
-    pass: boolean;
-    /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
-    score: number;
-    /** Mandatory cost in USD. Use 0 + raw.cost_unknown=1 only if truly unknown. */
-    costUsd: number;
-    tokenUsage: RunTokenUsage;
-    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
-    model: string;
-    /** sha256 of the effective prompt sent to the model. */
-    promptHash: string;
-    /** sha256 of the effective config (model, temperature, tools, judges, splits). */
-    configHash: string;
-    /** Optional extra numeric metrics to land in `outcome.raw`. */
-    raw?: Record<string, number>;
-    /** Canonical cross-agent failure class from the shared `FAILURE_CLASSES`
-     *  taxonomy. Propagated to `RunRecord.failureClass` so campaign runs
-     *  aggregate failures in the same vocabulary as every other producer. */
-    failureClass?: FailureClass;
-    /** Optional free-form failure detail, scoped under `failureClass`. */
-    failureMode?: string;
-    /** Optional judge metadata when a judge was used. */
-    judgeMetadata?: RunJudgeMetadata;
-    /**
-     * Optional per-judge / per-dim breakdown for ensemble-judged runs.
-     * Propagated to `outcome.judgeScores` on the resulting `RunRecord`.
-     * Single-judge or scalar-only runs leave this unset.
-     */
-    judgeScores?: JudgeScoresRecord;
-    /**
-     * Agent profile cell observed by the runner. When supplied, it overrides
-     * `EvalCampaignOptions.agentProfile` for this run and must match the
-     * outcome's `model` and `promptHash`.
-     */
-    agentProfile?: AgentProfileCell | AgentProfileCellInput;
-}
-type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
-type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
-interface EvalCampaignOptions<V> {
-    /**
-     * Stable id for the campaign. Used as the default `experimentId` on
-     * every run, and folded into the campaign fingerprint.
-     */
-    campaignId: string;
-    variants: CampaignVariant<V>[];
-    scenarios: CampaignScenario[];
-    /** Default `[0, 1, 2]`. */
-    seeds?: number[];
-    /** Default `'holdout'` — the split that anchors a launch decision. */
-    splitTag?: RunSplitTag;
-    /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */
-    commitSha: string;
-    /**
-     * LLM client config. Augmented per-run with `rawSink` and `traceContext`
-     * before being passed to the runner. The campaign asserts this config
-     * matches `routeRequirements` once at preflight.
-     */
-    llmOpts: LlmClientOptions;
-    /**
-     * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail
-     * loud if the campaign would silently fall back to the public router or
-     * run unauthenticated. Override with an empty object to disable.
-     */
-    routeRequirements?: LlmRouteRequirements;
-    /**
-     * Per-run TraceStore factory. Common shape: a fresh store per run keyed
-     * on `runId`. Implementations that share a store across the campaign
-     * are valid — the campaign only writes through `emitter`.
-     */
-    storeFactory: (params: CampaignFactoryParams) => TraceStore;
-    /**
-     * Per-run RawProviderSink factory. Defaults to `FileSystemRawProviderSink`
-     * rooted at `${workDir}/raw-events/${runId}` if `workDir` is supplied;
-     * otherwise required. Forensic capture is non-negotiable in a campaign
-     * run — pass `NoopRawProviderSink` explicitly if you want to opt out.
-     */
-    rawSinkFactory?: (params: CampaignFactoryParams) => RawProviderSink;
-    /**
-     * Filesystem root for default `rawSinkFactory`. Ignored if
-     * `rawSinkFactory` is supplied.
-     */
-    workDir?: string;
-    /**
-     * Extra `onRunComplete` hooks the campaign appends (after its own
-     * integrity-check hook). Pass `traceAnalystOnRunComplete(...)` here.
-     */
-    onRunComplete?: RunCompleteHook[];
-    /**
-     * Per-run integrity expectations. Defaults to:
-     *   `{ llmSpansMin: 1, requireRawCoverageOfLlmSpans: true, requireOutcome: true }`.
-     * Override (e.g. `{ llmSpansMin: 0 }`) for runs that don't call LLMs.
-     */
-    integrity?: RunIntegrityExpectations;
-    /** Behaviour when integrity fails. Default `'mark_failed'`. */
-    onIntegrityFailure?: CampaignIntegrityPolicy;
-    /**
-     * Per-run runner. Receives a fully-wired context; produces an outcome
-     * the campaign converts into a `RunRecord`.
-     */
-    runner: CampaignRunner<V>;
-    /**
-     * If set, the campaign computes `researchReport` at the end. `comparator`
-     * is a `variantId`. Other fields are forwarded verbatim.
-     */
-    report?: {
-        comparator?: string;
-    } & Omit<ResearchReportOptions, 'comparator' | 'preregistrationHash' | 'generatedAt'>;
-    /**
-     * Hash of a signed `HypothesisManifest` (see `pre-registration.ts`).
-     * Embedded in the campaign fingerprint and the research report.
-     */
-    preregistrationHash?: string;
-    /** Local concurrency. Default `1` (sequential). */
-    concurrency?: number;
-    /**
-     * Override the time source. Tests pass a mock to make wallMs deterministic.
-     */
-    now?: () => number;
-    /** Override the runId generator. Tests pin this. */
-    runId?: (params: CampaignFactoryParams) => string;
-    /**
-     * Agent profile cell for campaign runs. Static profiles can pass an object;
-     * routers or variant-specific harnesses can pass a factory. The campaign
-     * stamps the built cell onto every `RunRecord` and rejects profile/model or
-     * profile/prompt contradictions.
-     */
-    agentProfile?: AgentProfileCell | AgentProfileCellInput | ((params: CampaignFactoryParams & {
-        variant: V;
-        scenarioTags: Record<string, string>;
-    }) => AgentProfileCell | AgentProfileCellInput | Promise<AgentProfileCell | AgentProfileCellInput>);
-}
-interface CampaignFactoryParams {
-    campaignId: string;
-    runId: string;
-    variantId: string;
-    scenarioId: string;
-    seed: number;
-}
-interface FailedRun {
-    runId: string;
-    variantId: string;
-    scenarioId: string;
-    seed: number;
-    reason: string;
-    error?: string;
-}
-interface EvalCampaignResult {
-    campaignId: string;
-    /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */
-    campaignFingerprint: string;
-    preregistrationHash: string | null;
-    /** Successful runs only. Failed runs land in `failedRuns`. */
-    runs: RunRecord[];
-    /** Integrity reports for every successful run. */
-    integrityReports: RunIntegrityReport[];
-    failedRuns: FailedRun[];
-    /** Computed when `report` is set on options. */
-    report?: ResearchReport;
-    startedAt: string;
-    endedAt: string;
-}
-declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
-
-/**
- * Researcher interface — stable hook for an external autonomous-research
- * agent to drive the meta-loop.
- *
- * Implementations live downstream (typically in a private repo that
- * runs the actual LLM). This package ships only the contract + a
- * `NoopResearcher` so consumers can wire the surface without being
- * forced to implement every method up front.
- *
- * The four methods mirror the four stages of the paper "Two Loops,
- * Three Roles":
- *
- *   inspectFailures   — given the observed runs, what failure modes
- *                       are present? (data → diagnosis)
- *   proposeChange     — given diagnosed failure modes, what
- *                       structural changes should we try?
- *                       (diagnosis → plan delta)
- *   applyChange       — fold the proposed deltas into a concrete
- *                       experiment plan against an existing baseline.
- *                       (plan delta → executable plan)
- *   evaluateChange    — run the plan, return runs + the gate verdict.
- *                       (executable plan → verdict)
- *
- * Composition is the discipline: a Researcher implementation MUST
- * keep these four steps separate and inspectable. Conflating
- * "diagnose + propose + run" into a single LLM call defeats the
- * point of the framework — you can't audit which step lied.
- *
- * THIS INTERFACE IS STABLE. Breaking changes require a new module
- * (e.g. `Researcher2`) so existing implementations keep working.
- */
-
-/** A diagnosed failure mode with the run-IDs that exhibit it. */
-interface FailureMode {
-    /** Short machine-readable code. Must be stable across runs of the
-     *  same researcher to enable longitudinal tracking. */
-    code: string;
-    /** Human-readable description for the paper / dashboard. */
-    description: string;
-    evidence: {
-        /** Run IDs (from `RunRecord.runId`) where this failure mode was
-         *  observed. */
-        runIds: string[];
-        /** Number of run samples that informed the diagnosis. */
-        samples: number;
-    };
-}
-/** A single steering change the researcher wants to try. */
-interface SteeringChange {
-    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
-    /** Implementation-specific payload. Researcher implementations
-     *  define the schema — keep this `unknown` here to avoid coupling
-     *  the public interface to any one researcher's internal model. */
-    payload: unknown;
-    /** Why the researcher proposed this change. Goes into the audit
-     *  trail next to the failure-mode evidence. */
-    rationale: string;
-    /** Optional self-reported expected delta on the headline metric. */
-    expectedDelta?: number;
-}
-/** A single experiment plan, mapped onto the search/holdout splits. */
-interface ExperimentPlan {
-    baselineCandidateId: string;
-    proposedCandidateId: string;
-    changes: SteeringChange[];
-    /** USD ceiling for the entire experiment. The runner must stop
-     *  before exceeding this and report a partial result. */
-    evaluationBudgetUsd: number;
-    /** Item IDs (your dataset keys) for the search vs holdout splits. */
-    splits: {
-        search: string[];
-        holdout: string[];
-    };
-}
-/** Result of running a plan: every run, plus the gate verdict. */
-interface ExperimentResult {
-    plan: ExperimentPlan;
-    runs: RunRecord[];
-    gateDecision: GateDecision;
-}
-/**
- * The researcher loop. Stable, four-step, inspectable.
- *
- *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
- *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
- *   └──────────┘                   └──────────┘                └────┬─────┘
- *                                                                   │
- *                                                                   ▼
- *                              ┌────────────────┐  applyChange ┌────────┐
- *                              │ ExperimentPlan │ ◀────────────│  base  │
- *                              └────────┬───────┘              └────────┘
- *                                       │
- *                       evaluateChange  ▼
- *                              ┌────────────────┐
- *                              │ ExperimentResult│
- *                              └────────────────┘
- */
-interface Researcher {
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-interface CallbackResearcherOptions {
-    inspectFailures: Researcher['inspectFailures'];
-    proposeChange: Researcher['proposeChange'];
-    applyChange: Researcher['applyChange'];
-    evaluateChange: Researcher['evaluateChange'];
-}
-/**
- * Minimal concrete researcher for tests, scripts, and small integrations.
- * Larger autonomous researchers can still implement `Researcher` directly.
- */
-declare class CallbackResearcher implements Researcher {
-    private readonly callbacks;
-    constructor(callbacks: CallbackResearcherOptions);
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-/**
- * No-op researcher — fails loud on every method. Use as a placeholder
- * in code paths that wire the interface but don't have an implementation
- * yet. Importantly, this does NOT silently succeed: a no-op researcher
- * that returned empty arrays would muffle the loop's signal that
- * nobody implemented the brain.
- */
-declare class NoopResearcher implements Researcher {
-    private readonly hint;
-    constructor(hint?: string);
-    inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(_changes: SteeringChange[], _baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-
-export { CallbackResearcher as C, type ExperimentPlan as E, type FailureMode as F, NoopResearcher as N, type Researcher as R, type SteeringChange as S, type ExperimentResult as a, type EvalCampaignResult as b, type EvalCampaignOptions as c, type CallbackResearcherOptions as d, type CampaignFactoryParams as e, type CampaignIntegrityPolicy as f, type CampaignRunContext as g, type CampaignRunOutcome as h, type CampaignRunner as i, type CampaignScenario as j, type CampaignVariant as k, type FailedRun as l, runEvalCampaign as r };

package/dist/rubric-predictive-validity-p49lLVrE.d.ts

@@ -1,105 +0,0 @@
-import { R as RunRecord } from './run-record-BDH49H2E.js';
-import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
-
-/**
- * Rubric predictive validity — does our eval rubric predict deployment
- * outcomes?
- *
- * `correlationStudy` (already in this package) joins a `TraceStore` to an
- * `OutcomeStore` and computes Pearson + Spearman + bootstrap CI for each
- * (eval-metric, outcome-metric) pair. That answers "does X correlate with
- * Y at all." `rubricPredictiveValidity` is the campaign-shaped wrapper
- * around it: take a sequence of `RunRecord`s (the canonical campaign
- * artifact) and a `DeploymentOutcomeStore`, join on `runId`, return a
- * ranked verdict on every rubric whose dimension scores were captured in
- * `outcome.raw`.
- *
- * The point — quoting the methodology doc — is that **without this loop
- * every rubric is faith-based**. Once it's wired, you know which rubrics
- * have earned their promotion power and which ones are decoration.
- *
- *   const validity = await rubricPredictiveValidity({
- *     runs: lastQuarter,
- *     outcomes: shipFlagOutcomeStore,
- *     outcomeMetrics: ['revenue_lift', 'retention_30d', 'csat'],
- *     rubrics: ['anti_slop', 'semantic_concept', 'tool_recovery'],
- *   })
- *   for (const r of validity.ranked) {
- *     console.log(`${r.rubric} → ${r.bestOutcome}: ρ=${r.spearman.toFixed(2)}`)
- *   }
- *
- * The function is intentionally read-only. Use the verdict to deprecate
- * decorative rubrics, re-weight composite scores, or trigger a
- * recalibration sweep when predictive validity drops below a threshold.
- */
-
-interface RubricPredictiveValidityInput {
-    /**
-     * Canonical campaign output. Each record's `outcome.raw[<rubricId>]`
-     * provides the eval score; missing keys are silently skipped per pair.
-     */
-    runs: RunRecord[];
-    outcomes: OutcomeStore;
-    /**
-     * Outcome metric names to evaluate against. Each must appear in at
-     * least one `DeploymentOutcome.metrics` keyspace; pairs with too few
-     * joined samples are excluded from the result.
-     */
-    outcomeMetrics: string[];
-    /**
-     * Rubric ids to evaluate. Must appear as keys in `RunRecord.outcome.raw`.
-     * If omitted, every numeric key in `outcome.raw` across the run set is
-     * treated as a rubric.
-     */
-    rubrics?: string[];
-    /** Minimum joined-sample count before a pair is reported. Default 8. */
-    minSamples?: number;
-    /** Bootstrap resamples for CI. Default 500. */
-    bootstrapResamples?: number;
-    /** Random seed for the bootstrap (mulberry32). Default unset (Math.random). */
-    seed?: number;
-    /**
-     * Reduction when multiple outcomes attach to one runId. Default `'latest'`
-     * (most recently captured).
-     */
-    reduction?: 'latest' | 'mean' | 'max';
-}
-interface RubricOutcomePair {
-    rubric: string;
-    outcome: string;
-    n: number;
-    pearson: number;
-    spearman: number;
-    ci95: {
-        low: number;
-        high: number;
-    };
-    /**
-     * Verdict bucket. `load_bearing` ≥ 0.7, `informative` ≥ 0.4,
-     * `decorative` < 0.4 in absolute correlation. A negative correlation
-     * with a desired outcome is also `decorative` — actively misleading
-     * is worse than uninformative.
-     */
-    verdict: 'load_bearing' | 'informative' | 'decorative';
-}
-interface RubricRanking {
-    rubric: string;
-    /** Outcome metric this rubric correlated best with. */
-    bestOutcome: string;
-    spearman: number;
-    pearson: number;
-    n: number;
-    verdict: RubricOutcomePair['verdict'];
-}
-interface RubricPredictiveValidityReport {
-    pairs: RubricOutcomePair[];
-    /** Per-rubric best pair, sorted descending by |spearman|. */
-    ranked: RubricRanking[];
-    joinedSamples: number;
-    skippedRuns: number;
-    /** Rubrics that were declared but never produced a usable score. */
-    rubricsWithoutData: string[];
-}
-declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
-
-export { type RubricOutcomePair as R, type RubricPredictiveValidityInput as a, type RubricPredictiveValidityReport as b, type RubricRanking as c, rubricPredictiveValidity as r };