@tangle-network/agent-eval 0.86.0 → 0.89.0

package/dist/counterfactual-Dwibr5IW.d.ts

@@ -0,0 +1,85 @@
+import { T as TraceEmitter } from './emitter-DEZwY14K.js';
+import { T as TraceStore } from './store-CKUAgsJz.js';
+import { T as Trajectory, a as TrajectoryStep } from './trajectory-GEdXJCL5.js';
+
+/**
+ * Counterfactual replay — "what would have happened if we'd changed
+ * exactly one thing at turn N?"
+ *
+ * The framework does NOT drive the agent — it sets up the replay
+ * context (prior spans, prior state, mutation spec) and records the
+ * resulting divergence. Consumers supply an `executeFrom(ctx)` callback
+ * that runs their agent starting from turn N with the mutation applied.
+ *
+ * Counterfactual runs are recorded as a new Run with `layer='meta'` and
+ * `parentRunId = originalRunId`, so downstream diff + correlation
+ * pipelines see them natively.
+ */
+
+type CounterfactualMutation = {
+    kind: 'swap-model';
+    at: number;
+    newModel: string;
+} | {
+    kind: 'swap-tool-result';
+    at: number;
+    newResult: unknown;
+} | {
+    kind: 'truncate-after';
+    at: number;
+} | {
+    kind: 'inject-system-message';
+    at: number;
+    content: string;
+} | {
+    kind: 'custom';
+    at: number;
+    describe: string;
+    apply: (step: TrajectoryStep) => TrajectoryStep;
+};
+interface CounterfactualContext {
+    originalRunId: string;
+    originalTrajectory: Trajectory;
+    /** Steps up to (but not including) the mutation point — the prefix the
+     *  replayed agent inherits as its prior conversation/tool history. */
+    prefix: TrajectoryStep[];
+    mutation: CounterfactualMutation;
+    /** Pre-applied mutation on the step at `mutation.at`. Consumers use this
+     *  as the FIRST step the replayed agent emits (they decide whether to
+     *  re-emit it or continue from there). */
+    mutatedStep: TrajectoryStep;
+}
+interface CounterfactualResult {
+    counterfactualRunId: string;
+    originalRunId: string;
+    mutation: CounterfactualMutation;
+    /** Structured delta summary — caller can extend via scoring. */
+    delta: {
+        originalOutcomeScore: number | null;
+        counterfactualOutcomeScore: number | null;
+        deltaScore: number | null;
+    };
+}
+interface CounterfactualRunner {
+    /**
+     * Execute the agent from `ctx.prefix` with the mutation applied.
+     * MUST emit spans into the provided emitter so they become part of
+     * the counterfactual run. MUST call emitter.endRun() with a verdict.
+     */
+    executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>;
+}
+declare function runCounterfactual(store: TraceStore, originalRunId: string, mutation: CounterfactualMutation, runner: CounterfactualRunner): Promise<CounterfactualResult>;
+/**
+ * Aggregate a batch of counterfactuals into a simple attribution table:
+ * which mutation kinds move outcomes most? (Useful when you run a grid
+ * over the same trajectory — swap-model at every llm span, swap-tool
+ * at every tool span — and want a ranked summary.)
+ */
+declare function attributeCounterfactuals(results: CounterfactualResult[]): Array<{
+    mutationKind: CounterfactualMutation['kind'];
+    n: number;
+    meanAbsDelta: number;
+    meanSignedDelta: number;
+}>;
+
+export { type CounterfactualMutation as C, attributeCounterfactuals as a, type CounterfactualRunner as b, type CounterfactualContext as c, type CounterfactualResult as d, runCounterfactual as r };

package/dist/{dataset-B2kL-fSM.d.ts → dataset-BbGkaN2I.d.ts}

@@ -1,4 +1,4 @@
-import { V as ValidationError } from './errors-Dwqw-T_m.js';
+import { V as ValidationError } from './errors-CzMUYo7b.js';
 
 /**
  * Dataset — versioned, sliceable, content-hashed scenario collection.

package/dist/{registry-DrEQ3Luj.d.ts → default-registry-zoGHUQEH.d.ts}

@@ -1,4 +1,6 @@
-import { A as Analyst, a as AnalystContext, b as AnalystRunSummary, c as AnalystFinding, d as AnalystRunResult, C as ChatClient, e as AnalystRunInputs, f as AnalystRunEvent } from './types-Cu3u_x59.js';
+import { AxAIService } from '@ax-llm/ax';
+import { T as TraceAnalystKindSpec } from './kind-factory-5b7xXXOr.js';
+import { A as Analyst, a as AnalystContext, b as AnalystRunSummary, c as AnalystFinding, d as AnalystRunResult, C as ChatClient, e as AnalystRunInputs, f as AnalystRunEvent } from './types-2VVIL04s.js';
 
 /**
  * AnalystRegistry — orchestrate N analysts against one run.
@@ -125,4 +127,29 @@ declare class AnalystRegistry {
     private routeInput;
 }
 
-export { type AnalystHooks as A, type BudgetPolicy as B, type RegistryRunOpts as R, AnalystRegistry as a, type AnalystRegistryOptions as b };
+/**
+ * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers
+ * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`.
+ *
+ * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no
+ * model and is model-agnostic by construction). The agentic RLM kinds are
+ * registered only when an `ai` service is supplied — so a caller with no LLM
+ * still gets the full behavioral/efficiency diagnosis, and the substrate's
+ * "any model (including no model)" guarantee holds at the suite level.
+ */
+
+interface DefaultAnalystRegistryOptions {
+    /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
+    ai?: AxAIService;
+    /** Model for the agentic kinds (falls back to the ai service default). */
+    model?: string;
+    /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
+    kinds?: readonly TraceAnalystKindSpec[];
+    /** Set false to omit the deterministic behavioral analyst (default: include). */
+    includeBehavioral?: boolean;
+    /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */
+    registry?: AnalystRegistryOptions;
+}
+declare function buildDefaultAnalystRegistry(opts?: DefaultAnalystRegistryOptions): AnalystRegistry;
+
+export { AnalystRegistry as A, type BudgetPolicy as B, type DefaultAnalystRegistryOptions as D, type RegistryRunOpts as R, type AnalystHooks as a, buildDefaultAnalystRegistry as b, type AnalystRegistryOptions as c };

package/dist/diagnose.d.ts

@@ -0,0 +1,251 @@
+import { C as CounterfactualMutation, a as attributeCounterfactuals, b as CounterfactualRunner } from './counterfactual-Dwibr5IW.js';
+export { c as CounterfactualContext, d as CounterfactualResult } from './counterfactual-Dwibr5IW.js';
+import { S as Span } from './schema-m0gsnbt3.js';
+import { T as TraceStore } from './store-CKUAgsJz.js';
+import { a as TrajectoryStep, T as Trajectory } from './trajectory-GEdXJCL5.js';
+import { h as AnalystSeverity, c as AnalystFinding } from './types-2VVIL04s.js';
+import { C as CorpusRecord } from './corpus-BoR-041R.js';
+import { R as RunRecord } from './run-record-e7vj1uZQ.js';
+import './emitter-DEZwY14K.js';
+import './store-C1YxJDEK.js';
+import './types-Croy5h7V.js';
+import '@tangle-network/tcloud';
+import './llm-client-BeEcAokY.js';
+import './errors-CzMUYo7b.js';
+import './raw-provider-sink-C46HDghv.js';
+
+/**
+ * Causal sweep — WHY did this run fail?
+ *
+ * Orchestrates the dormant counterfactual primitives into a responsibility
+ * report: for each candidate step, run `reps` counterfactual replays per
+ * mutation (via `runCounterfactual` — the consumer's `CounterfactualRunner`
+ * is the execution seam) and reduce the per-rep score deltas into a mean
+ * effect + bootstrap confidence interval (via `confidenceInterval`).
+ *
+ * Why `reps` is REQUIRED: a single intervention delta is one stochastic
+ * draw — LLM re-execution from a prefix is sampled, so one replay cannot
+ * distinguish "this step caused the failure" from sampling noise. The
+ * signal is the distribution of deltas across reps; the CI over that
+ * distribution is what lets a caller say "this step's effect excludes
+ * zero" instead of eyeballing a point estimate.
+ *
+ * Budget discipline: the sweep never silently drops cells. When the
+ * remaining budget cannot fund a full `reps`-sized cell, the sweep halts
+ * and every step not fully probed is named in `uncovered`.
+ */
+
+/** Stable reference to a trajectory step — carried through reports,
+ *  findings, and corpus records so evidence stays addressable. */
+interface StepRef {
+    index: number;
+    spanId: string;
+    kind: Span['kind'];
+    name: string;
+}
+declare function stepRefOf(step: TrajectoryStep): StepRef;
+interface CausalSweepOptions {
+    store: TraceStore;
+    /** The failed run to diagnose. Its `outcome.score` is the baseline every
+     *  counterfactual delta is measured against. */
+    runId: string;
+    /** Execution seam — identical contract to `runCounterfactual`: re-runs the
+     *  agent from the mutation point and MUST `endRun` with a numeric score. */
+    runner: CounterfactualRunner;
+    /** Trajectory indices to probe. Default: every llm + tool span — the kinds
+     *  the existing `CounterfactualMutation` set targets. */
+    candidateSteps?: number[];
+    /**
+     * Mutations to probe a given step with. Returned mutations MUST target
+     * `step.index`. Default probes are the payload-free existing kinds:
+     *   - tool span → `swap-tool-result` with `newResult: null` (knockout:
+     *     how much did the run depend on this tool's information?)
+     *   - llm span → `truncate-after` (re-roll: how much did the realized
+     *     turn deviate from the policy's typical continuation?)
+     * `swap-model` / `inject-system-message` need consumer payloads, so they
+     * are opt-in via this callback.
+     */
+    mutationsPerStep?: (step: TrajectoryStep) => CounterfactualMutation[];
+    /** Replays per (step, mutation) cell. Minimum 2 — see module doc. */
+    reps: number;
+    /** Hard cap on total counterfactual replays across the whole sweep. */
+    budget: number;
+    /** Seed for the bootstrap CI resampler. Deterministic default so two
+     *  sweeps over the same deltas report identical intervals. */
+    ciSeed?: number;
+    /** Bootstrap CI confidence level. Default 0.95. */
+    ciConfidence?: number;
+}
+interface StepResponsibility {
+    stepRef: StepRef;
+    mutationKind: CounterfactualMutation['kind'];
+    /** Mean of per-rep score deltas (counterfactual − original). */
+    meanEffect: number;
+    /** Bootstrap CI over the per-rep deltas. */
+    ci: {
+        mean: number;
+        lower: number;
+        upper: number;
+    };
+    /** `ci.lower > 0 || ci.upper < 0` — the effect is distinguishable from noise. */
+    ciExcludesZero: boolean;
+    reps: number;
+    /** Raw per-rep deltas — downstream evidence, never re-derived. */
+    deltas: number[];
+    /** Replay run ids (layer='meta', parentRunId=original) for audit. */
+    counterfactualRunIds: string[];
+}
+interface CausalResponsibilityReport {
+    runId: string;
+    originalScore: number;
+    /** Ranked by |meanEffect| descending — the blame ordering. */
+    steps: StepResponsibility[];
+    /** Kind-level aggregate from the existing `attributeCounterfactuals`. */
+    byMutationKind: ReturnType<typeof attributeCounterfactuals>;
+    replaysUsed: number;
+    budget: number;
+    /** Steps planned but not fully probed before the budget ran out.
+     *  Named, never silent: an absent step is "no effect found"; an
+     *  uncovered step is "not measured". */
+    uncovered: StepRef[];
+}
+declare function causalSweep(opts: CausalSweepOptions): Promise<CausalResponsibilityReport>;
+
+/**
+ * Replay-validated repair — WHAT SHOULD HAVE HAPPENED?
+ *
+ * Takes the blamed steps from a `CausalResponsibilityReport`, asks a
+ * consumer-supplied `proposeFix` (LLM-backed in live use) for candidate
+ * mutations, and machine-verifies each candidate by replaying the run
+ * WITH the mutation applied (through the same `runCounterfactual` seam
+ * the sweep uses).
+ *
+ * A repair is "what should have happened" ONLY when every validation
+ * replay crosses `flipThreshold` — a prescription is never speculated,
+ * it is demonstrated. Candidates that don't flip, or whose replay
+ * errors, land in `rejected` with a typed reason; nothing is dropped
+ * silently.
+ */
+
+/** Context handed to `proposeFix` so an LLM-backed proposer can see the
+ *  full trajectory plus the responsibility evidence for the blamed step. */
+interface RepairContext {
+    runId: string;
+    trajectory: Trajectory;
+    originalScore: number;
+    responsibility: StepResponsibility;
+}
+interface PrescribeRepairOptions {
+    store: TraceStore;
+    /** The failed run the sweep diagnosed. */
+    runId: string;
+    /** Execution seam — same `CounterfactualRunner` contract as the sweep. */
+    runner: CounterfactualRunner;
+    /** Blamed steps from `causalSweep` — typically `report.steps.slice(0, k)`. */
+    blamed: StepResponsibility[];
+    /** Candidate-fix generator. Consumer-supplied; LLM-backed in live use.
+     *  Returned mutations MUST target the blamed step's index. */
+    proposeFix: (step: TrajectoryStep, context: RepairContext) => Promise<CounterfactualMutation[]>;
+    /** Score every validation replay must reach for the repair to count. Default 0.5. */
+    flipThreshold?: number;
+    /** Validation replays per candidate mutation. Default 3. */
+    repsToValidate?: number;
+    /** Max candidate mutations tried per step. Default: all proposed. */
+    maxAttemptsPerStep?: number;
+}
+interface ValidatedRepair {
+    stepRef: StepRef;
+    mutation: CounterfactualMutation;
+    /** Always true — presence in `repairs` IS the machine-verified claim. */
+    validated: true;
+    /** Mean counterfactual score across the validation reps. */
+    meanScore: number;
+    /** meanScore − originalScore. */
+    deltaScore: number;
+    reps: number;
+    /** Replay run ids backing the validation — audit trail. */
+    counterfactualRunIds: string[];
+}
+interface RejectedRepair {
+    stepRef: StepRef;
+    mutation: CounterfactualMutation;
+    reason: 'did-not-flip' | 'error';
+    /** Present for 'did-not-flip': mean delta over the reps that ran. */
+    deltaScore?: number;
+    /** Present for 'error': the message, preserved for diagnosis. */
+    error?: string;
+}
+interface RepairReport {
+    runId: string;
+    originalScore: number;
+    flipThreshold: number;
+    repairs: ValidatedRepair[];
+    rejected: RejectedRepair[];
+    replaysUsed: number;
+}
+declare function prescribeRepair(opts: PrescribeRepairOptions): Promise<RepairReport>;
+
+/**
+ * Remediation adapters — HOW DO WE MAKE IT HAPPEN?
+ *
+ * The diagnose chain ends by feeding existing improvement machinery,
+ * not by building new machinery:
+ *
+ *   - `toAnalystFindings` → the analyst contract (`makeFinding`), so
+ *     responsibility evidence flows into the same registry / steering /
+ *     diff pipeline every other analyst feeds.
+ *   - `toCorpusRecord` → the RL corpus (`CorpusRecord`), pinning the
+ *     diagnosed failure + validated repair as a permanent scenario.
+ *   - `suggestInvariant` → a plain-data hint in the shape the
+ *     trace-contracts machinery consumes (`never` / `without` clauses).
+ */
+
+declare const DIAGNOSE_ANALYST_ID = "diagnose-causal-sweep";
+/** Severity from causal effect size. Effects whose CI includes zero are
+ *  'info' regardless of magnitude — an indistinguishable-from-noise effect
+ *  must not steer remediation priority. */
+declare function severityFromEffect(responsibility: StepResponsibility): AnalystSeverity;
+/** Deterministic human-readable rendering of a mutation — used in
+ *  recommended actions, corpus completions, and invariant hints. */
+declare function describeMutation(mutation: CounterfactualMutation): string;
+/**
+ * Lift a responsibility report (and optionally its validated repairs) into
+ * `AnalystFinding`s via the real `makeFinding` factory. One finding per
+ * probed step; a validated repair for that step upgrades the finding with
+ * a `recommended_action` + the replay-validation evidence.
+ *
+ * Findings are OBSERVED causal probes (replay deltas), not judge verdicts,
+ * so `derived_from_judge` stays unset and they may steer.
+ */
+declare function toAnalystFindings(report: CausalResponsibilityReport, repairs?: RepairReport): AnalystFinding[];
+/**
+ * Pin the diagnosed failure as a permanent corpus scenario. Takes the
+ * original run's `RunRecord` projection plus a validated repair and emits
+ * a fresh `CorpusRecord` (new runId, so corpus dedup keeps both the raw
+ * failure and the diagnosed entry).
+ *
+ * `completion` defaults to the validated mutation's rendering — "what
+ * should have happened" in machine-derived form. Supply `prompt` (and
+ * optionally a richer `completion`) when the trajectory text is available
+ * so the record is harvestable by `buildDatasetFromCorpus`.
+ */
+declare function toCorpusRecord(run: RunRecord, repair: ValidatedRepair, opts?: {
+    prompt?: string;
+    completion?: string;
+}): CorpusRecord;
+/** Plain-data invariant hint. The trace-contracts machinery consumes this
+ *  shape: `never` is a pattern that must not appear in a passing trace;
+ *  `without` is a guard whose absence makes the failure reachable. */
+interface InvariantHint {
+    description: string;
+    never?: string;
+    without?: string;
+}
+/**
+ * Derive an invariant hint from a validated repair. Deterministic per
+ * mutation kind — the hint names the contract a trace must satisfy so
+ * the diagnosed failure cannot silently recur.
+ */
+declare function suggestInvariant(repair: ValidatedRepair): InvariantHint;
+
+export { type CausalResponsibilityReport, type CausalSweepOptions, CounterfactualMutation, CounterfactualRunner, DIAGNOSE_ANALYST_ID, type InvariantHint, type PrescribeRepairOptions, type RejectedRepair, type RepairContext, type RepairReport, type StepRef, type StepResponsibility, type ValidatedRepair, causalSweep, describeMutation, prescribeRepair, severityFromEffect, stepRefOf, suggestInvariant, toAnalystFindings, toCorpusRecord };