@tangle-network/agent-eval 0.95.0 → 0.95.1

package/dist/fuzz.d.ts

@@ -1 +1,547 @@
-export * from './fuzz/index'
+import { C as CostLedger, a as CostChannel } from './cost-ledger-DuSqlw5B.js';
+import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
+import { A as AdversarialMutation } from './adversarial-DIVcDoI_.js';
+
+/**
+ * Behavior-space exploration — types.
+ *
+ * One engine searches a space of inputs against a target, scores each run with a
+ * multi-objective verdict, keeps a quality-diversity archive, and admits only
+ * findings that pass the validity gates. Adversarial fuzzing is the headline
+ * preset (`fuzzAgent`); swapping the `Objective` re-points the same engine at
+ * novelty search, curriculum growth, or user-simulation.
+ *
+ * Two kinds of coordinates, deliberately distinct:
+ *  - INPUT axes (`space.axes`) are the stratification plan — enumerable up front,
+ *    so allocation and the coverage denominator (planned vs covered) are honest.
+ *  - MEASURED descriptors (`descriptor(scenario, ev)`) are read off the rollout —
+ *    they bin the archive by what the agent DID, and never inflate the coverage
+ *    denominator (a behavior you haven't seen yet is not a planned cell).
+ *
+ * An `Evaluation` IS a `DefaultVerdict` — same spine as judges and verifiers,
+ * never a parallel score shape.
+ */
+
+/** One input axis of the stratification plan (e.g. matterType, difficulty, personaRigor). */
+interface SpaceAxis {
+    name: string;
+    values: string[];
+}
+/** The input space to stratify. Cells are the cartesian product of the axes. */
+interface BehaviorSpace {
+    axes: SpaceAxis[];
+}
+/** One input cell: a coordinate in the stratification plan. */
+interface Cell {
+    /** Stable id, e.g. `matterType=nda|difficulty=hard`. */
+    id: string;
+    coords: Record<string, string>;
+}
+/**
+ * The outcome of running the target against one scenario: a `DefaultVerdict`
+ * (`valid`, headline `score` in [0,1], per-dimension `scores`, `notes`) plus the
+ * fields exploration needs. Keep `scores` populated — the coverage map surfaces
+ * WHICH dimension is weak only when evaluations carry it.
+ */
+interface Evaluation extends DefaultVerdict {
+    /** Measured behavior coordinates (e.g. `{ outcome: 'refused' }`). Bins the archive. */
+    descriptor?: Record<string, string>;
+    /** Surfaced output — drives exemplars + minimization. */
+    output?: string;
+    /** RunRecord id when the target persisted a trace. */
+    runId?: string;
+    /** Structured labels, e.g. failure classes (`hallucination`, `refusal`). */
+    labels?: string[];
+    /** Wall-clock for the evaluation, when the consumer measures it more precisely
+     *  than the engine can (e.g. excluding judge time). Engine-measured otherwise. */
+    latencyMs?: number;
+}
+/** Run the target against one scenario in a cell. */
+type Evaluator<S> = (scenario: S, cell: Cell) => Promise<Evaluation>;
+/** Context a proposer sees — prior elites + findings let a skill-backed proposer steer. */
+interface ProposeContext<S> {
+    cell: Cell;
+    seeds: S[];
+    /** Current archive elites whose input cell matches. */
+    elites: S[];
+    /** Verified findings so far (read-only) — probe new gaps, not re-found ones. */
+    findings: ReadonlyArray<Finding<S>>;
+    /** How many candidates to propose. */
+    count: number;
+    rng: () => number;
+}
+/**
+ * Produces candidate scenarios for a cell. A plain function — `mutationProposer`
+ * builds one from mutation operators; an agent running a generator skill IS one
+ * (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed.
+ */
+type Proposer<S> = (ctx: ProposeContext<S>) => Promise<S[]> | S[];
+/**
+ * What "interesting" means. `interest` in [0,1]; a candidate is notable (gate-
+ * checked, reported) when `interest >= threshold`. `adversarialObjective` (low
+ * score is interesting) and `noveltyObjective` (far from the archive) ship.
+ */
+interface Objective {
+    kind: string;
+    interest(ev: Evaluation, ctx: ObjectiveContext): number;
+    /** Default 0.5. */
+    threshold?: number;
+}
+interface ObjectiveContext {
+    archiveScores: number[];
+    archiveDescriptors: Array<Record<string, string> | undefined>;
+}
+/**
+ * Validity gates — the moat. A notable candidate is admitted ONLY when it is a
+ * fair, answerable task (`isValid`) AND reproduces under a meaning-preserving
+ * rephrase (`isUncontaminated`). Pass-through by default (testable without an
+ * LLM); live wiring supplies real gates.
+ */
+interface ValidityGates<S> {
+    isValid?: (scenario: S, ev: Evaluation, cell: Cell) => boolean | Promise<boolean>;
+    isUncontaminated?: (scenario: S, ev: Evaluation, cell: Cell) => boolean | Promise<boolean>;
+}
+/** A gate-verified, minimized finding — the unit the capsule reports. */
+interface Finding<S> {
+    id: string;
+    cell: Cell;
+    scenario: S;
+    /** The minimized trigger (== `scenario` when no minimizer is supplied). */
+    minimized: S;
+    /** Legible text of the minimized trigger, when `scenarioText` is supplied. */
+    text?: string;
+    /** The full multi-objective verdict. */
+    evaluation: Evaluation;
+    /** The objective's interest score that flagged it. */
+    interest: number;
+    /** Which objective flagged it. */
+    objective: string;
+}
+/** An archive elite — the most interesting scenario seen for one bin. */
+interface ArchiveEntry<S> {
+    /** Input cell + measured descriptor coords combined, e.g. `difficulty=hard|outcome=refused`. */
+    binId: string;
+    cell: Cell;
+    scenario: S;
+    evaluation: Evaluation;
+    interest: number;
+}
+/** Summary of a sample — every aggregate carries its spread, never a bare mean. */
+interface Distribution {
+    mean: number;
+    median: number;
+    p90: number;
+    min: number;
+    max: number;
+    n: number;
+}
+/** Per-INPUT-cell coverage — the planned-vs-covered map. */
+interface CoverageCell {
+    cell: Cell;
+    runs: number;
+    /** Headline score distribution in [0,1]; `null` when the cell was never run
+     *  (honestly uncovered — never a fabricated zero). */
+    score: Distribution | null;
+    /** Fraction of runs the objective flagged as notable. */
+    findingRate: number;
+    /** Per-dimension score distributions — surfaces WHICH dimension is weak and
+     *  how consistently. */
+    dimensions: Record<string, Distribution>;
+    /** Evaluation wall-clock per run; engine-measured unless the evaluation
+     *  carried its own `latencyMs`. `null` when the cell was never run. */
+    latencyMs: Distribution | null;
+    /** Known dollars spent in this cell — present only when cost tracking was
+     *  wired; runs with unknown cost are counted apart, never folded in as $0. */
+    costUsd?: number;
+    costUnknownRuns?: number;
+}
+/** The artifact every exploration produces. */
+interface CapsuleData<S> {
+    target: string;
+    objective: string;
+    /** Stamped by the caller — the engine stays clock-free and deterministic. */
+    generatedAt?: string;
+    coverage: CoverageCell[];
+    /** Verified findings, sorted by descending interest. */
+    findings: Finding<S>[];
+    /** QD archive elites (binned by input × measured coords). */
+    archive: ArchiveEntry<S>[];
+    /** Post-harden lift, filled by a second pass after an improvement. */
+    lift?: {
+        before: number;
+        after: number;
+        verdict: string;
+    };
+    stats: {
+        totalRuns: number;
+        /** Input-cell denominator — the stratification plan. */
+        cellsTotal: number;
+        cellsCovered: number;
+        /** Distinct measured-descriptor bins observed (never part of the denominator). */
+        behaviorBinsObserved: number;
+        candidateFindings: number;
+        verifiedFindings: number;
+        /** Distribution of per-cell mean scores across covered cells (cells weigh
+         *  equally — variance steering sends more runs to weak cells, so a
+         *  run-weighted average would bias low). `null` when nothing ran. */
+        robustness: Distribution | null;
+        /** Evaluation wall-clock across all runs. `null` when nothing ran. */
+        latencyMs: Distribution | null;
+        /** Known dollars spent on this exploration's runs. Present only when cost
+         *  tracking was wired (`costOf`) — absent means "not tracked", never $0. */
+        costUsd?: number;
+        /** Runs whose cost was unknown (`costOf` returned null) — counted apart,
+         *  never folded into `costUsd` as a fabricated $0. */
+        costUnknownRuns?: number;
+        /** Evaluations that threw (transport/backend failures). They consumed no
+         *  run budget and scored nothing — an infra axis, never folded into
+         *  robustness or reported as findings. */
+        evalErrors: number;
+        /** Present when the run stopped before its budget because consecutive
+         *  eval errors tripped the circuit breaker (a dead backend must not burn
+         *  the remaining budget). The capsule-so-far is complete and honest. */
+        stoppedEarly?: {
+            reason: 'eval-errors';
+            detail: string;
+        };
+    };
+}
+/**
+ * Known cost of one evaluated run. `model` attributes the spend in the ledger's
+ * per-model rollup; absent, the entry is labeled `unattributed` (the dollars are
+ * real either way — recorded as `actualCostUsd`, never an estimate).
+ */
+interface RunCost {
+    usd: number;
+    model?: string;
+}
+type ExploreEvent<S> = {
+    type: 'cell-allocated';
+    cell: Cell;
+    count: number;
+} | {
+    type: 'evaluated';
+    cell: Cell;
+    scenario: S;
+    evaluation: Evaluation;
+} | {
+    type: 'finding';
+    finding: Finding<S>;
+} | {
+    type: 'eval-error';
+    cell: Cell;
+    scenarioId: string;
+    message: string;
+} | {
+    type: 'round';
+    runsUsed: number;
+    budget: number;
+};
+interface ExploreOptions<S> {
+    /** Name of the target under exploration — labels the capsule. */
+    target: string;
+    /** The input stratification plan. */
+    space: BehaviorSpace;
+    /** Candidate generator. */
+    proposer: Proposer<S>;
+    /** Runs the target → multi-objective `Evaluation`. */
+    evaluate: Evaluator<S>;
+    /** Seed corpus per cell. */
+    seedsFor: (cell: Cell) => S[] | Promise<S[]>;
+    /** Stable id for a scenario (dedup + lineage). */
+    scenarioId: (scenario: S) => string;
+    /** Human-legible text — drives capsule exemplars. */
+    scenarioText?: (scenario: S) => string;
+    /** Measured behavior coords appended to the archive bin. Default: input cell only. */
+    descriptor?: (scenario: S, ev: Evaluation) => Record<string, string>;
+    /** What "interesting" means. Default: `adversarialObjective(0.5)`. */
+    objective?: Objective;
+    /** Validity gates. Default pass-through. */
+    gates?: ValidityGates<S>;
+    /** Budget steering across input cells. `variance` chases uncertainty; `uniform` is the unsteered ablation baseline. Default `variance`. */
+    allocation?: 'variance' | 'uniform';
+    /** Total target evaluations. */
+    budget: number;
+    /** Minimum evaluations per input cell before steering. Default 2. */
+    floorPerCell?: number;
+    /** Shrink a notable scenario to its minimal trigger. Default: identity. */
+    minimize?: (scenario: S, evaluate: Evaluator<S>, cell: Cell) => Promise<S> | S;
+    /** Max concurrent `evaluate` calls. Default 1. */
+    concurrency?: number;
+    /** Stop the run after this many CONSECUTIVE eval errors (a dead backend must
+     *  not burn the remaining budget). Successes reset the streak. Default 5. */
+    maxConsecutiveEvalErrors?: number;
+    /** Cooperative cancellation. */
+    signal?: AbortSignal;
+    /** Progress stream. */
+    onProgress?: (event: ExploreEvent<S>) => void;
+    /** Deterministic seed. Default 1. */
+    seed?: number;
+    /**
+     * Cost of one evaluated run — consumer-supplied; the explorer cannot know
+     * token usage. Return null when the cost is unknown: the run is COUNTED in
+     * `stats.costUnknownRuns`, never folded into the total as $0. Required by
+     * every other cost option (`costBudgetUsd` / `ledger` / `onCost`).
+     */
+    costOf?: (scenario: S, cell: Cell, ev: Evaluation) => RunCost | null;
+    /**
+     * Hard dollar ceiling on accumulated KNOWN cost (same semantics as the
+     * control-runtime `budget.maxCostUsd`: nonnegative finite, the session stops
+     * once spent ≥ ceiling; no new evaluation starts after that). Unknown-cost
+     * runs do not consume budget — they are reported separately, so the ceiling
+     * is honest about what it can see.
+     */
+    costBudgetUsd?: number;
+    /**
+     * Sink for per-run cost entries — each known `costOf` result is recorded
+     * with channel 'agent' and `actualCostUsd` (token axes are zero: the
+     * explorer only sees dollars). Pass the program's shared `CostLedger` so
+     * `costReport` stamps fuzz spend alongside judge/analyst spend.
+     */
+    ledger?: CostLedger;
+    /** Observer fired for every known-cost run recorded. */
+    onCost?: (entry: {
+        usd: number;
+        channel: CostChannel;
+    }) => void;
+}
+
+/**
+ * Input-space tiling + coverage projection.
+ *
+ * Cells are the cartesian product of the input axes — the stratification plan,
+ * enumerable up front so the planned-vs-covered denominator is honest. Coverage
+ * is projected from the evaluation log: per cell, the full DISTRIBUTION of the
+ * headline score, of each scored dimension, and of evaluation latency — a bare
+ * mean hides outliers, so every aggregate carries its spread. Per-cell cost is
+ * split known-dollars vs unknown-runs, never folded into a fabricated $0.
+ */
+
+/** One recorded evaluation — the unit coverage and the capsule are built from. */
+interface EvalRecord {
+    cell: Cell;
+    ev: Evaluation;
+    /** The objective's interest score for this evaluation. */
+    interest: number;
+    /** Evaluation wall-clock — engine-measured unless `ev.latencyMs` overrode it. */
+    latencyMs: number;
+    /** Known dollars for this run. `null` = cost tracking was wired but this
+     *  run's cost was unknowable (counted apart). Absent = not tracked at all. */
+    costUsd?: number | null;
+}
+/** Enumerate every input cell (cartesian product of the axes), in stable order. */
+declare function enumerateCells(space: BehaviorSpace): Cell[];
+/** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */
+declare function cellId(space: BehaviorSpace, coords: Record<string, string>): string;
+/**
+ * Project the evaluation log into the per-input-cell coverage map. A cell with
+ * no evaluations reports `score: null` (honestly uncovered), never zeros.
+ */
+declare function buildCoverage(cells: Cell[], log: EvalRecord[], threshold: number): CoverageCell[];
+
+/**
+ * The capsule — the artifact every exploration produces.
+ *
+ * `buildCapsule` assembles coverage + verified findings + the QD archive into a
+ * pure `CapsuleData` (no clock, no I/O — deterministic and snapshot-testable).
+ * `renderCapsuleHtml` turns it into a standalone page: the input-cell heat-map
+ * (planned vs covered), per-dimension weakness chips, and the minimized finding
+ * exemplars. One artifact — the hardening map and the shareable proof object.
+ */
+
+interface BuildCapsuleInput<S> {
+    target: string;
+    objective: string;
+    cells: Cell[];
+    log: EvalRecord[];
+    /** The objective's notable threshold — drives findingRate. */
+    threshold: number;
+    archive: ArchiveEntry<S>[];
+    findings: Finding<S>[];
+    candidateFindings: number;
+    runsUsed: number;
+    /** Known-dollar / unknown-run split — present only when cost tracking was
+     *  wired; the capsule never fabricates a $0 total. */
+    cost?: {
+        costUsd: number;
+        costUnknownRuns: number;
+    };
+    /** Evaluations that threw — infra outcomes, never folded into robustness. */
+    evalErrors: number;
+    /** Set when the consecutive-error circuit breaker stopped the run early. */
+    stoppedEarly?: {
+        reason: 'eval-errors';
+        detail: string;
+    };
+}
+declare function buildCapsule<S>(input: BuildCapsuleInput<S>): CapsuleData<S>;
+interface RenderCapsuleOptions {
+    /** Max finding exemplars to show. Default 8. */
+    maxFindings?: number;
+    /** ISO timestamp to stamp into the page (keeps the pure capsule clock-free). */
+    generatedAt?: string;
+}
+/** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */
+declare function renderCapsuleHtml<S>(capsule: CapsuleData<S>, opts?: RenderCapsuleOptions): string;
+
+/**
+ * The exploration engine — a stateful session over a behavior space.
+ *
+ * Each `step()`: allocate budget across INPUT cells (floor first, then variance
+ * steering toward the least-certain cells), propose candidates (the proposer
+ * reads current elites + findings, so the search deepens generationally),
+ * evaluate with bounded concurrency, archive the most interesting scenario per
+ * input×measured bin, and admit notable candidates that pass the validity gates.
+ * `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —
+ * the surface `makeExploreTools` exposes so an agent can drive the session.
+ *
+ * One evaluation log (`EvalRecord[]`) is the source of truth; allocation
+ * observations and coverage are projections of it.
+ */
+
+declare class BehaviorExplorer<S> {
+    private readonly opts;
+    private readonly cells;
+    private readonly cellById;
+    private readonly objective;
+    private readonly threshold;
+    private readonly floorPerCell;
+    private readonly perRoundBudget;
+    /** The single evaluation log — coverage + allocation are projections of it. */
+    private readonly log;
+    /** binId (input × measured coords) → the most interesting entry seen. */
+    private readonly archiveByBin;
+    private readonly _findings;
+    private runsUsed;
+    private candidateFindings;
+    private evalErrors;
+    private consecutiveEvalErrors;
+    private stoppedEarly;
+    private rngState;
+    /** Accumulated KNOWN dollars — unknown-cost runs never inflate it. */
+    private spentKnownUsd;
+    private costUnknownRuns;
+    constructor(opts: ExploreOptions<S>);
+    private rng;
+    private binId;
+    private allocate;
+    private objectiveContext;
+    /** Mirrors control-runtime: stop once accumulated KNOWN cost ≥ the ceiling. */
+    private costExhausted;
+    /** Fold one run's cost in: null counts as unknown (never $0); a known cost
+     *  accrues toward the budget, lands in the ledger, and fires `onCost`. */
+    /** Returns the run's known cost, `null` when tracked-but-unknown, `undefined`
+     *  when cost tracking is not wired — the log row mirrors this exactly. */
+    private recordRunCost;
+    /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
+    private elitesFor;
+    /** One allocate → propose → evaluate → gate → archive round. */
+    step(): Promise<{
+        runs: number;
+        findings: Finding<S>[];
+    }>;
+    /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
+     *  or no progress is made. */
+    run(): Promise<CapsuleData<S>>;
+    coverage(): CoverageCell[];
+    findings(): Finding<S>[];
+    capsule(): CapsuleData<S>;
+}
+
+/**
+ * `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.
+ *
+ * One call: explore the space to budget with the adversarial objective and
+ * return the capsule. For agent-driven, incremental, or multi-objective use,
+ * construct a `BehaviorExplorer` and drive it via `makeExploreTools`.
+ */
+
+type FuzzAgentOptions<S> = Omit<ExploreOptions<S>, 'objective'> & {
+    /** Score strictly below this is a candidate failure. Default 0.5. */
+    failureThreshold?: number;
+};
+declare function fuzzAgent<S>(opts: FuzzAgentOptions<S>): Promise<{
+    capsule: CapsuleData<S>;
+}>;
+
+/**
+ * Validity gates — what separates a fuzzer from a slop generator.
+ *
+ * A notable candidate is admitted only when it is fair and reproducible. None of
+ * these are on by default: the live wiring opts in, so reported findings carry
+ * their proof.
+ */
+
+/** Combine gate sets; a candidate must pass every gate in every set. */
+declare function composeGates<S>(...sets: Array<ValidityGates<S> | undefined>): ValidityGates<S>;
+/**
+ * Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the
+ * flagged scenario; keep the finding only when the rephrase ALSO scores below the
+ * threshold. A finding that flips under a cosmetic rewrite was keyed to surface
+ * form, not the task — a false signal we must not report. Costs one extra
+ * evaluation per candidate (candidates are rare, so cheap).
+ */
+declare function perturbationStabilityGate<S>(opts: {
+    evaluate: Evaluator<S>;
+    /** Produce a semantic-preserving rephrase. Return null to skip (treated as pass). */
+    perturb: (scenario: S) => S | null;
+    /** Score strictly below this still counts as failing. Default 0.5. */
+    failureThreshold?: number;
+}): ValidityGates<S>;
+/**
+ * Severity-floor gate. Reject borderline candidates whose score sits in a band
+ * just under the threshold — judge noise, not a real defect.
+ */
+declare function severityFloorGate<S>(opts: {
+    failureThreshold?: number;
+    margin?: number;
+}): ValidityGates<S>;
+
+/**
+ * Shipped policies for the exploration engine.
+ *
+ * `Proposer` is a plain function type — an agent running a generator skill IS a
+ * proposer (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed. `mutationProposer`
+ * builds the deterministic, LLM-free one from mutation operators. Objectives are
+ * interfaces because the engine reads `kind` + `threshold` off them.
+ */
+
+/**
+ * Perturbation-based search: apply the cell's mutation operators to the current
+ * elites + seeds, deduping by id. Elites first — mutating the most interesting
+ * scenario found so far is what makes the search deepen across rounds.
+ */
+declare function mutationProposer<S>(opts: {
+    mutationsFor: (cell: Cell) => AdversarialMutation<S>[];
+    scenarioId: (s: S) => string;
+}): Proposer<S>;
+/** Adversarial: a low headline score is interesting — find where the agent fails. */
+declare function adversarialObjective(threshold?: number): Objective;
+/**
+ * Novelty: interesting when far from the archive in score AND measured behavior
+ * descriptor — quality-diversity's diversity pressure; drives corpus growth
+ * rather than re-finding the same hole.
+ */
+declare function noveltyObjective(threshold?: number): Objective;
+
+/**
+ * Agent-drivable surface over a live exploration session.
+ *
+ * Framework-neutral tool defs ({name, description, parameters: JSON Schema,
+ * handler}) so the on-demand agent — not a batch script — drives the search:
+ * step it, read coverage, inspect findings, render the capsule. Transport
+ * encodings (OpenAI function shape, MCP) are one-line mappings the host owns.
+ */
+
+interface ExploreToolDef {
+    name: string;
+    description: string;
+    /** JSON Schema (draft-07+) for the arguments. */
+    parameters: Record<string, unknown>;
+    handler: (args: unknown, ctx?: {
+        signal?: AbortSignal;
+    }) => Promise<unknown>;
+}
+declare function makeExploreTools<S>(explorer: BehaviorExplorer<S>): ExploreToolDef[];
+
+export { type ArchiveEntry, BehaviorExplorer, type BehaviorSpace, type BuildCapsuleInput, type CapsuleData, type Cell, type CoverageCell, type EvalRecord, type Evaluation, type Evaluator, type ExploreEvent, type ExploreOptions, type ExploreToolDef, type Finding, type FuzzAgentOptions, type Objective, type ObjectiveContext, type ProposeContext, type Proposer, type RenderCapsuleOptions, type RunCost, type SpaceAxis, type ValidityGates, adversarialObjective, buildCapsule, buildCoverage, cellId, composeGates, enumerateCells, fuzzAgent, makeExploreTools, mutationProposer, noveltyObjective, perturbationStabilityGate, renderCapsuleHtml, severityFloorGate };