@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/provenance-P-bCL2Fo.d.ts

@@ -1,441 +0,0 @@
-import { o as Mutator, I as ImprovementDriver, S as Scenario, G as Gate, k as GateResult, j as GateContext, g as CampaignResult, M as MutableSurface, c as GateDecision } from './types-BU-7W85F.js';
-import { R as RedTeamCase } from './red-team-BWdoyleI.js';
-import { R as RunRecord } from './run-record-e7vj1uZQ.js';
-import { D as Direction } from './pareto-E-pembql.js';
-import { a as PairedBootstrapResult } from './statistics-CCJpTGOS.js';
-import { b as RunCampaignOptions, C as CampaignStorage } from './run-improvement-loop-DBahB8Ax.js';
-import { HostedClient, TraceSpanEvent } from './hosted/index.js';
-
-/**
- * @experimental
- *
- * `evolutionaryDriver` — adapts a stateless `Mutator` (population mutation:
- * GEPA / AxGEPA / reflective-mutation) into an `ImprovementDriver`. This is
- * the evolutionary strategy: each generation, mutate the current best surface
- * into N candidates, measure, select. No generation memory beyond the current
- * surface; the loop body handles ranking + promotion.
- *
- * The reflective alternative is agent-runtime's `improvementDriver` with a
- * `reflectiveGenerator` / `agenticGenerator`: it reasons over the report +
- * trace findings to propose targeted edits rather than blind mutations. Both
- * conform to `ImprovementDriver`; the improvement loop is identical regardless
- * of which drives it.
- */
-
-interface EvolutionaryDriverOptions<TFindings = unknown> {
-    mutator: Mutator<TFindings>;
-    /** External findings fed to the mutator each generation. Default: []. */
-    findings?: TFindings[];
-}
-declare function evolutionaryDriver<TFindings = unknown>(opts: EvolutionaryDriverOptions<TFindings>): ImprovementDriver<TFindings>;
-
-/**
- * @experimental
- *
- * Compose multiple `Gate` implementations — every gate must pass for the
- * composite to ship. Closes the alignment reviewer's "default-only
- * heldOutGate + costGate would happily promote a reward-hacked prompt"
- * concern by making safety gates first-class composable defaults.
- */
-
-/** Compose gates — all must `ship` for the composite to `ship`. First
- *  non-ship verdict short-circuits the composite verdict, but ALL gates run
- *  (so the result records every gate's reason — useful for diagnostics). */
-declare function composeGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(...gates: Array<Gate<TArtifact, TScenario>>): Gate<TArtifact, TScenario>;
-
-/**
- * @experimental
- *
- * `defaultProductionGate` — composes the substrate's existing safety
- * primitives (red-team / reward-hacking / canary / heldout) into a single
- * Gate.decide shape. Closes the alignment + Anthropic-SI reviewers' "safety
- * primitives are off the critical path" blocker.
- *
- * The composition is opinionated — when consumers wire `runImprovementLoop`,
- * THIS gate is the default. Consumers can still pass a custom gate to
- * override; the recommended pattern is to compose THIS gate with whatever
- * extra domain-specific gates they need (`composeGate(defaultProductionGate(...), customGate)`).
- */
-
-interface DefaultProductionGateOptions {
-    /** Required: scenarios held out from training; substrate compares
-     *  candidate-on-holdout vs baseline-on-holdout. */
-    holdoutScenarios: Scenario[];
-    /** Minimum held-out lift the **paired-bootstrap CI lower bound** must clear
-     *  to ship — NOT a point estimate. Default 0 ⇒ "confidently positive at the
-     *  confidence level". Interpreted in the judge's native composite scale (set
-     *  e.g. 2 for a 0-100 rubric to require a ≥2-point significant gain). */
-    deltaThreshold?: number;
-    /** Confidence level for the held-out + dimension bootstraps. Default 0.95. */
-    confidence?: number;
-    /** Bootstrap resamples. Default 2000. */
-    bootstrapResamples?: number;
-    /** Fixed bootstrap seed for a deterministic verdict. Default 1337. */
-    bootstrapSeed?: number;
-    /** Minimum paired holdout observations (scenarios × reps) before a
-     *  significance claim is allowed; below it the gate HOLDS with `few_runs`
-     *  rather than reading a degenerate CI. Default 3. */
-    minProductiveRuns?: number;
-    /** Critical judge dimensions that must NOT significantly regress even when
-     *  the net composite rises (anti-Goodhart). The gate HOLDS if any listed
-     *  dimension's paired-delta CI lower bound < −`regressionTolerance`. E.g.
-     *  `['hallucination_free']` for a legal agent. */
-    criticalDimensions?: string[];
-    /** Tolerance for the per-dimension regression guard, in the dimension's
-     *  native scale. When omitted it auto-scales off observed magnitudes:
-     *  0.05 on [0,1], 5 on 0-100. */
-    regressionTolerance?: number;
-    /** Total $ budget for ALL cells in this campaign — including baseline + candidate.
-     *  Composite verdict refuses to ship when spend exceeded budget. */
-    budgetUsd?: number;
-    /** Red-team cases to probe candidate outputs against. When omitted the
-     *  substrate uses `DEFAULT_RED_TEAM_CORPUS`. Provide a domain-specific
-     *  battery for tighter coverage. */
-    redTeamBattery?: RedTeamCase[];
-    /** Run records (oldest-first) needed for the reward-hacking detector.
-     *  Substrate populates from prior production-loop generations. */
-    recentRuns?: RunRecord[];
-    /** When true, the gate refuses to ship if the reward-hacking detector
-     *  fires at the `gaming` severity. Default true. */
-    blockOnRewardHackingGaming?: boolean;
-}
-declare function defaultProductionGate<TArtifact, TScenario extends Scenario>(options: DefaultProductionGateOptions): Gate<TArtifact, TScenario>;
-
-/**
- * @experimental
- *
- * Thin Gate adapter — exposes delta-threshold-on-holdout as a composable
- * `Gate`. Use when you want held-out as one of N composed gates instead of
- * the full `defaultProductionGate` stack.
- */
-
-interface HeldOutGateOptions<TScenario extends Scenario = Scenario> {
-    scenarios: TScenario[];
-    deltaThreshold?: number;
-}
-declare function heldOutGate<TArtifact, TScenario extends Scenario>(options: HeldOutGateOptions<TScenario>): Gate<TArtifact, TScenario>;
-
-/**
- * @experimental
- *
- * Promotion policy over the evidence VECTOR — the substrate's answer to "never
- * collapse the multi-objective promotion decision into one scalar." A
- * `defaultProductionGate` is one opinionated composition; this module factors
- * the decision into two reusable pieces so MANY policies can compete over the
- * SAME evidence (the quant-desk pattern: one evidence bus, plural strategies):
- *
- *   buildEvidenceVector(ctx, objectives, opts) -> EvidenceVector   // the bus
- *   PromotionPolicy = (ev: EvidenceVector) => GateResult           // a strategy
- *   paretoPolicy(ev)                                               // the default strategy
- *   paretoSignificanceGate(options): Gate                          // bus + policy as a Gate
- *
- * The Pareto policy is SYMMETRIC multi-objective: every objective is BOTH a
- * potential gain source AND a safety floor (unlike `defaultProductionGate`,
- * where only `composite` can win and `criticalDimensions` are pure floors). A
- * candidate ships iff it weakly DOMINATES the baseline at the confidence level —
- * no objective credibly worse (CI floor breach) AND at least one objective
- * credibly better (CI gain). Insufficient evidence on ANY axis -> need_more_work
- * (NOT folded into hold: "gather more reps" and "reject" are different actions).
- *
- * Cost/latency are NOT CI axes here — `GateContext` carries only an aggregate
- * per-side cost, no per-cell observation vector to bootstrap. Treat them as hard
- * constraints (compose with a budget gate via `composeGate`), not faked CIs.
- */
-
-/** Where an objective's per-cell scalar comes from. `composite` reads the
- *  judge's composite; `dimension` reads a named per-dimension score. */
-type ObjectiveSource = {
-    kind: 'composite';
-} | {
-    kind: 'dimension';
-    dimension: string;
-};
-interface PromotionObjective {
-    /** Stable label used in reports + `contributingGates`. */
-    name: string;
-    source: ObjectiveSource;
-    /** 'maximize' (quality dims) or 'minimize' (error/risk/length dims). Orients
-     *  the paired delta so a positive bootstrap always means "candidate better". */
-    direction: Direction;
-    /** The good-direction paired-delta CI lower bound must EXCEED this to count
-     *  as a significant gain on this axis. Interpreted in the judge's native
-     *  scale. Default 0 (⇒ "confidently better"). */
-    gainThreshold?: number;
-    /** A floor breach (regression) is declared when the good-direction CI lower
-     *  bound is below −floorTolerance. When omitted it auto-scales off observed
-     *  magnitudes (0.05 on [0,1], 5 on 0-100), matching `dimensionRegressions`. */
-    floorTolerance?: number;
-}
-/** Per-axis verdict from the good-direction paired bootstrap. */
-type AxisVerdict = 'improved' | 'regressed' | 'flat' | 'few_runs';
-interface AxisEvidence {
-    name: string;
-    source: ObjectiveSource;
-    direction: Direction;
-    /** Paired bootstrap on the GOOD-DIRECTION delta (oriented by `direction`):
-     *  a positive value means the candidate is better on this axis. */
-    bootstrap: PairedBootstrapResult;
-    /** Paired observations contributing to this axis. */
-    n: number;
-    gainThreshold: number;
-    floorTolerance: number;
-    verdict: AxisVerdict;
-}
-interface EvidenceVector {
-    /** One entry per objective — NOTHING averaged across axes. */
-    axes: AxisEvidence[];
-    /** Smallest paired n across axes that produced observations — the binding
-     *  evidence-sufficiency constraint. 0 when no axis produced observations. */
-    minN: number;
-    /** Aggregate per-side cost from the gate context (a constraint input, not a
-     *  CI axis — see the module header). */
-    cost: {
-        candidate: number;
-        baseline: number;
-    };
-}
-/** A promotion strategy: a pure function from the evidence vector to a verdict.
- *  Many policies can run over the same `EvidenceVector` and disagree — that's
- *  the point (competing strategies, shared evidence). */
-type PromotionPolicy = (ev: EvidenceVector) => GateResult;
-interface BuildEvidenceVectorOptions {
-    /** Minimum paired observations before an axis can claim significance; below
-     *  it the axis is `few_runs`. Default 3. */
-    minProductiveRuns?: number;
-    /** Confidence level for every axis bootstrap. Default 0.95. */
-    confidence?: number;
-    /** Bootstrap resamples. Default 2000. */
-    resamples?: number;
-    /** Fixed bootstrap seed for a deterministic, reproducible verdict. Default 1337. */
-    seed?: number;
-}
-/**
- * The Evidence Bus. For each objective, pair candidate vs baseline by full
- * cellId and bootstrap a CI on the good-direction paired delta. Reuses the
- * exact `pairHoldout` + `pairedBootstrap` machinery the held-out gate uses, so
- * a single source of truth governs pairing granularity + scale handling.
- */
-declare function buildEvidenceVector<TArtifact, TScenario extends Scenario>(ctx: GateContext<TArtifact, TScenario>, objectives: PromotionObjective[], opts?: BuildEvidenceVectorOptions): EvidenceVector;
-/**
- * The default strategy: symmetric multi-objective Pareto significance. Ship iff
- * the candidate weakly dominates the baseline at the confidence level — no axis
- * credibly worse AND ≥1 axis credibly better. Floor breach on any axis → hold
- * (anti-Goodhart, dominates everything). Insufficient evidence on any axis →
- * need_more_work. Statistically equivalent → hold (never ship noise).
- */
-declare const paretoPolicy: PromotionPolicy;
-interface ParetoSignificanceGateOptions extends BuildEvidenceVectorOptions {
-    /** The objective vector. Every axis is both a gain source and a safety floor. */
-    objectives: PromotionObjective[];
-    /** Strategy applied to the evidence vector. Default `paretoPolicy`. Override
-     *  to run a stricter/looser strategy over the SAME bus (competing policies). */
-    policy?: PromotionPolicy;
-    /** Override the gate name in reports. */
-    name?: string;
-}
-/**
- * Wrap the bus + a policy as a `Gate`. Plugs into the existing
- * `runImprovementLoop({ gate })` slot and composes via `composeGate`; default
- * loop behavior is unchanged because consumers opt in by passing this gate.
- */
-declare function paretoSignificanceGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(options: ParetoSignificanceGateOptions): Gate<TArtifact, TScenario>;
-
-/**
- * @experimental
- *
- * `runEval` — the simplest preset over `runCampaign`. No optimizer, no
- * gate, no auto-PR. Just: run scenarios through dispatch, score with
- * judges, return CampaignResult.
- *
- * The 80% case for consumers who want a scorecard, not an improvement loop.
- */
-
-interface RunEvalOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'runDir'> {
-    runDir: string;
-}
-declare function runEval<TScenario extends Scenario, TArtifact>(opts: RunEvalOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
-
-/**
- * @experimental
- *
- * Loop provenance — the durable, queryable record of WHAT a self-improvement
- * loop did and WHY, plus the OTel spans that let an OTLP collector pivot from
- * an eval-run to the underlying candidate→cell→gate→promote chain.
- *
- * Two artifacts, one source of truth:
- *
- *   1. `LoopProvenanceRecord` — a structured JSON record capturing every
- *      candidate (surfaceHash + label + rationale), its measured composite,
- *      the gate decision + reasons + delta, the held-out lift, the explicit
- *      baseline→candidate diff, and BACKEND PROVENANCE (the
- *      `assertRealBackend` verdict + worker call count + model). This is the
- *      ingestable audit artifact: the +lift recomputes from it, the "because
- *      Z" rationale survives in it, and a stub backend is detectable from it.
- *
- *   2. `loopProvenanceSpans()` — the same chain emitted as OTLP-ingestable
- *      `TraceSpanEvent`s, pivoted on the substrate's standard
- *      `tangle.runId` / `tangle.scenarioId` / `tangle.cellId` /
- *      `tangle.generation` attributes (the same pivots `/adapters/otel`
- *      reads). The hosted `/v1/ingest/traces` endpoint receives the FULL loop,
- *      not just the `cost.*` spans `runCampaign` already emits per cell.
- *
- * The record is built from the substrate's own loop result + the per-call
- * `RunRecord`s the worker emitted — no new measurement, no recomputation that
- * could drift from what the gate actually saw.
- */
-
-/** Stable sha256 (full hex) of a surface's effective text. Code surfaces hash
- *  their worktree+base identity since the content lives in git. Distinct from
- *  `surfaceHash` (16-char content fingerprint used as a loop identity key);
- *  this is the byte-identical-verifiable content hash the provenance record +
- *  `RunRecord.promptHash` carry. */
-declare function surfaceContentHash(surface: MutableSurface): string;
-interface LoopProvenanceCandidate {
-    /** Generation index this candidate was proposed in. */
-    generation: number;
-    /** 16-char loop-identity fingerprint (matches `GenerationCandidate.surfaceHash`). */
-    surfaceHash: string;
-    /** Full sha256 content hash — byte-identical-verifiable. */
-    contentHash: string;
-    /** Driver label, when the driver returned a `ProposedCandidate`. */
-    label?: string;
-    /** Driver rationale — the "because Z". When the driver returned a bare
-     *  surface (blind mutator) this is absent. */
-    rationale?: string;
-    /** Mean composite this candidate scored on the search split. */
-    composite: number;
-    /** Whether this candidate was promoted out of its generation. */
-    promoted: boolean;
-}
-interface LoopProvenanceBackend {
-    /** `assertRealBackend`-grade verdict over the worker call records. */
-    verdict: 'real' | 'mixed' | 'stub';
-    /** Number of worker LLM calls captured (the audit's "worker call count"). */
-    workerCallCount: number;
-    /** Distinct model ids observed across worker calls. */
-    models: string[];
-    totalInputTokens: number;
-    totalOutputTokens: number;
-    totalCostUsd: number;
-}
-/**
- * The durable provenance record. Aligns to the hosted `EvalRunEvent` path but
- * ADDS the rationale + the explicit baseline→candidate diff (both omitted from
- * the bare hosted event) + backend provenance.
- */
-interface LoopProvenanceRecord {
-    schema: 'tangle.loop-provenance.v1';
-    runId: string;
-    runDir: string;
-    timestamp: string;
-    /** Baseline + winner surface content hashes — distinguishable, byte-verifiable. */
-    baselineContentHash: string;
-    winnerContentHash: string;
-    /** Driver label/rationale for the promoted change. Absent ⇒ winner == baseline. */
-    winnerLabel?: string;
-    winnerRationale?: string;
-    /** The explicit baseline→winner unified diff the gate decided on. */
-    diff: string;
-    /** Every candidate across every generation, each carrying its rationale. */
-    candidates: LoopProvenanceCandidate[];
-    /** The gate verdict — decision + reasons + contributing gates + delta. */
-    gate: {
-        decision: GateDecision;
-        reasons: string[];
-        delta?: number;
-        contributingGates: Array<{
-            name: string;
-            passed: boolean;
-        }>;
-    };
-    /** baseline-on-holdout composite mean. */
-    baselineHoldoutComposite: number;
-    /** winner-on-holdout composite mean. */
-    winnerHoldoutComposite: number;
-    /** winnerHoldout - baselineHoldout — RECOMPUTABLE from this record. */
-    heldOutLift: number;
-    /** Backend provenance: stub-vs-real verdict + worker call count + models. */
-    backend: LoopProvenanceBackend;
-    totalCostUsd: number;
-    totalDurationMs: number;
-}
-interface BuildLoopProvenanceArgs<TArtifact, TScenario extends Scenario> {
-    runId: string;
-    runDir: string;
-    timestamp: string;
-    baselineSurface: MutableSurface;
-    winnerSurface: MutableSurface;
-    winnerLabel?: string;
-    winnerRationale?: string;
-    diff: string;
-    /** Per-generation candidate records straight off the loop result. */
-    generations: Array<{
-        generationIndex: number;
-        candidates: Array<{
-            surfaceHash: string;
-            composite: number;
-            label?: string;
-            rationale?: string;
-        }>;
-        promoted: string[];
-        /** Surfaces measured this generation, keyed positionally to candidates so
-         *  the content hash can be computed from the real surface text. */
-        surfaces: Array<{
-            surfaceHash: string;
-            surface: MutableSurface;
-        }>;
-    }>;
-    gate: GateResult;
-    baselineOnHoldout: CampaignResult<TArtifact, TScenario>;
-    winnerOnHoldout: CampaignResult<TArtifact, TScenario>;
-    /** Worker call records — the source for backend provenance. */
-    workerRecords: ReadonlyArray<RunRecord>;
-    totalCostUsd: number;
-    totalDurationMs: number;
-}
-/** Build the durable provenance record from a completed loop result. */
-declare function buildLoopProvenanceRecord<TArtifact, TScenario extends Scenario>(args: BuildLoopProvenanceArgs<TArtifact, TScenario>): LoopProvenanceRecord;
-/**
- * Build the loop's OTLP-ingestable spans from a provenance record. One root
- * span per loop (`tangle.runId`), one span per generation, one span per
- * candidate (carrying its surfaceHash + label), and one span for the gate
- * decision (carrying reasons + delta + lift). Candidate + gate spans pivot on
- * the same `tangle.runId` / `tangle.generation` attributes `/adapters/otel`
- * reads, so the hosted collector reconstructs the full tree.
- *
- * Times are synthesized monotonically off a single base so the span tree is
- * orderable; the substrate does not retain per-candidate wall-clock starts.
- */
-declare function loopProvenanceSpans(record: LoopProvenanceRecord, opts?: {
-    baseTimeMs?: number;
-}): TraceSpanEvent[];
-/** Canonical durable paths under the run dir. */
-declare function provenanceRecordPath(runDir: string): string;
-declare function provenanceSpansPath(runDir: string): string;
-interface EmitLoopProvenanceResult {
-    record: LoopProvenanceRecord;
-    spans: TraceSpanEvent[];
-    /** Absolute paths the record + spans were written to, when storage persists. */
-    recordPath: string;
-    spansPath: string;
-}
-interface EmitLoopProvenanceArgs<TArtifact, TScenario extends Scenario> extends BuildLoopProvenanceArgs<TArtifact, TScenario> {
-    /** Storage the record + spans are written through. */
-    storage: CampaignStorage;
-    /** When set, the spans are also shipped to the hosted `/v1/ingest/traces`
-     *  endpoint so the collector receives the full loop, not just `cost.*`. */
-    hostedClient?: HostedClient;
-}
-/**
- * Build the provenance record + OTel spans and persist them durably under the
- * run dir (and ship spans to a hosted collector when one is wired). Returns
- * both artifacts so the caller can assert on / re-derive from them.
- *
- * Fail-loud: the durable write throws on storage failure (a swallowed write is
- * exactly the "emitted but lost" failure this closes). The hosted span ship is
- * the one best-effort leg — its failure is logged, not thrown, so an offline
- * collector never fails the loop (the durable artifact is the source of truth).
- */
-declare function emitLoopProvenance<TArtifact, TScenario extends Scenario>(args: EmitLoopProvenanceArgs<TArtifact, TScenario>): Promise<EmitLoopProvenanceResult>;
-
-export { type AxisEvidence as A, type BuildEvidenceVectorOptions as B, type DefaultProductionGateOptions as D, type EvidenceVector as E, type HeldOutGateOptions as H, type LoopProvenanceRecord as L, type ObjectiveSource as O, type ParetoSignificanceGateOptions as P, type RunEvalOptions as R, type AxisVerdict as a, type EvolutionaryDriverOptions as b, type PromotionObjective as c, type PromotionPolicy as d, buildEvidenceVector as e, composeGate as f, defaultProductionGate as g, evolutionaryDriver as h, heldOutGate as i, paretoSignificanceGate as j, type BuildLoopProvenanceArgs as k, type EmitLoopProvenanceArgs as l, type EmitLoopProvenanceResult as m, type LoopProvenanceBackend as n, type LoopProvenanceCandidate as o, paretoPolicy as p, buildLoopProvenanceRecord as q, runEval as r, emitLoopProvenance as s, loopProvenanceSpans as t, provenanceRecordPath as u, provenanceSpansPath as v, surfaceContentHash as w };

package/dist/query-B7GGjRox.d.ts

@@ -1,32 +0,0 @@
-import { L as LlmSpan, J as JudgeSpan, R as Run, F as FailureClass, T as ToolSpan } from './schema-m0gsnbt3.js';
-import { T as TraceStore } from './store-BcFXE6LG.js';
-
-/**
- * Typed query helpers over TraceStore.
- *
- * Not a full SQL engine — a minimal, composable set of operators that
- * cover the canned-pipeline use cases. For ad-hoc analytics, persist to
- * NDJSON and point DuckDB at it; the schema is stable so external SQL
- * tooling works out of the box.
- */
-
-declare function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]>;
-declare function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]>;
-declare function toolSpans(store: TraceStore, runId?: string, toolName?: string): Promise<ToolSpan[]>;
-declare function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]>;
-/** Group spans by any key selector. */
-declare function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]>;
-/** Hash tool arguments to an orderless-key-stable string for de-duplication. */
-declare function argHash(args: unknown): string;
-/** Sum an LLM-span array into aggregate token + cost. */
-declare function aggregateLlm(spans: LlmSpan[]): {
-    inputTokens: number;
-    outputTokens: number;
-    cachedTokens: number;
-    reasoningTokens: number;
-    costUsd: number;
-};
-/** Pick the outcome's failure class when present, else derive 'success' from run status. */
-declare function runFailureClass(run: Run): FailureClass;
-
-export { aggregateLlm as a, argHash as b, runsForScenario as c, groupBy as g, judgeSpans as j, llmSpans as l, runFailureClass as r, toolSpans as t };