@tangle-network/agent-eval 0.144.6 → 0.144.8

package/dist/promotion-policy-ChWhTDBH.d.ts

@@ -0,0 +1,289 @@
+import { R as Scenario, h as GateContext, p as Gate, v as GateResult } from "./types-DYuNHo9R.js";
+import { y as PairedBootstrapResult } from "./statistics-D6Uebe_4.js";
+import { i as PairedMcNemarEvidence, r as PairedDecisionStatistic, t as PairedDecisionMethod } from "./paired-promotion-decision-B6zJ3gYM.js";
+//#region src/campaign/gates/power-preflight.d.ts
+/**
+ * Power preflight — "can this budget detect the effect you are hunting?"
+ *
+ * The failure it prevents (measured, twice): a live prompt-improvement campaign ran
+ * 333 sandbox cells over 5.6 hours and produced a +0.08 holdout lift the ship gate
+ * (paired bootstrap, CI.low > 0.05) could not distinguish from zero — because at
+ * that holdout size and worker variance the MINIMUM DETECTABLE lift was larger than
+ * any effect a prompt change plausibly produces. The budget was spent learning what
+ * a 30-second calculation on the baseline cells already knew. No eval framework we
+ * know of surfaces this; every underpowered improvement run everywhere ends in an
+ * uninformative "hold".
+ *
+ * Model: the ship rule is `CI.low(paired Δ) > deltaThreshold`. Approximating the
+ * bootstrap CI as normal, `CI.low ≈ effect − z·sd_Δ/√n`, so the smallest shippable
+ * true effect is `MDE = deltaThreshold + z·sd_Δ/√n`. The paired-delta SD is unknown
+ * before the candidate exists; we bound it by the zero-correlation case
+ * `sd_Δ ≤ √2·sd_baseline` — a CONSERVATIVE (upper) MDE, which is the correct
+ * direction for a warning. Pairing is per cell (`scenario:rep`), so reps multiply n.
+ *
+ * Standalone by design: feed it any baseline composites (a `gate:'none'` run, a
+ * live-proof table) BEFORE budgeting the real search; `selfImprove` also attaches
+ * it to every result and warns when the run was structurally unable to ship.
+ */
+interface PowerPreflightOptions {
+  /** Per-cell baseline composites on the HOLDOUT scenarios (one per scenario:rep cell). */
+  baselineComposites: number[];
+  /** Paired observations the budgeted comparison will produce
+   *  (holdout scenarios × reps). Defaults to `baselineComposites.length`. */
+  pairedN?: number;
+  /** The ship gate's effect-size threshold. Default 0.05 (defaultProductionGate). */
+  deltaThreshold?: number;
+  /** CI confidence the gate uses. Default 0.95. */
+  confidence?: number;
+  /** True when the holdout is scored by the SAME judge/scorer family as the gate
+   *  (selfImprove's default composition — one judge scores everything). Under a
+   *  shared channel, raising paired n reduces only the IDIOSYNCRATIC noise share;
+   *  systematic judge bias is untouched, so the MDE here is a lower bound and the
+   *  only full debiaser is an independent second scoring channel
+   *  (recursive-self-improvement S1c, closed form in EXP-023 P0). Default false. */
+  sharedScorerChannel?: boolean;
+}
+interface PowerPreflight {
+  /** Paired observations the comparison will have. */
+  n: number;
+  /** Baseline per-cell composite standard deviation (the variance the effect must beat). */
+  sd: number;
+  /** Minimum detectable lift: the smallest TRUE effect the gate could ship at this budget. */
+  mde: number;
+  /** Baseline holdout composite mean. */
+  baselineMean: number;
+  /** Headroom to a perfect 1.0 composite (the largest achievable lift on a [0,1] judge). */
+  headroom: number;
+  /** True when even the largest achievable effect (headroom) is below the MDE —
+   *  the run is structurally unable to ship regardless of proposal quality.
+   *  Only asserted for [0,1]-scaled judges (see `scaleAssumed`). */
+  underpowered: boolean;
+  /** True when composites look [0,1]-scaled; headroom/underpowered are only
+   *  meaningful under that convention (0-100 judges get mde/sd/n but no verdict). */
+  scaleAssumed: boolean;
+  deltaThreshold: number;
+  confidence: number;
+  /** Set when the holdout shares the gate's scoring channel: more cells cannot
+   *  buy back systematic judge bias — treat the MDE as a lower bound. */
+  sharedChannelCaveat?: string;
+  /** One actionable sentence for humans and logs. */
+  recommendation: string;
+}
+/** Estimate the minimum detectable lift a paired-holdout improvement run can
+ *  ship at a given budget, from the baseline holdout composites — call it BEFORE
+ *  spending a search to learn whether the effect you are hunting is even
+ *  observable at this holdout size and worker variance. */
+declare function powerPreflight(opts: PowerPreflightOptions): PowerPreflight;
+//#endregion
+//#region src/pareto.d.ts
+/**
+ * Pareto frontier — multi-objective optimization over candidate runs.
+ *
+ * Lifted from ADC pareto.ts and blueprint-agent frontier.ts. When you're
+ * trading off (cost, latency, quality) or (passRate, tokenBudget,
+ * ttfb), you rarely have a single "winner" — you have a set of
+ * non-dominated candidates. This module exposes:
+ *
+ *   - `paretoFrontier`: filter a set of candidates to the non-dominated ones
+ *   - `dominates`: does A dominate B across all objectives?
+ *
+ * Each objective is declared with a direction: 'maximize' (higher=better)
+ * or 'minimize' (lower=better). Candidates are any object; pass an
+ * `objective(candidate)` accessor.
+ */
+type Direction = 'maximize' | 'minimize';
+interface Objective<T> {
+  /** Stable label used in reports. */
+  name: string;
+  direction: Direction;
+  value: (candidate: T) => number;
+}
+interface ParetoResult<T> {
+  frontier: T[];
+  dominated: T[];
+  /** Index map: frontier[i] dominates each of dominatedBy[i]. */
+  dominanceMap: Array<{
+    dominator: T;
+    dominated: T[];
+  }>;
+}
+/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
+declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
+/**
+ * Compute the non-dominated frontier. Candidates with NaN/Infinity on any
+ * objective are excluded (can't rank them). A candidate enters the frontier
+ * iff no other candidate dominates it.
+ */
+declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
+/**
+ * Weighted-sum scalarisation. Use as a tie-break / single-winner selector
+ * when callers don't want to consume a frontier. Each objective contributes
+ * its normalised value (0..1 via min-max across the candidate pool) times
+ * its weight; missing weights default to 1/N.
+ *
+ * Direction is honoured automatically — `minimize` axes have their values
+ * inverted before scaling so "higher scalar = better" always holds.
+ */
+declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], options?: {
+  weights?: Partial<Record<string, number>>;
+}): Array<{
+  candidate: T;
+  score: number;
+}>;
+/**
+ * NSGA-II crowding distance — secondary sort for ties on the frontier.
+ *
+ * When the Pareto front collapses to a single point (or many candidates tie
+ * on dominance), naive selection picks arbitrarily and the population
+ * degenerates over generations. NSGA-II preserves diversity by preferring
+ * candidates with more empty space around them on the frontier.
+ *
+ * Returns an array of `{ candidate, distance }` in the SAME order as the
+ * input. Higher distance = more isolated = should be preferred when
+ * preserving diversity.
+ */
+declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+  candidate: T;
+  distance: number;
+}>;
+/**
+ * Pareto frontier with tie-break by crowding distance — the canonical
+ * NSGA-II selection step. Returns the frontier sorted by descending crowding
+ * distance so callers can `.slice(0, k)` to pick K diverse winners.
+ */
+declare function paretoFrontierWithCrowding<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+  candidate: T;
+  distance: number;
+}>;
+//#endregion
+//#region src/campaign/gates/promotion-policy.d.ts
+/** Where an objective's per-cell scalar comes from. `composite` reads the
+ *  judge's composite; `dimension` reads a named per-dimension score. */
+type ObjectiveSource = {
+  kind: 'composite';
+} | {
+  kind: 'dimension';
+  dimension: string;
+};
+interface PromotionObjective {
+  /** Stable label used in reports + `contributingGates`. */
+  name: string;
+  source: ObjectiveSource;
+  /** 'maximize' (quality dims) or 'minimize' (error/risk/length dims). Orients
+   *  the paired delta so a positive bootstrap always means "candidate better". */
+  direction: Direction;
+  /** The good-direction paired-delta CI lower bound must EXCEED this to count
+   *  as a significant gain on this axis. Interpreted in the judge's native
+   *  scale. Default 0 (⇒ "confidently better"). */
+  gainThreshold?: number;
+  /** A floor breach (regression) is declared when the good-direction CI lower
+   *  bound is below −floorTolerance, or when the exact small-sample test proves
+   *  a drop past it. When omitted it auto-scales off observed magnitudes
+   *  (0.05 on [0,1], 5 on 0-100), matching `dimensionRegressions`. */
+  floorTolerance?: number;
+}
+/** Per-axis verdict from the good-direction paired bootstrap. */
+type AxisVerdict = 'improved' | 'regressed' | 'flat' | 'few_runs';
+interface AxisEvidence {
+  name: string;
+  source: ObjectiveSource;
+  direction: Direction;
+  /** Paired bootstrap on the GOOD-DIRECTION delta (oriented by `direction`):
+   *  a positive value means the candidate is better on this axis.
+   *
+   *  DIAGNOSTIC on a pass/fail axis: there the verdict is decided on Tango's
+   *  score interval instead, because a percentile bootstrap over a three-atom
+   *  delta lattice is not a valid interval at the nonzero margin `floorTolerance`
+   *  and `gainThreshold` create. `ci` carries the interval that decided. */
+  bootstrap: PairedBootstrapResult;
+  /** Which paired statistic `bootstrap.low`/`.high` bracket. `'mean'` unless the
+   *  caller asked for the median — on a pass/fail axis the median and its whole
+   *  CI are pinned at 0 by tie domination and can see neither a gain nor a
+   *  regression. `bootstrap.median` still carries the median point estimate. */
+  bootstrapStatistic: 'median' | 'mean';
+  /** The interval the axis verdict was actually decided on, good-direction and
+   *  in the axis's native units. */
+  ci: {
+    low: number;
+    high: number;
+  };
+  /** Which estimator produced `ci`. */
+  decisionStatistic: PairedDecisionStatistic;
+  /** McNemar's exact evidence on a pass/fail axis; null otherwise. */
+  mcnemar: PairedMcNemarEvidence | null;
+  /** `ci` has zero width — no evidence in either direction, so the axis is
+   *  neither improved nor regressed however the point estimate sits. */
+  indeterminate: boolean;
+  /** Paired observations contributing to this axis. */
+  n: number;
+  minimumRequired: number;
+  decisionMethod: PairedDecisionMethod;
+  gainThreshold: number;
+  floorTolerance: number;
+  verdict: AxisVerdict;
+}
+interface EvidenceVector {
+  /** One entry per objective — NOTHING averaged across axes. */
+  axes: AxisEvidence[];
+  /** Smallest paired n across axes that produced observations — the binding
+   *  evidence-sufficiency constraint. 0 when no axis produced observations. */
+  minN: number;
+  /** Aggregate per-side cost from the gate context (a constraint input, not a
+   *  CI axis — see the module header). */
+  cost: {
+    candidate: number;
+    baseline: number;
+  };
+}
+/** A promotion strategy: a pure function from the evidence vector to a verdict.
+ *  Many policies can run over the same `EvidenceVector` and disagree — that's
+ *  the point (competing strategies, shared evidence). */
+type PromotionPolicy = (ev: EvidenceVector) => GateResult;
+interface BuildEvidenceVectorOptions {
+  /** Minimum paired observations before an axis can claim significance; below
+   *  it the axis is `few_runs`. The exact small-sample test may require more
+   *  observations at the selected confidence. */
+  minProductiveRuns?: number;
+  /** Confidence level for every axis bootstrap. Default 0.95. */
+  confidence?: number;
+  /** Bootstrap resamples. Default 2000. */
+  resamples?: number;
+  /** Fixed bootstrap seed for a deterministic, reproducible verdict. Default 1337. */
+  seed?: number;
+  /** Paired statistic every axis CI is computed on. Default `'mean'` — see
+   *  {@link DECISION_PAIRED_DELTA_STATISTIC} for why the median is not. */
+  statistic?: 'mean' | 'median';
+}
+/**
+ * The Evidence Bus. For each objective, pair candidate vs baseline by full
+ * cellId and bootstrap a CI on the good-direction paired delta. Reuses the
+ * exact `pairHoldout` + `pairedBootstrap` machinery the held-out gate uses, so
+ * a single source of truth governs pairing granularity + scale handling.
+ */
+declare function buildEvidenceVector<TArtifact, TScenario extends Scenario>(ctx: GateContext<TArtifact, TScenario>, objectives: PromotionObjective[], opts?: BuildEvidenceVectorOptions): EvidenceVector;
+/**
+ * The default strategy: symmetric multi-objective Pareto significance. Ship iff
+ * the candidate weakly dominates the baseline at the confidence level — no axis
+ * credibly worse AND ≥1 axis credibly better. Floor breach on any axis → hold
+ * (anti-Goodhart, dominates everything). Insufficient evidence on any axis →
+ * need_more_work. Statistically equivalent → hold (never ship noise).
+ */
+declare const paretoPolicy: PromotionPolicy;
+interface ParetoSignificanceGateOptions extends BuildEvidenceVectorOptions {
+  /** The objective vector. Every axis is both a gain source and a safety floor. */
+  objectives: PromotionObjective[];
+  /** Strategy applied to the evidence vector. Default `paretoPolicy`. Override
+   *  to run a stricter/looser strategy over the SAME bus (competing policies). */
+  policy?: PromotionPolicy;
+  /** Override the gate name in reports. */
+  name?: string;
+}
+/**
+ * Wrap the bus + a policy as a `Gate`. Plugs into the existing
+ * `runImprovementLoop({ gate })` slot and composes via `composeGate`; default
+ * loop behavior is unchanged because consumers opt in by passing this gate.
+ */
+declare function paretoSignificanceGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(options: ParetoSignificanceGateOptions): Gate<TArtifact, TScenario>;
+//#endregion
+export { powerPreflight as S, paretoFrontier as _, ObjectiveSource as a, PowerPreflight as b, PromotionPolicy as c, paretoSignificanceGate as d, Direction as f, dominates as g, crowdingDistance as h, EvidenceVector as i, buildEvidenceVector as l, ParetoResult as m, AxisVerdict as n, ParetoSignificanceGateOptions as o, Objective as p, BuildEvidenceVectorOptions as r, PromotionObjective as s, AxisEvidence as t, paretoPolicy as u, paretoFrontierWithCrowding as v, PowerPreflightOptions as x, scalarScore as y };
+//# sourceMappingURL=promotion-policy-ChWhTDBH.d.ts.map

package/dist/promotion-policy-ChWhTDBH.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"promotion-policy-ChWhTDBH.d.ts","names":[],"sources":["../src/campaign/gates/power-preflight.ts","../src/pareto.ts","../src/campaign/gates/promotion-policy.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;UAwBiB;;EAEf;;;EAGA;;EAEA;;EAEA;;;;;;;EAOA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;;EAGA;EACA;EACA;;;EAGA;;EAEA;;;;;;iBAec,eAAe,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;KClEjD;UAEK,UAAU;;EAEzB;EACA,WAAW;EACX,QAAQ,WAAW;;UAGJ,aAAa;EAC5B,UAAU;EACV,WAAW;;EAEX,cAAc;IAAQ,WAAW;IAAG,WAAW;;;;iBAIjC,UAAU,GAAG,GAAG,GAAG,GAAG,GAAG,YAAY,UAAU;;;;;;iBAmB/C,eAAe,GAAG,YAAY,KAAK,YAAY,UAAU,OAAO,aAAa;;;;;;;;;;iBA4B7E,YAAY,GAC1B,YAAY,KACZ,YAAY,UAAU,MACtB;EAAW,UAAU,QAAQ;IAC5B;EAAQ,WAAW;EAAG;;;;;;;;;;;;;;iBAyCT,iBAAiB,GAC/B,YAAY,KACZ,YAAY,UAAU,OACrB;EAAQ,WAAW;EAAG;;;;;;;iBA6BT,2BAA2B,GACzC,YAAY,KACZ,YAAY,UAAU,OACrB;EAAQ,WAAW;EAAG;;;;;;KCtHb;EAAoB;;EAAwB;EAAmB;;UAE1D;;EAEf;EACA,QAAQ;;;EAGR,WAAW;;;;EAIX;;;;;EAKA;;;KAIU;UAEK;EACf;EACA,QAAQ;EACR,WAAW;;;;;;;;EAQX,WAAW;;;;;EAKX;;;EAGA;IAAM;IAAa;;;EAEnB,mBAAmB;;EAEnB,SAAS;;;EAGT;;EAEA;EACA;EACA,gBAAgB;EAChB;EACA;EACA,SAAS;;UAGM;;EAEf,MAAM;;;EAGN;;;EAGA;IAAQ;IAAmB;;;;;;KAMjB,mBAAmB,IAAI,mBAAmB;UAErC;;;;EAIf;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;;iBASc,oBAAoB,WAAW,kBAAkB,UAC/D,KAAK,YAAY,WAAW,YAC5B,YAAY,sBACZ,OAAM,6BACL;;;;;;;;cAwIU,cAAc;UAiFV,sCAAsC;;EAErD,YAAY;;;EAGZ,SAAS;;EAET;;;;;;;iBAQc,uBAAuB,qBAAqB,kBAAkB,WAAW,UACvF,SAAS,gCACR,KAAK,WAAW"}