agentfootprint 9.41.0 → 9.44.0

package/dist/esm/lib/context-bisect/arms/types.d.ts

@@ -0,0 +1,319 @@
+/**
+ * Strategy arms — the SUBSTITUTION tier of the counterfactual engine.
+ *
+ * Pattern: sibling spec type, shared statistics.
+ * Role:    answer "scorer A versus scorer B", "topK versus a re-ranker",
+ *          "sliding-window versus token-budget" with the SAME seeded-rerun
+ *          discipline the ablation tier uses — and without pretending a
+ *          substitution is a removal.
+ *
+ * ## Why this is NOT a fifth arm on `AblationSpec`
+ *
+ * Ablation is REMOVAL. `AblationSpec` names inputs an agent is BUILT from and
+ * `applyAblations` filters them out; the ablated agent is strictly smaller than
+ * the original. Substitution is REPLACEMENT: the agent that re-ranks instead of
+ * taking top-K is not smaller, it is different. Three concrete costs decided
+ * this, and each one is a place a shared union would have produced a
+ * confident-looking verdict for an incoherent comparison:
+ *
+ *  1. **`bisectCulprits` would accept it.** The union feeds a delta-debugging
+ *     search (`bisect.ts`) whose premise is that probes are SUBSETS of removals:
+ *     it halves the candidate set, probes each half, and returns a "minimal
+ *     culprit set". Substitutions do not compose that way — swapping the scorer
+ *     and swapping the window at once is a THIRD arm, not a bigger removal — and
+ *     the answer it would print ("minimal culprit set = {scorer swap}") is the
+ *     wrong tier of claim entirely: an arm is an alternative configuration, not
+ *     a culprit. A type that lets an incoherent comparison reach a verdict is
+ *     worse than the limitation it removes.
+ *  2. **`rerunWithoutSources` would report it under `removed`.** Its result
+ *     type, its `removableSources` vocabulary and every sentence it composes say
+ *     "removing X". A substitution arriving through its `ignore` escape hatch
+ *     would be described, in a field NAME, as something it is not.
+ *  3. **`applyAblations` has nowhere to put it.** It returns filtered
+ *     `{tools, injections, memoryEntries}`. A substitution is not a filter of
+ *     those inputs; it is a construction choice made before they exist.
+ *
+ * The rejected third option — a `'strategy'` arm that genuinely ABLATES, i.e.
+ * falls back to the library default — was rejected because "no retrieval at
+ * all" and "re-rank instead of top-K" are different experiments and only the
+ * second is what a researcher means. The manifest already treats `memories: []`
+ * as an arm of its own; an arm that silently meant "the default" would answer a
+ * question nobody asked while looking like it supported the one they did.
+ *
+ * So: sibling TYPE, shared STATISTICS. `AblationRunStats`, `SimilarityStats`,
+ * `resolveSamples`, `similarityStats`, `costStatsFrom`, `probeFlipped` and
+ * `defaultOutcomeComparator` are reused verbatim; nothing in `ablation.ts`,
+ * `bisect.ts`, `rerun.ts` or `cost.ts` changed, so the four existing spec arms
+ * behave identically (pinned by test/lib/context-bisect/ablation-union-pinned).
+ *
+ * ## What the library can and cannot DO with an arm
+ *
+ * For a removal the library can perform the intervention: `applyAblations`
+ * filters the inputs. For a substitution it cannot — construction is the
+ * consumer's code, and no library can reach into it and replace a scorer. What
+ * it can do instead is DECLARE the arm and then VERIFY it took effect, because
+ * 9.41.0 made configuration self-describing: `agentfootprint.agent.run_configured`
+ * names the provider, model, react mode, window, each memory's strategy /
+ * retrieval / embedder, the graph's posture and classifier, and the evidence
+ * posture. {@link ArmFacets} is deliberately that same vocabulary, so a run's
+ * own manifest — not the experimenter's bookkeeping — says which arm it belonged
+ * to, and an arm whose runs contradict its declaration gets no verdict at all.
+ *
+ * An arm may ALSO carry `ablations`: removals ride inside an arm through the
+ * unchanged removal machinery (`applyArm` → `applyAblations`). Composition, not
+ * union — each type still means exactly one thing.
+ */
+import type { Embedder } from '../../influence-core/index.js';
+import type { AblationRunStats, AblationSpec, AblationVerdict, OutcomeComparator, RunCost, SimilarityStats } from '../types.js';
+/**
+ * One mounted memory as the run manifest describes it — structurally satisfied
+ * by `RunConfiguredMemoryPayload`.
+ */
+export interface ManifestMemoryLike {
+    readonly id?: string;
+    readonly type?: string;
+    readonly strategy?: string;
+    readonly retrieval?: string;
+    readonly embedderId?: string;
+    readonly flavor?: string;
+}
+/**
+ * The run-configuration manifest, structurally — satisfied by
+ * `AgentRunConfiguredPayload` (the payload of
+ * `agentfootprint.agent.run_configured`, 9.41.0).
+ *
+ * Declared structurally, and every member optional, for the same reason
+ * `CapturedEventLike` and `QualityTriggerLookup` are: `context-bisect` is a
+ * LEAF. It must not import the event registry to read one payload, and a
+ * consumer replaying manifests out of a saved recording must be able to hand in
+ * any bag shaped like this.
+ */
+export interface RunManifestLike {
+    readonly agentId?: string;
+    readonly llm?: {
+        readonly provider?: string;
+        readonly model?: string;
+    };
+    readonly reactMode?: string;
+    readonly memories?: readonly ManifestMemoryLike[];
+    readonly window?: string;
+    readonly skillGraph?: {
+        readonly routing?: string;
+        readonly continuity?: string;
+        readonly scorer?: string;
+    };
+    readonly evidenceGate?: string;
+}
+/** The memory facet of an arm — matched against the manifest's memory rows. */
+export interface ArmMemoryFacet {
+    /** Narrow the match to ONE mounted memory by `defineMemory({ id })`. Without
+     *  it, ANY mounted memory satisfying the other fields counts as a match. */
+    readonly id?: string;
+    readonly type?: string;
+    /** The compiled strategy KIND (`'window'`, `'topK'`, …). */
+    readonly strategy?: string;
+    /** `RetrievalStrategy.name` — the spelled-out retrieval rule. This is the
+     *  field a "topK versus re-rank" study varies. */
+    readonly retrieval?: string;
+    readonly embedderId?: string;
+    readonly flavor?: string;
+}
+/**
+ * What ONE arm holds constant, in the vocabulary the run manifest speaks.
+ *
+ * Every field here has a counterpart in `AgentRunConfiguredPayload`, and that
+ * is the whole point: a facet a consumer declares is a facet the engine can
+ * CHECK against the run that actually happened. A configuration difference the
+ * manifest cannot name (a prompt tweak, a temperature) is not expressible here
+ * on purpose — declaring it would produce an arm label nothing can verify.
+ */
+export interface ArmFacets {
+    /** `LLMProvider.name` of the effective provider. */
+    readonly provider?: string;
+    readonly model?: string;
+    readonly reactMode?: 'classic' | 'dynamic' | 'dynamic-grouped';
+    /** `WindowStrategy.name` — the "sliding-window versus token-budget" axis. */
+    readonly window?: string;
+    /** `IntentScorer.name` — the "scorer A versus scorer B" axis. */
+    readonly scorer?: string;
+    readonly routing?: 'assist' | 'guard' | 'rails';
+    readonly continuity?: 'turn' | 'conversation';
+    readonly evidenceGate?: 'assist' | 'guard' | 'rails';
+    readonly memory?: ArmMemoryFacet;
+}
+/**
+ * One configuration under test.
+ *
+ * `facets` is what the run manifest should say about this arm; `ablations` is
+ * anything this arm additionally REMOVES, applied by the unchanged removal
+ * machinery (`applyArm`). An arm needs at least one of the two — except the
+ * baseline arm, which may declare nothing at all and mean "the configuration as
+ * it stands".
+ */
+export interface StrategyArm {
+    /** Stable label. Appears in every verdict sentence and keys the readout. */
+    readonly id: string;
+    readonly facets?: ArmFacets;
+    /** Removals this arm also applies — the four `AblationSpec` kinds, unchanged. */
+    readonly ablations?: readonly AblationSpec[];
+}
+/**
+ * What one seeded run of one arm reported back.
+ *
+ * `manifest` is the connection to 9.41.0: capture it with
+ * `agent.on('agentfootprint.agent.run_configured', e => …)` inside the runner
+ * (or hand the run's captured events to `manifestFromEvents`). Supplying it
+ * turns "the experimenter says this run was arm B" into a checked fact.
+ */
+export interface ArmRunResult {
+    readonly output: string;
+    readonly cost?: RunCost;
+    readonly manifest?: RunManifestLike;
+}
+/**
+ * Consumer-supplied arm runner: BUILD the agent under `arm`'s configuration,
+ * run the same scenario, return the output.
+ *
+ * The engine takes a factory, not an instance — the same shape `AblationRunner`
+ * already has, and the reason a strategy comparison fits here at all. Contract,
+ * identical in spirit to `AblationRunner`:
+ * - `run.seed` varies 0..N-1 within ONE arm. Thread it into any stochastic
+ *   knob so reruns are deterministic-but-distinct.
+ * - Build a FRESH agent/provider per call (scripted mock providers are
+ *   stateful).
+ * - Apply `arm.ablations` with `applyArm` at construction.
+ * - Returning a bare string is fine; the object form additionally unlocks the
+ *   cost readout and the manifest check.
+ */
+export type ArmRunner = (arm: StrategyArm, run: {
+    readonly seed: number;
+}) => Promise<string | ArmRunResult>;
+/** One declared facet the run manifest contradicted. */
+export interface ArmFacetMismatch {
+    /** Dotted facet path — `'model'`, `'memory.retrieval'`, `'skillGraph.scorer'`. */
+    readonly facet: string;
+    readonly declared: string;
+    /** What the manifest said. ABSENT means the manifest did not declare that
+     *  field at all — which, per the manifest's own rule, means "not configured"
+     *  and is therefore still a contradiction of a declared arm. */
+    readonly observed?: string;
+}
+/** Did the arm the consumer DECLARED match the runs that actually happened? */
+export interface ArmApplication {
+    /** How many of this arm's seeded runs reported a manifest. */
+    readonly manifestsSeen: number;
+    /**
+     * The declared facets were really compared against a manifest. FALSE when the
+     * runner reported none, or when the arm declares only `ablations` (no
+     * manifest names a tool catalog). Read this BEFORE `applied`.
+     */
+    readonly checked: boolean;
+    /** Every seen manifest agreed with every declared facet. Meaningless when
+     *  `checked` is false. */
+    readonly applied: boolean;
+    readonly mismatches: readonly ArmFacetMismatch[];
+}
+/**
+ * The inert-intervention control for an arm comparison.
+ *
+ * The cost tier's leave-one-out placebo does NOT transfer: it works by holding
+ * out one member of a POPULATION of peer suspects, and a comparison of two arms
+ * has no such population — leaving an arm out leaves an experiment, not a
+ * control. What does transfer is the placebo IDEA, in a stronger form: the
+ * inert intervention is RE-RUNNING THE SAME CONFIGURATION, and the engine
+ * already pays for it (the baseline arm's own seeded runs). Its similarity
+ * spread against the reference is the null distribution a challenger must fall
+ * outside of; `floor` is the strictest point of it.
+ *
+ * Two axes, two controls, and they are not interchangeable:
+ *  - the COMPARATOR axis's control is the baseline arm's own flip count, which
+ *    must be ZERO (the stability gate — same zero-tolerance rule `bisect.ts`
+ *    and `localize.ts` use);
+ *  - the SIMILARITY axis's control is this band.
+ * The band only participates in a verdict when the flip comparator IS the
+ * similarity comparator (`gates`). With a domain comparator the two axes are
+ * different instruments, and vetoing a genuine decision flip with an embedding
+ * statistic would suppress a true finding — so it is reported and not applied.
+ */
+export interface NullBand {
+    readonly baselineArmId: string;
+    /** The baseline arm's own similarity to the reference across its seeds. */
+    readonly similarity: SimilarityStats;
+    /** The floor a challenger's mean similarity must fall BELOW to be
+     *  distinguishable from seed-to-seed noise (`similarity.min`). */
+    readonly floor: number;
+    /** The baseline reproduced EXACTLY on every seed (zero spread). The band is
+     *  then a hard floor that any change clears — honest for a deterministic
+     *  scenario, and only as strong as that determinism (a mock provider is
+     *  reproducible in a way a real one is not). */
+    readonly degenerate: boolean;
+    /** The band participated in the verdicts (default comparator only). */
+    readonly gates: boolean;
+    /** Plain-language recap. PRESENTATION ONLY — read the fields as data. */
+    readonly note: string;
+}
+/** One arm's evidence. */
+export interface ArmOutcome {
+    readonly armId: string;
+    readonly isBaseline: boolean;
+    /** The same per-probe statistics the ablation tier reports (samples, flips,
+     *  similarity mean/min/max/stdev, and `cost` when the runner reported one). */
+    readonly runs: AblationRunStats;
+    /** Every seeded answer, in seed order. */
+    readonly answers: readonly string[];
+    /** Mean similarity to the reference fell below the null band's floor. Always
+     *  false for the baseline arm — an arm cannot be outside its own band. */
+    readonly outsideNullBand: boolean;
+    readonly application: ArmApplication;
+    /**
+     * The CAUSAL-tier claim for this arm versus the baseline arm. Absent on the
+     * baseline arm itself (it is the reference, not a challenger).
+     */
+    readonly verdict?: AblationVerdict;
+}
+/** The finished comparison. */
+export interface StrategyComparison {
+    /**
+     * The answer every arm's similarity and flip count was measured against.
+     * `'prior-run'` — the caller's `originalAnswer`, i.e. the run that actually
+     * happened. `'baseline-arm'` — no prior answer was supplied, so the baseline
+     * arm's seed-0 answer became the reference; that seed then contributes a
+     * trivially identical comparison and stability rests on the remaining N−1.
+     */
+    readonly reference: {
+        readonly from: 'prior-run' | 'baseline-arm';
+        readonly text: string;
+    };
+    readonly baselineArmId: string;
+    /** The baseline arm reproduced the reference on EVERY seed. False ⇒ every
+     *  verdict is `'inconclusive'`; nothing is trustworthy on an unstable
+     *  scenario. */
+    readonly baselineStable: boolean;
+    readonly nullBand: NullBand;
+    /** One row per arm, declaration order, baseline included. */
+    readonly arms: readonly ArmOutcome[];
+    /** Consumer-runner invocations spent (`samples × arms`). */
+    readonly runsUsed: number;
+    /** Plain-language recap. PRESENTATION ONLY — read the fields as data. */
+    readonly summary: string;
+}
+export interface CompareStrategyArmsOptions {
+    /** At least two, distinct, each non-baseline arm declaring something. */
+    readonly arms: readonly StrategyArm[];
+    readonly runner: ArmRunner;
+    /** Embedder for the similarity readout, the null band and the default
+     *  comparator. `mockEmbedder()` is fine offline. */
+    readonly embedder: Embedder;
+    /** Which arm is the incumbent. Default: the first arm. */
+    readonly baselineArmId?: string;
+    /** The answer the run that actually happened produced. Omit for a fresh A/B
+     *  — the baseline arm's seed-0 answer becomes the reference. */
+    readonly originalAnswer?: string;
+    /** Seeded runs PER ARM. Default 3; clamped to ≥ 2 (never single-run claims). */
+    readonly samples?: number;
+    /** Did the answer CHANGE? Default: embedding similarity < `flipThreshold`.
+     *  Passing one turns the null band OFF as a gate (see {@link NullBand}). */
+    readonly answerChanged?: OutcomeComparator;
+    /** Similarity floor for the DEFAULT comparator. Default 0.8. */
+    readonly flipThreshold?: number;
+}

package/dist/esm/lib/context-bisect/arms/types.js

@@ -0,0 +1,67 @@
+/**
+ * Strategy arms — the SUBSTITUTION tier of the counterfactual engine.
+ *
+ * Pattern: sibling spec type, shared statistics.
+ * Role:    answer "scorer A versus scorer B", "topK versus a re-ranker",
+ *          "sliding-window versus token-budget" with the SAME seeded-rerun
+ *          discipline the ablation tier uses — and without pretending a
+ *          substitution is a removal.
+ *
+ * ## Why this is NOT a fifth arm on `AblationSpec`
+ *
+ * Ablation is REMOVAL. `AblationSpec` names inputs an agent is BUILT from and
+ * `applyAblations` filters them out; the ablated agent is strictly smaller than
+ * the original. Substitution is REPLACEMENT: the agent that re-ranks instead of
+ * taking top-K is not smaller, it is different. Three concrete costs decided
+ * this, and each one is a place a shared union would have produced a
+ * confident-looking verdict for an incoherent comparison:
+ *
+ *  1. **`bisectCulprits` would accept it.** The union feeds a delta-debugging
+ *     search (`bisect.ts`) whose premise is that probes are SUBSETS of removals:
+ *     it halves the candidate set, probes each half, and returns a "minimal
+ *     culprit set". Substitutions do not compose that way — swapping the scorer
+ *     and swapping the window at once is a THIRD arm, not a bigger removal — and
+ *     the answer it would print ("minimal culprit set = {scorer swap}") is the
+ *     wrong tier of claim entirely: an arm is an alternative configuration, not
+ *     a culprit. A type that lets an incoherent comparison reach a verdict is
+ *     worse than the limitation it removes.
+ *  2. **`rerunWithoutSources` would report it under `removed`.** Its result
+ *     type, its `removableSources` vocabulary and every sentence it composes say
+ *     "removing X". A substitution arriving through its `ignore` escape hatch
+ *     would be described, in a field NAME, as something it is not.
+ *  3. **`applyAblations` has nowhere to put it.** It returns filtered
+ *     `{tools, injections, memoryEntries}`. A substitution is not a filter of
+ *     those inputs; it is a construction choice made before they exist.
+ *
+ * The rejected third option — a `'strategy'` arm that genuinely ABLATES, i.e.
+ * falls back to the library default — was rejected because "no retrieval at
+ * all" and "re-rank instead of top-K" are different experiments and only the
+ * second is what a researcher means. The manifest already treats `memories: []`
+ * as an arm of its own; an arm that silently meant "the default" would answer a
+ * question nobody asked while looking like it supported the one they did.
+ *
+ * So: sibling TYPE, shared STATISTICS. `AblationRunStats`, `SimilarityStats`,
+ * `resolveSamples`, `similarityStats`, `costStatsFrom`, `probeFlipped` and
+ * `defaultOutcomeComparator` are reused verbatim; nothing in `ablation.ts`,
+ * `bisect.ts`, `rerun.ts` or `cost.ts` changed, so the four existing spec arms
+ * behave identically (pinned by test/lib/context-bisect/ablation-union-pinned).
+ *
+ * ## What the library can and cannot DO with an arm
+ *
+ * For a removal the library can perform the intervention: `applyAblations`
+ * filters the inputs. For a substitution it cannot — construction is the
+ * consumer's code, and no library can reach into it and replace a scorer. What
+ * it can do instead is DECLARE the arm and then VERIFY it took effect, because
+ * 9.41.0 made configuration self-describing: `agentfootprint.agent.run_configured`
+ * names the provider, model, react mode, window, each memory's strategy /
+ * retrieval / embedder, the graph's posture and classifier, and the evidence
+ * posture. {@link ArmFacets} is deliberately that same vocabulary, so a run's
+ * own manifest — not the experimenter's bookkeeping — says which arm it belonged
+ * to, and an arm whose runs contradict its declaration gets no verdict at all.
+ *
+ * An arm may ALSO carry `ablations`: removals ride inside an arm through the
+ * unchanged removal machinery (`applyArm` → `applyAblations`). Composition, not
+ * union — each type still means exactly one thing.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/esm/lib/context-bisect/arms/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../src/lib/context-bisect/arms/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG"}

package/dist/esm/lib/context-bisect/arms/validate.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Arm validation — the teaching refusals.
+ *
+ * Every refusal here fires BEFORE a single model call. That is the whole design
+ * rule: an incoherent comparison must cost nothing. The engine's other refusals
+ * (an unstable baseline, an arm that did not take effect) can only be known
+ * after the runs, and those refuse a VERDICT rather than throw — refusing after
+ * spending N model calls would be hostile. These six can be known from the
+ * declaration alone, so they throw.
+ */
+import type { ArmFacets, StrategyArm } from './types.js';
+/**
+ * A canonical key for one arm's CONFIGURATION (facets + removals), used only to
+ * detect two arms that are secretly the same arm. Hand-rolled rather than
+ * `JSON.stringify` because key order in an object literal is authoring
+ * accident, and two arms written `{model, scorer}` and `{scorer, model}` are
+ * the same arm.
+ */
+export declare function armConfigKey(arm: StrategyArm): string;
+/** How many facet leaves an arm declares — 0 means it names no configuration. */
+export declare function declaredFacetCount(facets: ArmFacets | undefined): number;
+/**
+ * Refuse every comparison that cannot mean anything, and resolve which arm is
+ * the incumbent. Returns the resolved baseline arm id.
+ */
+export declare function validateStrategyArms(arms: readonly StrategyArm[], baselineArmId: string | undefined): string;

package/dist/esm/lib/context-bisect/arms/validate.js

@@ -0,0 +1,122 @@
+/**
+ * Arm validation — the teaching refusals.
+ *
+ * Every refusal here fires BEFORE a single model call. That is the whole design
+ * rule: an incoherent comparison must cost nothing. The engine's other refusals
+ * (an unstable baseline, an arm that did not take effect) can only be known
+ * after the runs, and those refuse a VERDICT rather than throw — refusing after
+ * spending N model calls would be hostile. These six can be known from the
+ * declaration alone, so they throw.
+ */
+/** Message prefix — every refusal names the door the caller used. */
+const PREFIX = 'compareStrategyArms:';
+/**
+ * A canonical key for one arm's CONFIGURATION (facets + removals), used only to
+ * detect two arms that are secretly the same arm. Hand-rolled rather than
+ * `JSON.stringify` because key order in an object literal is authoring
+ * accident, and two arms written `{model, scorer}` and `{scorer, model}` are
+ * the same arm.
+ */
+export function armConfigKey(arm) {
+    const facets = canonical(arm.facets ?? {});
+    const ablations = [...(arm.ablations ?? [])].map(canonical).sort().join('|');
+    return `${facets}::${ablations}`;
+}
+/** Stable, sorted, depth-first rendering of a plain value. */
+function canonical(value) {
+    if (value === undefined)
+        return 'undefined';
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value);
+    if (Array.isArray(value)) {
+        // Spec arrays (`ignoredTools`, `excludeInjectionIds`) are SETS in meaning —
+        // two arms differing only in the order they list two tool names are one arm.
+        return `[${value.map(canonical).sort().join(',')}]`;
+    }
+    const entries = Object.entries(value)
+        .filter(([, v]) => v !== undefined)
+        .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+        .map(([k, v]) => `${k}:${canonical(v)}`);
+    return `{${entries.join(',')}}`;
+}
+/** How many facet leaves an arm declares — 0 means it names no configuration. */
+export function declaredFacetCount(facets) {
+    if (facets === undefined)
+        return 0;
+    let count = 0;
+    for (const [key, value] of Object.entries(facets)) {
+        if (value === undefined)
+            continue;
+        if (key === 'memory' && typeof value === 'object') {
+            count += Object.values(value).filter((v) => v !== undefined).length;
+            continue;
+        }
+        count += 1;
+    }
+    return count;
+}
+/**
+ * Refuse every comparison that cannot mean anything, and resolve which arm is
+ * the incumbent. Returns the resolved baseline arm id.
+ */
+export function validateStrategyArms(arms, baselineArmId) {
+    // 1. A comparison needs two sides. One arm re-run N times is a VARIANCE
+    //    measurement, and the library already has a door for it.
+    if (arms.length < 2) {
+        throw new Error(`${PREFIX} a comparison needs at least two arms — got ${arms.length}. To measure one ` +
+            `configuration's own run-to-run variance, probe it with runAblationProbe(config, []) ` +
+            `instead; to remove a source from one configuration, use rerunWithoutSources.`);
+    }
+    // 2 + 3. Ids are the readout's primary key and every verdict sentence's
+    //        subject. A blank or repeated one makes the result unreadable.
+    const seen = new Set();
+    for (const arm of arms) {
+        if (typeof arm.id !== 'string' || arm.id.trim() === '') {
+            throw new Error(`${PREFIX} every arm needs a non-empty id — it names the arm in each verdict and keys ` +
+                `the readout. Give each configuration a short label ('topK', 'rerank').`);
+        }
+        if (seen.has(arm.id)) {
+            throw new Error(`${PREFIX} two arms share the id '${arm.id}' — arms are reported one row per id, so the ` +
+                `second would overwrite the first. Give them distinct labels.`);
+        }
+        seen.add(arm.id);
+    }
+    // 4. The incumbent must exist: it is the reference AND the null band.
+    const resolved = baselineArmId ?? arms[0].id;
+    if (!seen.has(resolved)) {
+        throw new Error(`${PREFIX} baselineArmId '${resolved}' matches no arm — declared arms: ` +
+            `[${arms.map((a) => a.id).join(', ')}]. The baseline arm is the incumbent: the one the ` +
+            `others are compared against, and the one whose own reruns form the null band.`);
+    }
+    // 5. A challenger that names no configuration cannot be checked against the
+    //    run manifest and cannot be told apart from the incumbent. The BASELINE
+    //    arm may name nothing — "the configuration as it stands" is a real arm,
+    //    and it is the factual one.
+    for (const arm of arms) {
+        if (arm.id === resolved)
+            continue;
+        if (declaredFacetCount(arm.facets) === 0 && (arm.ablations ?? []).length === 0) {
+            throw new Error(`${PREFIX} arm '${arm.id}' declares no configuration — no facets and no ablations. An ` +
+                `arm that names nothing cannot be checked against the run manifest and cannot be told ` +
+                `apart from the baseline arm '${resolved}'. Declare at least one facet (e.g. ` +
+                `{ memory: { retrieval: 'rerank' } }) or an ablation.`);
+        }
+    }
+    // 6. THE incoherent-spec refusal: two arms that are the same configuration.
+    //    Comparing an arm with itself measures seed variance and would report it
+    //    as a difference between strategies.
+    const byConfig = new Map();
+    for (const arm of arms) {
+        const key = armConfigKey(arm);
+        const twin = byConfig.get(key);
+        if (twin !== undefined) {
+            throw new Error(`${PREFIX} arms '${twin}' and '${arm.id}' declare the SAME configuration — comparing an ` +
+                `arm with itself measures seed-to-seed variance, not a difference between strategies. ` +
+                `Change one arm's facets, or raise \`samples\` on a single arm if variance is what you ` +
+                `want to see.`);
+        }
+        byConfig.set(key, arm.id);
+    }
+    return resolved;
+}
+//# sourceMappingURL=validate.js.map

package/dist/esm/lib/context-bisect/arms/validate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../../../../src/lib/context-bisect/arms/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,qEAAqE;AACrE,MAAM,MAAM,GAAG,sBAAsB,CAAC;AAEtC;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,GAAgB;IAC3C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7E,OAAO,GAAG,MAAM,KAAK,SAAS,EAAE,CAAC;AACnC,CAAC;AAED,8DAA8D;AAC9D,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,WAAW,CAAC;IAC5C,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9E,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,4EAA4E;QAC5E,6EAA6E;QAC7E,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACtD,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC;SAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC;SAClC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAClC,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,kBAAkB,CAAC,MAA6B;IAC9D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,KAAK,SAAS;YAAE,SAAS;QAClC,IAAI,GAAG,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAClD,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,KAAgC,CAAC,CAAC,MAAM,CAC7D,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CACvB,CAAC,MAAM,CAAC;YACT,SAAS;QACX,CAAC;QACD,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,IAA4B,EAC5B,aAAiC;IAEjC,wEAAwE;IACxE,6DAA6D;IAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,+CAA+C,IAAI,CAAC,MAAM,mBAAmB;YACpF,sFAAsF;YACtF,8EAA8E,CACjF,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,uEAAuE;IACvE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,8EAA8E;gBACrF,wEAAwE,CAC3E,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,2BAA2B,GAAG,CAAC,EAAE,+CAA+C;gBACvF,8DAA8D,CACjE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;IAED,sEAAsE;IACtE,MAAM,QAAQ,GAAG,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,mBAAmB,QAAQ,oCAAoC;YACtE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,oDAAoD;YACxF,+EAA+E,CAClF,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,4EAA4E;IAC5E,4EAA4E;IAC5E,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,EAAE,KAAK,QAAQ;YAAE,SAAS;QAClC,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/E,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,SAAS,GAAG,CAAC,EAAE,+DAA+D;gBACrF,uFAAuF;gBACvF,gCAAgC,QAAQ,sCAAsC;gBAC9E,sDAAsD,CACzD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,6EAA6E;IAC7E,yCAAyC;IACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,UAAU,IAAI,UAAU,GAAG,CAAC,EAAE,kDAAkD;gBACvF,uFAAuF;gBACvF,wFAAwF;gBACxF,cAAc,CACjB,CAAC;QACJ,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/esm/lib/context-bisect/arms/verdict.d.ts

@@ -0,0 +1,53 @@
+/**
+ * The arm verdict — the same three tiers, phrased for a substitution.
+ *
+ * ## Why not widen `verdictFor`
+ *
+ * `verdictFor` already carries an `action` word (`'ablating'` | `'restoring'`)
+ * and a third would have been one line. It ends its NOT-CONFIRMED sentence with
+ * "its ranking remains a correlational proxy only" — true of a suspect, which
+ * was RANKED before it was probed, and meaningless for an arm, which was never
+ * ranked at all. Widening the union would have forced that sentence onto the
+ * new tier or edited a string three existing tests pin byte-for-byte. So the
+ * tier LOGIC is shared (`probeFlipped`, the majority rule, the zero-tolerance
+ * baseline gate) and only the prose is new.
+ *
+ * ## The tiers, in refusal order
+ *
+ * 1. **Unstable baseline** → inconclusive. The incumbent could not reproduce
+ *    the reference answer across its own seeds, so a difference measured
+ *    against it is measuring the scenario, not the arm. Zero tolerance, exactly
+ *    as `bisect.ts` and `localize.ts` apply it: ONE un-intervened flip is
+ *    enough, because the majority rule would let a 1-in-3 flaky scenario reach
+ *    a causal verdict.
+ * 2. **Arm did not take effect** → inconclusive. The run manifest contradicted
+ *    what the arm declared. Any difference here is evidence about a
+ *    configuration nobody meant to test.
+ * 3. **Inside the null band** → inconclusive. A majority flip whose mean
+ *    similarity still sits within the baseline arm's own seed-to-seed spread is
+ *    not separable from noise. Only applied when the flip comparator and the
+ *    band are the same instrument (see `NullBand.gates`).
+ * 4. **Majority flip, outside the band, stable baseline** → confirmed. This is
+ *    a causal claim of the same tier an ablation verdict makes: a
+ *    counterfactual intervention, seeded reruns, variance reported, and a
+ *    baseline that held.
+ * 5. **Minority flips** → inconclusive. 6. **No flips** → not-confirmed.
+ *
+ * Every claim is bounded by the scenario and the comparator, and says so — an
+ * arm confirmed on one question has been confirmed on one question.
+ */
+import type { AblationRunStats, AblationVerdict } from '../types.js';
+import type { ArmApplication, NullBand } from './types.js';
+/** Everything the verdict needs beyond the arm's own numbers. */
+export interface ArmVerdictContext {
+    readonly baselineArmId: string;
+    /** The baseline arm reproduced the reference on EVERY seed. */
+    readonly baselineStable: boolean;
+    readonly baselineStats: AblationRunStats;
+    readonly band: NullBand;
+    /** Mean similarity fell below the band's floor. */
+    readonly outsideNullBand: boolean;
+    readonly application: ArmApplication;
+}
+/** Translate one arm's probe evidence into its verdict. See module docs. */
+export declare function verdictForArm(armId: string, stats: AblationRunStats, context: ArmVerdictContext): AblationVerdict;