agentfootprint 9.41.0 → 9.44.0

package/dist/esm/lib/context-bisect/arms/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Strategy arms — SUBSTITUTION counterfactuals, kept distinct from ablation's
+ * removals and sharing its statistics. See README.md in this folder for the
+ * design argument, and `types.ts` for why this is not a fifth `AblationSpec`
+ * arm.
+ *
+ * @beta Beta feature (RFC-003 Part B). The API works and is tested, but may
+ * change before GA.
+ */
+export { applyArm } from './apply.js';
+export { compareStrategyArms } from './compare.js';
+export { armFacetsFromManifest, armLabel, checkArmApplied, manifestFromEvents, matchArm, RUN_CONFIGURED_EVENT, } from './manifest.js';
+export { checkArmApplication, collectArmRuns, nullBandFrom, scoreArmRuns, type ArmRuns, } from './probe.js';
+export { armConfigKey, declaredFacetCount, validateStrategyArms } from './validate.js';
+export { verdictForArm, type ArmVerdictContext } from './verdict.js';
+export type { ArmApplication, ArmFacetMismatch, ArmFacets, ArmMemoryFacet, ArmOutcome, ArmRunResult, ArmRunner, CompareStrategyArmsOptions, ManifestMemoryLike, NullBand, RunManifestLike, StrategyArm, StrategyComparison, } from './types.js';

package/dist/esm/lib/context-bisect/arms/index.js

@@ -0,0 +1,16 @@
+/**
+ * Strategy arms — SUBSTITUTION counterfactuals, kept distinct from ablation's
+ * removals and sharing its statistics. See README.md in this folder for the
+ * design argument, and `types.ts` for why this is not a fifth `AblationSpec`
+ * arm.
+ *
+ * @beta Beta feature (RFC-003 Part B). The API works and is tested, but may
+ * change before GA.
+ */
+export { applyArm } from './apply.js';
+export { compareStrategyArms } from './compare.js';
+export { armFacetsFromManifest, armLabel, checkArmApplied, manifestFromEvents, matchArm, RUN_CONFIGURED_EVENT, } from './manifest.js';
+export { checkArmApplication, collectArmRuns, nullBandFrom, scoreArmRuns, } from './probe.js';
+export { armConfigKey, declaredFacetCount, validateStrategyArms } from './validate.js';
+export { verdictForArm } from './verdict.js';
+//# sourceMappingURL=index.js.map

package/dist/esm/lib/context-bisect/arms/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/lib/context-bisect/arms/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EACL,qBAAqB,EACrB,QAAQ,EACR,eAAe,EACf,kBAAkB,EAClB,QAAQ,EACR,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,YAAY,EACZ,YAAY,GAEb,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACvF,OAAO,EAAE,aAAa,EAA0B,MAAM,cAAc,CAAC"}

package/dist/esm/lib/context-bisect/arms/manifest.d.ts

@@ -0,0 +1,79 @@
+/**
+ * The manifest bridge — which arm did this run belong to, according to the RUN
+ * rather than according to the experimenter.
+ *
+ * Before 9.41.0 an arm label was bookkeeping: the consumer set a variable to
+ * `'rerank'`, built an agent it believed re-ranked, and nothing in the library
+ * could contradict it. `agentfootprint.agent.run_configured` names the adapters
+ * and strategies a run is about to use, so the belief became checkable — and
+ * {@link ArmFacets} is deliberately spelled in that event's own vocabulary so
+ * the check is a field-by-field comparison and not a translation layer.
+ *
+ * Two directions, both here:
+ *  - **verify** — `checkArmApplied(arm, manifest)`: the arm said `retrieval:
+ *    'rerank'`; did the run agree? A contradiction costs the arm its verdict
+ *    (`compare.ts`), because a difference measured between two arms that were
+ *    secretly one configuration is not evidence about either.
+ *  - **classify** — `matchArm(arms, manifest)`: hand it a recorded run's
+ *    manifest and it says which declared arm that run belongs to. This is the
+ *    door for grouping runs a study already made, offline, out of saved
+ *    recordings.
+ *
+ * ## Absence is a contradiction, not a wildcard
+ *
+ * The manifest's own rule is that an absent field means "not configured", never
+ * a guessed default. So an arm declaring `window: 'tokenBudget'` against a
+ * manifest with no `window` is a MISMATCH, reported with `observed` absent. The
+ * alternative — treating absence as "could be anything" — would let a run with
+ * no window stage pass as the token-budget arm, which is exactly the mis-wiring
+ * this file exists to catch.
+ */
+import type { CapturedEventLike } from '../types.js';
+import type { ArmFacetMismatch, ArmFacets, RunManifestLike, StrategyArm } from './types.js';
+/** The event type the run manifest rides on (9.41.0). */
+export declare const RUN_CONFIGURED_EVENT = "agentfootprint.agent.run_configured";
+/**
+ * Pull the run manifest out of a run's captured typed events — the same
+ * `CapturedEventLike[]` bag `localizeContextBug` already accepts, so a consumer
+ * who collected events with `agent.on('*', …)` needs no second capture.
+ *
+ * One manifest per run by design; the FIRST is returned. A bag holding events
+ * from several runs is the caller's own mixing — filter by `meta.runId` first.
+ */
+export declare function manifestFromEvents(events: readonly CapturedEventLike[] | undefined): RunManifestLike | undefined;
+/**
+ * Project a manifest onto the arm-facet vocabulary — the inverse of what an arm
+ * declares. Memory rows collapse to the FIRST one; a study varying retrieval
+ * across several mounted memories should declare `memory.id` and read the rows
+ * itself rather than trust a projection to pick.
+ */
+export declare function armFacetsFromManifest(manifest: RunManifestLike): ArmFacets;
+/**
+ * A stable label for a set of facets — sorted `key=value`, so two runs of one
+ * configuration produce the same string whatever order the fields were written
+ * in. Useful as a grouping key for N recorded runs.
+ */
+export declare function armLabel(facets: ArmFacets): string;
+/**
+ * Compare one arm's DECLARED facets against a run's manifest. Empty result =
+ * the run agreed with everything the arm claimed.
+ *
+ * Only declared facets are compared. An arm is a claim about the facets it
+ * names and says nothing about the rest — checking undeclared fields would fail
+ * every arm for differing from the manifest in ways it never claimed.
+ */
+export declare function checkArmApplied(arm: StrategyArm, manifest: RunManifestLike): readonly ArmFacetMismatch[];
+/**
+ * Which declared arm does this run belong to? The offline door: group N
+ * recorded runs into arms by what each run's own manifest says.
+ *
+ * Rules, all conservative:
+ * - only arms that DECLARE facets are candidates (an arm naming nothing matches
+ *   every run and would swallow the whole study);
+ * - a candidate matches when the manifest contradicts none of its facets;
+ * - the MOST SPECIFIC match wins (most declared facets), and a tie returns
+ *   `undefined` — two arms fitting one run equally well is an ambiguity, and
+ *   guessing which the experimenter meant is exactly the bookkeeping this
+ *   function replaces.
+ */
+export declare function matchArm(arms: readonly StrategyArm[], manifest: RunManifestLike): StrategyArm | undefined;

package/dist/esm/lib/context-bisect/arms/manifest.js

@@ -0,0 +1,199 @@
+/**
+ * The manifest bridge — which arm did this run belong to, according to the RUN
+ * rather than according to the experimenter.
+ *
+ * Before 9.41.0 an arm label was bookkeeping: the consumer set a variable to
+ * `'rerank'`, built an agent it believed re-ranked, and nothing in the library
+ * could contradict it. `agentfootprint.agent.run_configured` names the adapters
+ * and strategies a run is about to use, so the belief became checkable — and
+ * {@link ArmFacets} is deliberately spelled in that event's own vocabulary so
+ * the check is a field-by-field comparison and not a translation layer.
+ *
+ * Two directions, both here:
+ *  - **verify** — `checkArmApplied(arm, manifest)`: the arm said `retrieval:
+ *    'rerank'`; did the run agree? A contradiction costs the arm its verdict
+ *    (`compare.ts`), because a difference measured between two arms that were
+ *    secretly one configuration is not evidence about either.
+ *  - **classify** — `matchArm(arms, manifest)`: hand it a recorded run's
+ *    manifest and it says which declared arm that run belongs to. This is the
+ *    door for grouping runs a study already made, offline, out of saved
+ *    recordings.
+ *
+ * ## Absence is a contradiction, not a wildcard
+ *
+ * The manifest's own rule is that an absent field means "not configured", never
+ * a guessed default. So an arm declaring `window: 'tokenBudget'` against a
+ * manifest with no `window` is a MISMATCH, reported with `observed` absent. The
+ * alternative — treating absence as "could be anything" — would let a run with
+ * no window stage pass as the token-budget arm, which is exactly the mis-wiring
+ * this file exists to catch.
+ */
+import { declaredFacetCount } from './validate.js';
+/** The event type the run manifest rides on (9.41.0). */
+export const RUN_CONFIGURED_EVENT = 'agentfootprint.agent.run_configured';
+/**
+ * Pull the run manifest out of a run's captured typed events — the same
+ * `CapturedEventLike[]` bag `localizeContextBug` already accepts, so a consumer
+ * who collected events with `agent.on('*', …)` needs no second capture.
+ *
+ * One manifest per run by design; the FIRST is returned. A bag holding events
+ * from several runs is the caller's own mixing — filter by `meta.runId` first.
+ */
+export function manifestFromEvents(events) {
+    const event = events?.find((e) => e.type === RUN_CONFIGURED_EVENT);
+    if (event === undefined)
+        return undefined;
+    const payload = event.payload;
+    // Structural, not a cast-and-pray: a payload that is not an object at all is
+    // no manifest, and reporting `undefined` keeps `checked: false` honest.
+    return payload !== null && typeof payload === 'object' ? payload : undefined;
+}
+/**
+ * Project a manifest onto the arm-facet vocabulary — the inverse of what an arm
+ * declares. Memory rows collapse to the FIRST one; a study varying retrieval
+ * across several mounted memories should declare `memory.id` and read the rows
+ * itself rather than trust a projection to pick.
+ */
+export function armFacetsFromManifest(manifest) {
+    const memory = manifest.memories?.[0];
+    const provider = manifest.llm?.provider;
+    const model = manifest.llm?.model;
+    const reactMode = manifest.reactMode;
+    const scorer = manifest.skillGraph?.scorer;
+    const routing = manifest.skillGraph?.routing;
+    const continuity = manifest.skillGraph?.continuity;
+    const evidenceGate = manifest.evidenceGate;
+    return {
+        ...(provider !== undefined && { provider }),
+        ...(model !== undefined && { model }),
+        ...(isReactMode(reactMode) && { reactMode }),
+        ...(manifest.window !== undefined && { window: manifest.window }),
+        ...(scorer !== undefined && { scorer }),
+        ...(isPosture(routing) && { routing }),
+        ...(isContinuity(continuity) && { continuity }),
+        ...(isPosture(evidenceGate) && { evidenceGate }),
+        ...(memory !== undefined && { memory: { ...memory } }),
+    };
+}
+/**
+ * A stable label for a set of facets — sorted `key=value`, so two runs of one
+ * configuration produce the same string whatever order the fields were written
+ * in. Useful as a grouping key for N recorded runs.
+ */
+export function armLabel(facets) {
+    const parts = [];
+    for (const [key, value] of Object.entries(facets)) {
+        if (value === undefined)
+            continue;
+        if (key === 'memory') {
+            for (const [mk, mv] of Object.entries(value)) {
+                if (mv !== undefined)
+                    parts.push(`memory.${mk}=${String(mv)}`);
+            }
+            continue;
+        }
+        parts.push(`${key}=${String(value)}`);
+    }
+    return parts.sort().join(',');
+}
+/**
+ * Compare one arm's DECLARED facets against a run's manifest. Empty result =
+ * the run agreed with everything the arm claimed.
+ *
+ * Only declared facets are compared. An arm is a claim about the facets it
+ * names and says nothing about the rest — checking undeclared fields would fail
+ * every arm for differing from the manifest in ways it never claimed.
+ */
+export function checkArmApplied(arm, manifest) {
+    const facets = arm.facets;
+    if (facets === undefined)
+        return [];
+    const out = [];
+    compare(out, 'provider', facets.provider, manifest.llm?.provider);
+    compare(out, 'model', facets.model, manifest.llm?.model);
+    compare(out, 'reactMode', facets.reactMode, manifest.reactMode);
+    compare(out, 'window', facets.window, manifest.window);
+    compare(out, 'skillGraph.scorer', facets.scorer, manifest.skillGraph?.scorer);
+    compare(out, 'skillGraph.routing', facets.routing, manifest.skillGraph?.routing);
+    compare(out, 'skillGraph.continuity', facets.continuity, manifest.skillGraph?.continuity);
+    compare(out, 'evidenceGate', facets.evidenceGate, manifest.evidenceGate);
+    const memory = facets.memory;
+    if (memory !== undefined) {
+        const rows = manifest.memories ?? [];
+        if (memory.id !== undefined) {
+            // Narrowed to ONE row: a missing row is a mismatch on the id itself, and
+            // the remaining fields are then compared against that row alone.
+            const row = rows.find((r) => r.id === memory.id);
+            if (row === undefined) {
+                out.push({
+                    facet: 'memory.id',
+                    declared: memory.id,
+                    ...(rows.length > 0 && { observed: rows.map((r) => r.id ?? '?').join(', ') }),
+                });
+            }
+            else {
+                for (const field of MEMORY_FIELDS) {
+                    compare(out, `memory.${field}`, memory[field], row[field]);
+                }
+            }
+        }
+        else if (!rows.some((row) => MEMORY_FIELDS.every((f) => matches(memory[f], row[f])))) {
+            // Un-narrowed: ANY mounted memory satisfying every declared field counts.
+            // Reported as ONE mismatch — naming a per-field winner across rows would
+            // invent a memory that does not exist.
+            out.push({
+                facet: 'memory',
+                declared: armLabel({ memory }),
+                ...(rows.length > 0 && {
+                    observed: rows.map((row) => armLabel({ memory: row })).join(' | '),
+                }),
+            });
+        }
+    }
+    return out;
+}
+/**
+ * Which declared arm does this run belong to? The offline door: group N
+ * recorded runs into arms by what each run's own manifest says.
+ *
+ * Rules, all conservative:
+ * - only arms that DECLARE facets are candidates (an arm naming nothing matches
+ *   every run and would swallow the whole study);
+ * - a candidate matches when the manifest contradicts none of its facets;
+ * - the MOST SPECIFIC match wins (most declared facets), and a tie returns
+ *   `undefined` — two arms fitting one run equally well is an ambiguity, and
+ *   guessing which the experimenter meant is exactly the bookkeeping this
+ *   function replaces.
+ */
+export function matchArm(arms, manifest) {
+    const fits = arms
+        .filter((arm) => declaredFacetCount(arm.facets) > 0)
+        .filter((arm) => checkArmApplied(arm, manifest).length === 0)
+        .map((arm) => ({ arm, specificity: declaredFacetCount(arm.facets) }))
+        .sort((a, b) => b.specificity - a.specificity);
+    if (fits.length === 0)
+        return undefined;
+    if (fits.length > 1 && fits[0].specificity === fits[1].specificity)
+        return undefined;
+    return fits[0].arm;
+}
+// ─── internals ───────────────────────────────────────────────────────
+const MEMORY_FIELDS = ['type', 'strategy', 'retrieval', 'embedderId', 'flavor'];
+function matches(declared, observed) {
+    return declared === undefined || declared === observed;
+}
+function compare(out, facet, declared, observed) {
+    if (matches(declared, observed))
+        return;
+    out.push({ facet, declared: declared, ...(observed !== undefined && { observed }) });
+}
+function isReactMode(value) {
+    return value === 'classic' || value === 'dynamic' || value === 'dynamic-grouped';
+}
+function isPosture(value) {
+    return value === 'assist' || value === 'guard' || value === 'rails';
+}
+function isContinuity(value) {
+    return value === 'turn' || value === 'conversation';
+}
+//# sourceMappingURL=manifest.js.map

package/dist/esm/lib/context-bisect/arms/manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../../../src/lib/context-bisect/arms/manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AASnD,yDAAyD;AACzD,MAAM,CAAC,MAAM,oBAAoB,GAAG,qCAAqC,CAAC;AAE1E;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAgD;IAEhD,MAAM,KAAK,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,oBAAoB,CAAC,CAAC;IACnE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,6EAA6E;IAC7E,wEAAwE;IACxE,OAAO,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAE,OAA2B,CAAC,CAAC,CAAC,SAAS,CAAC;AACpG,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAAyB;IAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC;IACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC;IAClC,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;IACrC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC3C,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7C,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACnD,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;IAC3C,OAAO;QACL,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC3C,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;QACrC,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QAC5C,GAAG,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QACjE,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC;QACvC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;QACtC,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC;QAC/C,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;QAChD,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;KACvD,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAiB;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,KAAK,SAAS;YAAE,SAAS;QAClC,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;YACrB,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAA2B,CAAC,EAAE,CAAC;gBACnE,IAAI,EAAE,KAAK,SAAS;oBAAE,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAC7B,GAAgB,EAChB,QAAyB;IAEzB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACpC,MAAM,GAAG,GAAuB,EAAE,CAAC;IAEnC,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzD,OAAO,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,EAAE,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC9E,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACjF,OAAO,CAAC,GAAG,EAAE,uBAAuB,EAAE,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;IAEzE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;QACrC,IAAI,MAAM,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;YAC5B,yEAAyE;YACzE,iEAAiE;YACjE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC;YACjD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,GAAG,CAAC,IAAI,CAAC;oBACP,KAAK,EAAE,WAAW;oBAClB,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBAC9E,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;oBAClC,OAAO,CAAC,GAAG,EAAE,UAAU,KAAK,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACvF,0EAA0E;YAC1E,yEAAyE;YACzE,uCAAuC;YACvC,GAAG,CAAC,IAAI,CAAC;gBACP,KAAK,EAAE,QAAQ;gBACf,QAAQ,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;gBAC9B,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI;oBACrB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;iBACnE,CAAC;aACH,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,QAAQ,CACtB,IAA4B,EAC5B,QAAyB;IAEzB,MAAM,IAAI,GAAG,IAAI;SACd,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;SACnD,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;SAC5D,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SACpE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACrB,CAAC;AAED,wEAAwE;AAExE,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,CAAU,CAAC;AAEzF,SAAS,OAAO,CAAC,QAA4B,EAAE,QAA4B;IACzE,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,QAAQ,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CACd,GAAuB,EACvB,KAAa,EACb,QAA4B,EAC5B,QAA4B;IAE5B,IAAI,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAAE,OAAO;IACxC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAkB,EAAE,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,WAAW,CAClB,KAAyB;IAEzB,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,iBAAiB,CAAC;AACnF,CAAC;AAED,SAAS,SAAS,CAAC,KAAyB;IAC1C,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,CAAC;AACtE,CAAC;AAED,SAAS,YAAY,CAAC,KAAyB;IAC7C,OAAO,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,cAAc,CAAC;AACtD,CAAC"}

package/dist/esm/lib/context-bisect/arms/probe.d.ts

@@ -0,0 +1,54 @@
+/**
+ * The arm probe — seeded runs in, shared statistics out.
+ *
+ * Split in two on purpose. `collectArmRuns` only RUNS (it cannot know the
+ * reference yet: when no prior answer was supplied, the reference IS the
+ * baseline arm's first answer). `scoreArmRuns` only SCORES. Keeping them apart
+ * is what lets the reference be chosen between the two halves without a second
+ * pass over the model — the same "one ablation, two readouts" thrift
+ * `runAblationProbe` applies to cost.
+ *
+ * Everything statistical is borrowed, not re-derived: `similarityStats`,
+ * `costStatsFrom` and `resolveSamples` come from `ablation.ts` unchanged, so an
+ * arm's `AblationRunStats` is the same number computed the same way as an
+ * ablation's — which is the point of sharing the statistics while keeping the
+ * spec types apart.
+ */
+import type { AblationRunStats, Embedder, OutcomeComparator } from '../types.js';
+import type { ArmApplication, ArmRunner, NullBand, RunManifestLike, StrategyArm } from './types.js';
+/** The raw harvest of one arm's seeded runs — no scoring yet. */
+export interface ArmRuns {
+    readonly answers: readonly string[];
+    readonly loops: readonly number[];
+    readonly tokens: readonly number[];
+    readonly manifests: readonly RunManifestLike[];
+}
+/**
+ * Call the consumer's runner once per seed (0..samples-1) for ONE arm and
+ * normalize the two return shapes (`string` | `{output, cost?, manifest?}`) —
+ * the same normalization `runAblationProbe` performs, so a bare-string runner
+ * costs the caller nothing in ceremony.
+ */
+export declare function collectArmRuns(arm: StrategyArm, runner: ArmRunner, samples: number): Promise<ArmRuns>;
+/**
+ * Score one arm's harvested answers against the reference: similarity per seed
+ * (mean/min/max/stdev) plus the outcome-flip count. Shape-identical to what an
+ * ablation probe reports.
+ */
+export declare function scoreArmRuns(runs: ArmRuns, reference: string, embedder: Embedder, outcomeChanged: OutcomeComparator): Promise<AblationRunStats>;
+/**
+ * Build the inert-intervention control from the baseline arm's OWN seeded runs.
+ * See {@link NullBand} for why leave-one-out does not transfer and this does.
+ */
+export declare function nullBandFrom(baselineArmId: string, baselineStats: AblationRunStats, gates: boolean): NullBand;
+/**
+ * Did this arm actually take effect? Compares every manifest the runner
+ * reported against the arm's declared facets.
+ *
+ * `checked: false` — and therefore no refusal — in exactly two cases, both of
+ * which are absence of evidence rather than evidence of absence: the runner
+ * reported no manifest, or the arm declares only ablations (no manifest names a
+ * tool catalog). Refusing a verdict in either case would punish a consumer for
+ * a capture they were never required to make.
+ */
+export declare function checkArmApplication(arm: StrategyArm, manifests: readonly RunManifestLike[]): ArmApplication;

package/dist/esm/lib/context-bisect/arms/probe.js

@@ -0,0 +1,125 @@
+/**
+ * The arm probe — seeded runs in, shared statistics out.
+ *
+ * Split in two on purpose. `collectArmRuns` only RUNS (it cannot know the
+ * reference yet: when no prior answer was supplied, the reference IS the
+ * baseline arm's first answer). `scoreArmRuns` only SCORES. Keeping them apart
+ * is what lets the reference be chosen between the two halves without a second
+ * pass over the model — the same "one ablation, two readouts" thrift
+ * `runAblationProbe` applies to cost.
+ *
+ * Everything statistical is borrowed, not re-derived: `similarityStats`,
+ * `costStatsFrom` and `resolveSamples` come from `ablation.ts` unchanged, so an
+ * arm's `AblationRunStats` is the same number computed the same way as an
+ * ablation's — which is the point of sharing the statistics while keeping the
+ * spec types apart.
+ */
+import { cosineSimilarity } from '../../../memory/embedding/cosine.js';
+import { costStatsFrom, similarityStats } from '../ablation.js';
+import { checkArmApplied } from './manifest.js';
+import { declaredFacetCount } from './validate.js';
+/**
+ * Call the consumer's runner once per seed (0..samples-1) for ONE arm and
+ * normalize the two return shapes (`string` | `{output, cost?, manifest?}`) —
+ * the same normalization `runAblationProbe` performs, so a bare-string runner
+ * costs the caller nothing in ceremony.
+ */
+export async function collectArmRuns(arm, runner, samples) {
+    const answers = [];
+    const loops = [];
+    const tokens = [];
+    const manifests = [];
+    for (let seed = 0; seed < samples; seed++) {
+        const raw = await runner(arm, { seed });
+        if (typeof raw === 'string') {
+            answers.push(raw);
+            continue;
+        }
+        answers.push(raw.output);
+        if (raw.cost?.loops !== undefined)
+            loops.push(raw.cost.loops);
+        if (raw.cost?.tokens !== undefined)
+            tokens.push(raw.cost.tokens);
+        if (raw.manifest !== undefined)
+            manifests.push(raw.manifest);
+    }
+    return { answers, loops, tokens, manifests };
+}
+/**
+ * Score one arm's harvested answers against the reference: similarity per seed
+ * (mean/min/max/stdev) plus the outcome-flip count. Shape-identical to what an
+ * ablation probe reports.
+ */
+export async function scoreArmRuns(runs, reference, embedder, outcomeChanged) {
+    const referenceVec = await embedder.embed({ text: reference });
+    const similarities = [];
+    let flips = 0;
+    for (const answer of runs.answers) {
+        const vec = await embedder.embed({ text: answer });
+        similarities.push(cosineSimilarity(referenceVec, vec));
+        if (await outcomeChanged(reference, answer))
+            flips++;
+    }
+    const cost = costStatsFrom(runs.answers.length, runs.loops, runs.tokens);
+    return {
+        samples: runs.answers.length,
+        flips,
+        similarity: similarityStats(similarities),
+        ...(cost !== undefined ? { cost } : {}),
+    };
+}
+/**
+ * Build the inert-intervention control from the baseline arm's OWN seeded runs.
+ * See {@link NullBand} for why leave-one-out does not transfer and this does.
+ */
+export function nullBandFrom(baselineArmId, baselineStats, gates) {
+    const similarity = baselineStats.similarity;
+    const degenerate = similarity.stdev === 0;
+    const note = !gates
+        ? `Reported, not applied: a custom answerChanged comparator decides the flips, so an ` +
+            `embedding-similarity band is a different instrument and must not veto its findings. The ` +
+            `control on that axis is the baseline arm's own flip count, which must be zero.`
+        : degenerate
+            ? `The baseline arm '${baselineArmId}' reproduced identically on every seed (spread 0), so ` +
+                `the floor is a hard ${similarity.min.toFixed(3)}: any change clears it. That is honest ` +
+                `for a deterministic scenario and only as strong as the determinism.`
+            : `A challenger must sit further from the reference than the baseline arm '${baselineArmId}' ` +
+                `ever did across its own seeds (floor ${similarity.min.toFixed(3)}, spread ±${similarity.stdev.toFixed(3)}).`;
+    return { baselineArmId, similarity, floor: similarity.min, degenerate, gates, note };
+}
+/**
+ * Did this arm actually take effect? Compares every manifest the runner
+ * reported against the arm's declared facets.
+ *
+ * `checked: false` — and therefore no refusal — in exactly two cases, both of
+ * which are absence of evidence rather than evidence of absence: the runner
+ * reported no manifest, or the arm declares only ablations (no manifest names a
+ * tool catalog). Refusing a verdict in either case would punish a consumer for
+ * a capture they were never required to make.
+ */
+export function checkArmApplication(arm, manifests) {
+    const declares = declaredFacetCount(arm.facets) > 0;
+    if (!declares || manifests.length === 0) {
+        return { manifestsSeen: manifests.length, checked: false, applied: false, mismatches: [] };
+    }
+    // EVERY run must agree — one seed that quietly built the incumbent is enough
+    // to poison the arm's numbers, and it would be invisible in an average.
+    const mismatches = [];
+    const seen = new Set();
+    for (const manifest of manifests) {
+        for (const mismatch of checkArmApplied(arm, manifest)) {
+            const key = `${mismatch.facet}=${mismatch.observed ?? '<absent>'}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            mismatches.push(mismatch);
+        }
+    }
+    return {
+        manifestsSeen: manifests.length,
+        checked: true,
+        applied: mismatches.length === 0,
+        mismatches,
+    };
+}
+//# sourceMappingURL=probe.js.map

package/dist/esm/lib/context-bisect/arms/probe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"probe.js","sourceRoot":"","sources":["../../../../../src/lib/context-bisect/arms/probe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAUhE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAUnD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAgB,EAChB,MAAiB,EACjB,OAAe;IAEf,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9D,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjE,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS;YAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AAC/C,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAa,EACb,SAAiB,EACjB,QAAkB,EAClB,cAAiC;IAEjC,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACnD,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,MAAM,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC;YAAE,KAAK,EAAE,CAAC;IACvD,CAAC;IACD,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;QAC5B,KAAK;QACL,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC;QACzC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,aAAqB,EACrB,aAA+B,EAC/B,KAAc;IAEd,MAAM,UAAU,GAAG,aAAa,CAAC,UAAU,CAAC;IAC5C,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,KAAK,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,CAAC,KAAK;QACjB,CAAC,CAAC,oFAAoF;YACpF,0FAA0F;YAC1F,gFAAgF;QAClF,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,qBAAqB,aAAa,wDAAwD;gBAC1F,uBAAuB,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,yCAAyC;gBACzF,qEAAqE;YACvE,CAAC,CAAC,2EAA2E,aAAa,IAAI;gBAC5F,wCAAwC,UAAU,CAAC,GAAG,CAAC,OAAO,CAC5D,CAAC,CACF,aAAa,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAClD,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACvF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CACjC,GAAgB,EAChB,SAAqC;IAErC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACpD,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxC,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IAC7F,CAAC;IACD,6EAA6E;IAC7E,wEAAwE;IACxE,MAAM,UAAU,GAAuB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,QAAQ,IAAI,eAAe,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YACtD,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,QAAQ,IAAI,UAAU,EAAE,CAAC;YACnE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO;QACL,aAAa,EAAE,SAAS,CAAC,MAAM;QAC/B,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC;QAChC,UAAU;KACX,CAAC;AACJ,CAAC"}