cognitive-core 0.2.5 → 0.3.1

package/dist/eval/live/live-runner.d.ts

@@ -0,0 +1,609 @@
+import type { ValidationMutationMode } from '../../types/index.js';
+import type { AgentBackend, AgentDelegate, AgentDelegateOptions, AgentDelegateResult } from '../../runtime/types.js';
+import type { LLMProvider } from '../../agents/types.js';
+import type { MemorySystem } from '../../memory/system.js';
+import type { Experience } from '../../types/memory.js';
+import { type Playbook } from '../../types/index.js';
+import type { EvalResult, EvalTask, TaskResult } from '../types.js';
+import type { LiveBenchmark } from './live-benchmark.js';
+import type { AblationChannel } from '../ablation.js';
+import { type StoppingConfig } from './stopping.js';
+import { type DifficultyRegistryFile } from './difficulty-registry.js';
+/**
+ * Default per-window cap on live-replay rollouts for `--live-replay-gating`.
+ * The single-parent gate scores parent + child across a minibatch per proposal,
+ * and a scaled run (24–40 tasks) fires several proposals across its dreams; the
+ * old default (20) was quietly exhausted, silently displacing the real-verifier
+ * signal with the LLM-judge fallback. 64 comfortably covers a scaled run's
+ * replayable minibatch while still bounding runaway rollout spend. Rolling
+ * window is 1h (BudgetedReplayJudge default), so long runs refresh naturally.
+ */
+export declare const DEFAULT_LIVE_REPLAY_ROLLOUT_BUDGET = 64;
+/**
+ * Honest replayability predicate for the live-replay gate: an experience is
+ * faithfully replayable ONLY when the solve path stamped a reconstructable
+ * ReplayDescriptor carrying a stable `taskId`. Without it there is no way to
+ * re-run the CORRECT task, so the gate must fall back to the LLM judge rather
+ * than silently re-executing the last-solved task. Exported for testing.
+ */
+export declare function isTaskIdReplayable(experience: Experience): boolean;
+/**
+ * Rebuild the {@link Task} for a replay rollout from an experience, re-stamping
+ * the stable task handle (`taskId`/`family`/`difficulty`) captured at solve time
+ * onto `task.metadata` so a stateful benchmark delegate drives the EXPERIENCE's
+ * task. Exported for testing.
+ */
+export declare function reconstructTaskFromReplayDescriptor(experience: Experience): EvalTask['task'];
+/**
+ * Resolve which curated-playbook dirs a given arm seeds. `seedPlaybookDirs`
+ * apply to BOTH arms (recipe cold-start); `learningOnlySeedDirs` apply ONLY to
+ * the learning arm (oracle smoke). Seeding the cold arm with the latter would
+ * hand the baseline the answer and collapse the measured delta — the bug this
+ * isolation fixes. Exported for unit testing.
+ */
+export declare function resolveSeedDirs(seedPlaybookDirs: string[] | undefined, learningOnlySeedDirs: string[] | undefined, isLearningArm: boolean): string[];
+/**
+ * Run `fn` over `items` with at most `limit` concurrent invocations, returning
+ * results in INPUT order (not completion order). `limit <= 1` runs sequentially.
+ * A worker-pool design: `limit` workers pull the next index until exhausted.
+ * If any `fn` rejects, the whole pool rejects (callers handle per-item errors
+ * inside `fn` so this only fires on programmer error).
+ */
+export declare function mapPool<I, O>(items: I[], limit: number, fn: (item: I, index: number) => Promise<O>): Promise<O[]>;
+import type { TaskBoundaryHook } from './run-checkpoint.js';
+export type { TaskBoundaryHook } from './run-checkpoint.js';
+/** Config for a live (real-agent) evaluation run. */
+export interface LiveRunConfig {
+    benchmark: LiveBenchmark;
+    /** Tasks per arm (per seed). Keep small — every task is a real agent call. */
+    taskCount: number;
+    /** Seeds to average over. Default [1]; more seeds = more real calls. */
+    seeds: number[];
+    /** Rolling-window size for the learning curve. */
+    rollingWindow: number;
+    /** Model selector for the Claude CLI (e.g. "haiku"). */
+    model?: string;
+    /** API provider override (e.g. "bedrock", "vertex"). */
+    provider?: string;
+    /** AWS region for Bedrock provider. */
+    awsRegion?: string;
+    /** Per-call timeout (ms). */
+    timeoutMs?: number;
+    /** Suppress per-task progress logging to stderr. */
+    quiet?: boolean;
+    /** Curated playbook dirs to seed into Atlas (B1 cold-start). Empty = no seed.
+     * Seeds BOTH arms equally — for recipe packs that lift absolute performance,
+     * not the learning delta. Do NOT use for oracle playbooks (see below). */
+    seedPlaybookDirs?: string[];
+    /** Playbook dirs to seed into the LEARNING arm ONLY (cold arm stays empty).
+     * This is the oracle-smoke channel: seeding both arms would hand the answer
+     * to the baseline too, collapsing the delta. Empty = no seed. */
+    learningOnlySeedDirs?: string[];
+    /**
+     * Enable the committed cold-start seed channels in BOTH arms: bundled
+     * curated playbook packs, meta-playbook seeds, and meta-strategy routing
+     * priors (sets `validation.allowSeeds` so validation controls don't strip
+     * them). Phase-1 seed-lift A/B: run the same benchmark/seeds twice — once
+     * without this flag (baseline) and once with it (seeded) — and compare.
+     * Seeding both arms keeps the learning delta interpretable within each run;
+     * cross-run cold-arm delta isolates pure seed lift, cross-run learning-arm
+     * delta shows the seeds × learning interaction.
+     */
+    bundledSeeds?: boolean;
+    /**
+     * ADDITIVE LEARNING (default ON via `features.additiveConsolidation`) — A/B
+     * lever. When explicitly set (true OR false), it overrides the config default
+     * on the eval Atlas: `true` REFINES/REUSES existing playbooks (inline
+     * match-and-refine + injection-aligned frontier accounting); `false` restores
+     * the legacy proliferating behavior (a unique `distilled-<uuid>` per
+     * trajectory). Leave `undefined` (library callers) to inherit the config
+     * default (true). The eval CLI resolves this from `--additive` / `--no-additive`.
+     */
+    additive?: boolean;
+    /**
+     * Embedding provider type for semantic retrieval.
+     *   - `'local'` (default when embed=true): LocalEmbeddingProvider, all-MiniLM-L6-v2, 384-dim,
+     *     deterministic, no API key. First run downloads ~25MB to the HF cache.
+     *   - `'hash'`: HashEmbeddingProvider, zero-dep bag-of-words, lexical-only baseline.
+     *   - `'none'` (default when embed omitted): no embedding; text-similarity scoring.
+     * Mutually exclusive with the legacy boolean `embed`.
+     */
+    embedProvider?: 'local' | 'hash' | 'none' | 'openai' | 'auto';
+    /** @deprecated Use embedProvider instead. When true and embedProvider is unset, defaults to 'local'. */
+    embed?: boolean;
+    /**
+     * Max concurrent episodes (default 1 = sequential). The cold/no-learning arm
+     * runs independent tasks (fresh Atlas each) so they parallelize cleanly; each
+     * concurrent episode spawns its own delegate + bridge + Claude CLI call. The
+     * learning arm stays sequential (task N+1's memory depends on task N). The
+     * harness is I/O-bound (≈0% CPU while waiting on the model), so K>1 gives ~Kx
+     * speedup on the cold arm at near-zero CPU cost. Bound by model rate limits.
+     */
+    concurrency?: number;
+    /** Run only the learning arm (no cold baseline) — for ablation/attribution. */
+    singleArm?: boolean;
+    /**
+     * Per-seed arm-order alternation. When true with multiple seeds:
+     *   odd seeds  → learning first, then cold (current/default order)
+     *   even seeds → cold first, then learning
+     * Single seed: flag is a no-op (warning logged).
+     */
+    interleaveArms?: boolean;
+    /** Fired after each task completes — for progress UI, checkpointing, etc. */
+    onTaskBoundary?: TaskBoundaryHook;
+    /**
+     * Reuse cold-arm results from a prior run instead of re-running the arm
+     * (see cold-reuse.ts). The cold arm can't see seeds/memory/config-under-test,
+     * so across A/B arms of the same {benchmark, tasks, model} it re-measures an
+     * identical control — reuse halves the cost of every arm after the first.
+     * Keyed by seed; rows are validated against this run's task stream (count +
+     * per-position taskId) and REFUSED on mismatch (falls back to running cold).
+     */
+    reuseColdBySeed?: Record<number, TaskResult[]>;
+    /**
+     * Sequential stopping (Phase B, DESIGN-meta-learning-validation.md §10):
+     * after each completed seed, evaluate threshold/futility/precision rules on
+     * the pooled two-arm posterior and skip the remaining seeds when the
+     * learning-vs-cold question is already answered (or provably unanswerable
+     * within the remaining budget). Only applies to multi-seed two-arm runs;
+     * checks happen at seed boundaries only (≤ seeds-1 peeks). The final report
+     * aggregates over the seeds actually run.
+     */
+    stopping?: {
+        enabled: boolean;
+    } & StoppingConfig;
+    /**
+     * Discriminative task selection (Phase B(2), design doc §10): oversample the
+     * benchmark pool, then weight selection toward families in the p≈0.5
+     * discriminative band using the accumulated difficulty registry. Advisory:
+     * families with < minRuns ingested runs sample uniformly. Selection is
+     * deterministic per (registry, benchmark, seed), so both arms of a seed see
+     * the identical stream. NOTE: changes the estimand — numbers from a
+     * discriminative stream are not comparable to fixed-stream runs.
+     */
+    taskSelection?: {
+        mode: 'discriminative';
+        registry: DifficultyRegistryFile;
+        /** Pool multiplier before down-selection (default 3× taskCount). */
+        oversample?: number;
+        /** Runs per family before its weight becomes non-uniform (default 3). */
+        minRuns?: number;
+    };
+    /**
+     * Fail fast on infrastructure errors, trading retry resilience for wall time:
+     *   - model/CLI step calls run a single attempt (no in-call retries)
+     *   - an env_error episode is recorded immediately (no full-episode retry)
+     * Failure-heavy runs spend 3-5x longer on doomed episodes than passing ones;
+     * with fail-fast a hung call costs one timeout instead of up to 3, and an
+     * infra-dead episode is not re-run. env_error rows stay excluded from
+     * success-rate aggregates, so this changes cost, not measurement.
+     */
+    failFast?: boolean;
+    /** Partial AtlasConfig override — lets benchmarks tweak Atlas behavior. */
+    configPatch?: Record<string, unknown>;
+    /**
+     * Validation mutation mode for live eval Atlas instances. Defaults to
+     * `active` so the learning arm actually learns (dream-queue writes on).
+     * The original `frozen` default silently disabled playbook extraction in
+     * every live run 2026-06-27 → 2026-07-01 (see the harness-defect entry in
+     * docs/DESIGN-meta-learning-validation.md). Pass `frozen` explicitly for
+     * retrieval-only smokes with seeded playbooks.
+     */
+    mutationMode?: ValidationMutationMode;
+    /**
+     * Directory to write per-task JSONL trace files (including diagnostics entries).
+     * When undefined, tracing is disabled. Created automatically if missing.
+     */
+    traceDir?: string;
+    /**
+     * Backends for offline workspace analysis agents. When present, live evals keep
+     * benchmark delegates on the solve path but use these backends for dream-phase
+     * playbook extraction.
+     */
+    analysisBackends?: AgentBackend[];
+    /**
+     * File or directory paths containing meta-playbook JSON files. These seed the
+     * isolated meta-playbook bank used by learning-analysis agents such as the
+     * workspace playbook extractor; they are not inserted into the solver-facing
+     * task playbook library.
+     */
+    metaPlaybookDirs?: string[];
+    /** Playbook retriever used for solver-facing playbook injection. */
+    playbookRetriever?: 'legacy' | 'skill-tree-hybrid';
+    /** Minimum playbook relevance score required for injection. */
+    minPlaybookScore?: number;
+    /** Exp 4: playbook incorporation mode — 'full' or 'sectioned' (task-relevant subset). */
+    playbookIncorporation?: 'full' | 'sectioned';
+    /** Enable the opt-in post-retrieval playbook applicability gate. */
+    playbookApplicabilityGate?: boolean;
+    /**
+     * Procedural-memory authority for live eval Atlas instances. The default is
+     * SQLite-backed temp storage; set to `filesystem` with an absolute
+     * `proceduralMemoryDir` to validate migrated `.atlas/playbooks/<name>/SKILL.md`
+     * artifacts in the otherwise isolated live harness.
+     */
+    proceduralMemorySource?: 'sqlite' | 'filesystem';
+    /** Canonical procedural-memory directory. Relative paths are resolved by Atlas against its eval temp base. */
+    proceduralMemoryDir?: string;
+    /** Fail Atlas startup if filesystem procedural memory has warnings. */
+    proceduralMemoryStrictStartup?: boolean;
+    /** Prune SQLite cache entries that are not represented by canonical files. */
+    proceduralMemoryPruneSqliteOnly?: boolean;
+    /**
+     * Influence sampling: every Nth task (learning arm only), call
+     * `AgentManager.compareWithKnowledge` and record whether the injected context
+     * changed the agent's output. 0 = off (default). Set to 5 for the oracle run.
+     */
+    influenceSampleEvery?: number;
+    /**
+     * Trace-only ablation awareness: channels stripped by the live delegate before
+     * the prompt reaches the benchmark agent. Keeps diagnostics aligned with the
+     * delivered prompt instead of the pre-ablation Atlas retrieval result.
+     */
+    diagnosticStripChannels?: AblationChannel[];
+    /**
+     * Enable LLM-backed distillation on the LEARNING arm only.
+     *
+     * When true, a ClaudeCliLLMProvider (using this run's model/provider/awsRegion)
+     * is wired into the learning-arm Atlas via `setLearningProvider`. This causes
+     * the distiller and convention-distiller to run with a real model rather than
+     * falling back to heuristics.
+     *
+     * The cold arm (runCold) never receives a learning provider — its Atlas is
+     * stateless by design.
+     *
+     * For ALFWorld (playbookMinTrajectories undefined / extraction off by default),
+     * enabling distillation also lowers minTrajectories to `distillMinTrajectories`
+     * (default 3) so the mid-run dream actually fires and exercises the distiller
+     * before the run ends.
+     */
+    distill?: boolean;
+    /**
+     * When `distill` is true and the benchmark has no explicit
+     * `playbookMinTrajectories`, use this threshold to enable extraction so the
+     * distiller fires mid-stream. Default: 3.
+     */
+    distillMinTrajectories?: number;
+    /**
+     * Wire the playbook mutation/merge gating stack on the LEARNING arm so eval
+     * runs can exercise the GEPA-style optimization loop. When true:
+     *
+     *   1. A learning LLM provider is bound via `setLearningProvider` even
+     *      without `--distill` — installing the LLMJudgeReplay gate judge and
+     *      the reflective-mutation generator (without a judge, the
+     *      `reflective-mutation` and `pareto-merge-proposing` maintenance
+     *      tasks self-gate to no-ops).
+     *   2. The maintenance scheduler is pinned to fire after EVERY dream
+     *      (batchInterval: 1; also the production default) so short eval
+     *      streams reach the mutation tasks regardless of config drift.
+     *   3. Like `distill`, extraction is enabled at `distillMinTrajectories`
+     *      when the benchmark doesn't set `playbookMinTrajectories`, so mid-run
+     *      dreams actually occur.
+     *
+     * The cold arm never receives any of this — it stays stateless. Library
+     * default false (tests and embedders opt in explicitly); the eval CLI
+     * defaults it ON for non-frozen runs (`--no-mutation-gating` opts out) so
+     * the measured learning arm exercises the full loop by default.
+     */
+    mutationGating?: boolean;
+    /**
+     * FIX 1 — gate playbook mutations on the REAL task verifier instead of the
+     * decoupled haiku LLM-judge proxy. When true (and `mutationGating` is on),
+     * `wireLearningArmMutation` installs a {@link ConditionalReplayJudge} via
+     * `atlas.enableLiveReplayGating`: faithfully-replayable experiences are
+     * re-executed through the benchmark delegate and scored by the domain
+     * `OutcomeVerifier` (for benchmark delegates `result.success` IS the
+     * ground-truth oracle), and only non-replayable experiences fall back to
+     * the LLM judge.
+     *
+     * DEFAULT OFF (opt-in). In the current live-runner topology, replay
+     * re-executes through the SHARED benchmark delegate — invasive to its
+     * per-task/cost state and expensive (real model calls). So it is left off
+     * by default and the choice (live vs LLM judge) is logged explicitly; turn
+     * it on with `--live-replay-gating` for a run that wants real-verifier
+     * gating. `liveReplayMaxRolloutsPerWindow` bounds the rollout spend.
+     */
+    liveReplayGating?: boolean;
+    /**
+     * Cap on live-replay rollouts per rolling window when `liveReplayGating` is
+     * on (routes overflow to the LLM-judge fallback via `BudgetedReplayJudge`).
+     * Defaults to {@link DEFAULT_LIVE_REPLAY_ROLLOUT_BUDGET} — sized so a scaled
+     * (24–40 task) run's real-verifier scoring isn't quietly displaced by the
+     * LLM-judge fallback. Ignored when `liveReplayGating` is off.
+     */
+    liveReplayMaxRolloutsPerWindow?: number;
+    /**
+     * Override for the learning-arm LLM provider used by `distill` /
+     * `mutationGating` (tests inject a deterministic fake here). Defaults to a
+     * ClaudeCliLLMProvider built from this run's model/provider/awsRegion.
+     */
+    learningLlmProvider?: LLMProvider;
+    /**
+     * F4: Enable the B3 in-episode think-act-refine loop for interactive
+     * benchmarks that support it (currently ALFWorld only).
+     *
+     * When true, the AlfworldDelegate runs the inner THINK/PRUNE/ACT loop each
+     * environment step, letting the agent curate the injected knowledge set
+     * mid-episode.  Prune verdicts flow into the persistent layers (playbook
+     * failureCount, MetaMemory misleading retrieval).
+     *
+     * This is the experiment-config counterpart to the `--refine` CLI flag.
+     * Both enable the same delegate option; they compose (either can enable it).
+     *
+     * NOTE: refine is ALFWorld-specific.  Generalising it to SkillFlow / other
+     * interactive delegates is a follow-up (F4-ext) — for now only AlfworldBenchmark
+     * reads this field.  F5's injection-level gating is the cross-benchmark mechanism.
+     *
+     * Default false (preserves prior behavior).
+     */
+    refine?: boolean;
+    /**
+     * Complete train/eval isolation for measuring benchmark lift. When enabled,
+     * each seed's stream is deterministically partitioned PER FAMILY into a TRAIN
+     * split (the learner extracts playbooks from these — oracle on) and a
+     * disjoint held-out EVAL split (scored — oracle off, solver firewall intact,
+     * learner frozen). Only the EVAL split feeds the reported metric, so the
+     * learner is never measured on a task it trained on. The cold baseline is run
+     * over the SAME held-out EVAL split.
+     *
+     * See docs/DESIGN-meta-learning-validation.md § "Train-time oracle signal vs
+     * solve-time firewall". Deterministic + seeded via `splitSeed`.
+     */
+    trainEvalSplit?: {
+        enabled: boolean;
+        /** Fraction of each family's tasks used for TRAINING (default 0.5). */
+        trainRatio?: number;
+        /** Deterministic seed offset for the per-family partition (default 0). */
+        splitSeed?: number;
+    };
+    /**
+     * Stable working directory for RESUMABLE learning-arm state. When set, each
+     * seed's learning/train Atlas `baseDir` is a deterministic subdir
+     * (`<runWorkDir>/learn-seed-<seed>`, `<runWorkDir>/train-seed-<seed>`) instead
+     * of an ephemeral `mkdtempSync` temp dir, and it is NOT deleted in `finally`
+     * — so if the run dies, the persisted SQLite (`eval.db`) + filesystem
+     * playbooks survive and reopening the same dir restores accumulated memory.
+     *
+     * The cold arm keeps temp dirs (it is stateless per task); its already-paid
+     * episodes are recovered via the checkpoint on `resumeFrom`, not this dir.
+     *
+     * The caller owns cleanup on SUCCESS (mirroring checkpoint `finalize()`); the
+     * runner never removes it, so an aborted run leaves it for `--resume`.
+     */
+    runWorkDir?: string;
+    /**
+     * Resume an aborted run from its `<out>.checkpoint.json` partial file. When
+     * set, each (arm, seed) SKIPS the task indices already recorded there and
+     * adopts their `TaskResult`s, so the aggregated report is identical to an
+     * uninterrupted run. The learning arm resumes at the first incomplete index
+     * for its seed after reopening the persisted `runWorkDir` baseDir (so memory
+     * reflects the already-completed tasks); a fully-complete seed skips Atlas
+     * construction entirely. Pair with `runWorkDir` pointing at the same dir the
+     * aborted run used.
+     */
+    resumeFrom?: string;
+}
+/**
+ * Deterministically partition a task stream into disjoint TRAIN and held-out
+ * EVAL splits, PER FAMILY, so both splits share families (the precondition for
+ * measuring transfer: trained on task A's oracle → helps held-out task B in the
+ * same family). Seeded, so both arms of a seed see the identical partition.
+ *
+ * Rules:
+ *   - Group by `family` (fallback `skillTag`), preserving stream order.
+ *   - A singleton family cannot be held out → its lone task goes to TRAIN.
+ *   - Otherwise trainCount = round(k * trainRatio), clamped to [1, k-1] so both
+ *     sub-splits are non-empty for every multi-task family.
+ *   - Selection uses a per-family seeded shuffle so train/eval both span the
+ *     family's difficulty range rather than a clean early/late cut.
+ *
+ * Exported for unit testing.
+ */
+export declare function partitionTrainEval(stream: EvalTask[], trainRatio: number, splitSeed: number): {
+    train: EvalTask[];
+    evalHeldOut: EvalTask[];
+};
+/** A delegate plus an optional hook to announce the task about to be solved. */
+export interface BenchmarkDelegate extends AgentDelegate {
+    setCurrentTask?(task: EvalTask['task']): void;
+    getTotalCostUsd?(): number;
+    /**
+     * Isolation hooks for the mutation-gate live-replay path: read the current
+     * task and overwrite the accumulated cost so `spawnViaDelegate` can save +
+     * restore the delegate's mutable state around an isolated replay rollout (a
+     * rollout must not corrupt the primary run's cost or leave the delegate
+     * pointed at the wrong task). Optional — a delegate without them simply
+     * can't be isolated (and won't be live-replayed).
+     */
+    getCurrentTask?(): EvalTask['task'] | undefined;
+    setTotalCostUsd?(value: number): void;
+    /** Give the delegate a memory handle so refine prune verdicts can persist. */
+    setMemory?(memory: MemorySystem): void;
+}
+/** Factory an interactive benchmark exposes to build its multi-turn delegate. */
+export type DelegateFactory = (benchmark: LiveBenchmark, opts: {
+    model?: string;
+    timeoutMs?: number;
+    provider?: string;
+    awsRegion?: string;
+    refine?: boolean;
+    /** Per-model-call retry override (failFast → 0 = single attempt). */
+    cliRetries?: number;
+}) => BenchmarkDelegate;
+/**
+ * Live learning-curve runner.
+ *
+ * Same two-arm design as the mock runner, but every task is solved by a REAL
+ * agent (Claude CLI) and scored against ground truth:
+ *   - "learning": one Atlas carries memory across the stream — accumulated
+ *     experiences/playbooks are injected into the agent's context.
+ *   - "no-learning": a fresh cold Atlas per task — the agent solves each task
+ *     with no injected memory (the stateless baseline).
+ *
+ * The delta answers the real question the mock can only simulate: does
+ * cognitive-core's accumulated memory actually improve task success?
+ *
+ * Single-turn benchmarks (GSM8K) use the ClaudeBenchmarkDelegate. Interactive
+ * benchmarks (ALFWorld) supply their own multi-turn delegate via
+ * `delegateFactory`.
+ */
+export declare class LiveEvalRunner {
+    private readonly config;
+    /** Override for interactive benchmarks that need a multi-turn delegate. */
+    private readonly delegateFactory?;
+    /**
+     * Mutation/merge activity accumulated across learning-arm seeds (summed).
+     * Populated after each learning-arm Atlas closes; attached to the learning
+     * ConditionResult so run reports prove whether the mutation path executed.
+     */
+    private learningMutationActivity;
+    /** Adopted per-(arm, seed) results from a prior aborted run, or null. */
+    private readonly resume;
+    constructor(config: LiveRunConfig, 
+    /** Override for interactive benchmarks that need a multi-turn delegate. */
+    delegateFactory?: DelegateFactory | undefined);
+    /** Completed results for (arm, seed) from the resume checkpoint, if any. */
+    private completed;
+    /**
+     * Resolve the learning/train Atlas baseDir for a seed. With `runWorkDir`, this
+     * is a STABLE, deterministic subdir that survives process death (persistent =
+     * true → the caller must not delete it in `finally`). Without it, an ephemeral
+     * temp dir (persistent = false → deleted in `finally` as before).
+     */
+    private learningBaseDir;
+    run(): Promise<EvalResult>;
+    /**
+     * Load the task stream for a seed. With discriminative task selection,
+     * oversample the pool and down-select toward the p≈0.5 band using the
+     * difficulty registry; otherwise load exactly taskCount tasks.
+     */
+    private loadStream;
+    private runArm;
+    private log;
+    /**
+     * Prior cold-arm rows for this seed, or undefined when absent/mismatched.
+     * Mismatch (different task count or taskIds) means the prior run used a
+     * different config — adopting it would compare arms across different task
+     * sets, so we warn and fall back to actually running the cold arm.
+     */
+    private reusableColdRows;
+    /** Learning arm: one Atlas + delegate carries memory across the stream.
+     * env_error tasks are retried once; if they still env_error, they are
+     * recorded with status='env_error' and excluded from learning writes. */
+    private runCarry;
+    /**
+     * Wire learning-arm-only mutation levers onto a freshly-built Atlas:
+     * bind the learning LLM provider when `distill` or `mutationGating` is on
+     * (installs the LLMJudgeReplay gate judge + reflective generator), and —
+     * for `mutationGating` — make the maintenance scheduler fire after every
+     * dream so short eval streams reach the mutation tasks at all.
+     */
+    private wireLearningArmMutation;
+    /**
+     * FIX 1 — pick the mutation-gate judge and log the choice explicitly.
+     *
+     * With `--live-replay-gating`, install a {@link ConditionalReplayJudge}
+     * (via `atlas.enableLiveReplayGating`) so faithfully-replayable experiences
+     * are gated on the REAL verifier (re-execution outcome — the same
+     * ground-truth signal the harness computes) and only non-replayable ones
+     * fall back to the LLM judge. Otherwise the LLM judge stays — but the
+     * decision is always logged so a run's gating signal is never ambiguous.
+     *
+     * FAITHFUL REPLAY (see docs/DESIGN-meta-learning-validation.md §8): the gate
+     * is only honest if the rollout re-runs the CORRECT task. We route by the
+     * ReplayDescriptor the solve path stamps onto each experience — an experience
+     * is replayable ONLY when it carries a stable `taskId` — and reconstruct that
+     * task (with its `taskId`/`family`/`difficulty` metadata) so the stateful
+     * benchmark delegate drives the right task. Experiences without a descriptor
+     * (no taskId) honestly fall back to the LLM judge instead of silently
+     * re-running the last-solved task. The rollout itself is isolated from the
+     * solve path's cost/current-task state (see `AgentSpawnConfig.replay`).
+     */
+    private wireGateJudge;
+    /**
+     * Read the learning-arm pipeline's accumulated mutation counters and fold
+     * them into the run-level accumulator (summed across seeds). Reads
+     * in-memory state only, so it is safe after `atlas.close()`.
+     */
+    private captureMutationActivity;
+    /** Attach the accumulated learning-arm mutation counters to the report row. */
+    private attachMutationActivity;
+    /**
+     * Train/eval isolation arm: one Atlas trains on the TRAIN split (oracle on,
+     * playbooks extracted mid-stream), is flushed + FROZEN, then scored on the
+     * disjoint held-out EVAL split with the trained memory served read-only.
+     * Returns ONLY the held-out eval results (the measured set).
+     *
+     * The freeze (`setValidationMutationMode('frozen')`) is what makes the
+     * isolation complete: during eval, `processTrajectory` is a no-op, so eval
+     * trajectories are neither stored nor extracted — the learner never trains on
+     * a task it is measured on. Retrieval/serving stay live, so trained playbooks
+     * are still injected. See docs/DESIGN-meta-learning-validation.md §
+     * "Train-time oracle signal vs solve-time firewall".
+     */
+    private runTrainThenEval;
+    /** No-learning arm: a fresh cold Atlas + delegate per task.
+     * env_error tasks are retried once; on second failure recorded as env_error. */
+    private runCold;
+    /**
+     * Solve a single task with one automatic retry on env_error.
+     * On second env_error, return a result with status='env_error'.
+     * env_error results are excluded from success-rate aggregates and learning.
+     */
+    private solveWithRetry;
+    private solveOne;
+    private makeDelegate;
+    private makeAtlas;
+    private loadMetaPlaybooks;
+}
+export declare function findPlaybookJsonFiles(inputPath: string): string[];
+export declare function revivePlaybook(raw: Partial<Playbook>): Playbook;
+export declare function createLiveEvalRunner(config: LiveRunConfig, delegateFactory?: DelegateFactory): LiveEvalRunner;
+/**
+ * Ablation config: describes one component to disable and re-run.
+ */
+export interface AblationArm {
+    label: string;
+    /** Atlas config patch applied for this arm (deep-merged over the base config). */
+    configPatch: Record<string, unknown>;
+    /**
+     * Knowledge channels to strip from the agent's injected context for this arm.
+     * Mirrors the mock-backend AblatingBackend mechanism for the live path.
+     */
+    channels?: AblationChannel[];
+}
+/**
+ * Delegate wrapper that strips chosen knowledge channels before forwarding to
+ * the inner delegate. Mirrors AblatingBackend for the live path.
+ *
+ * IMPORTANT: live delegates consume the pre-formatted `systemContext` STRING
+ * (built by AgentManager.spawnViaDelegate before the delegate is called), not
+ * the structured `injectedKnowledge` object — so both must be stripped.
+ * Stripping only the object would be a silent no-op on the live path.
+ */
+export declare class ChannelStrippingDelegate implements AgentDelegate {
+    private readonly inner;
+    private readonly channels;
+    constructor(inner: AgentDelegate, channels: AblationChannel[]);
+    execute(prompt: string, options: AgentDelegateOptions): Promise<AgentDelegateResult>;
+}
+/**
+ * Live ablation runner: runs the learning arm multiple times, each time
+ * disabling one component, to measure each component's contribution.
+ * Results are single-arm ConditionResults that can be compared to a full run.
+ */
+export declare class LiveAblationRunner {
+    private readonly baseConfig;
+    private readonly arms;
+    private readonly delegateFactory?;
+    constructor(baseConfig: LiveRunConfig, arms: AblationArm[], delegateFactory?: DelegateFactory | undefined);
+    run(): Promise<{
+        full: EvalResult;
+        ablations: Array<{
+            label: string;
+            result: EvalResult;
+        }>;
+    }>;
+}
+export declare function createLiveAblationRunner(baseConfig: LiveRunConfig, arms: AblationArm[], delegateFactory?: DelegateFactory): LiveAblationRunner;
+//# sourceMappingURL=live-runner.d.ts.map

package/dist/eval/live/live-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"live-runner.d.ts","sourceRoot":"","sources":["../../../src/eval/live/live-runner.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAe,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAIhF,OAAO,KAAK,EACV,YAAY,EACZ,aAAa,EACb,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAA4C,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAG/F,OAAO,KAAK,EAAmB,UAAU,EAAiB,QAAQ,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpG,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAItD,OAAO,EAAmC,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACrF,OAAO,EAA6B,KAAK,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AASlG;;;;;;;;GAQG;AACH,eAAO,MAAM,kCAAkC,KAAK,CAAC;AAErD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAGlE;AAED;;;;;GAKG;AACH,wBAAgB,mCAAmC,CAAC,UAAU,EAAE,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAW5F;AA2BD;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,gBAAgB,EAAE,MAAM,EAAE,GAAG,SAAS,EACtC,oBAAoB,EAAE,MAAM,EAAE,GAAG,SAAS,EAC1C,aAAa,EAAE,OAAO,GACrB,MAAM,EAAE,CAEV;AAED;;;;;;GAMG;AACH,wBAAsB,OAAO,CAAC,CAAC,EAAE,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GACzC,OAAO,CAAC,CAAC,EAAE,CAAC,CAad;AA0BD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,YAAY,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,qDAAqD;AACrD,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,aAAa,CAAC;IACzB,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,kDAAkD;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,wDAAwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;8EAE0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;;qEAEiE;IACjE,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC;;;;;;;;;OASG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC9D,wGAAwG;IACxG,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAC/C;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,GAAG,cAAc,CAAC;IACjD;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,gBAAgB,CAAC;QACvB,QAAQ,EAAE,sBAAsB,CAAC;QACjC,oEAAoE;QACpE,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,yEAAyE;QACzE,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,sBAAsB,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,YAAY,EAAE,CAAC;IAClC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,QAAQ,GAAG,mBAAmB,CAAC;IACnD,+DAA+D;IAC/D,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,yFAAyF;IACzF,qBAAqB,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;IAC7C,oEAAoE;IACpE,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC;IACjD,8GAA8G;IAC9G,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,uEAAuE;IACvE,6BAA6B,CAAC,EAAE,OAAO,CAAC;IACxC,8EAA8E;IAC9E,+BAA+B,CAAC,EAAE,OAAO,CAAC;IAC1C;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,eAAe,EAAE,CAAC;IAC5C;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;;;;;;;;;;;OAgBG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;OAMG;IACH,8BAA8B,CAAC,EAAE,MAAM,CAAC;IACxC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC;IAClC;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;;;;;;;OAWG;IACH,cAAc,CAAC,EAAE;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,uEAAuE;QACvE,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,0EAA0E;QAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF;;;;;;;;;;;;;OAaG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,QAAQ,EAAE,EAClB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAChB;IAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAAC,WAAW,EAAE,QAAQ,EAAE,CAAA;CAAE,CA4BhD;AAED,gFAAgF;AAChF,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACtD,cAAc,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAC9C,eAAe,CAAC,IAAI,MAAM,CAAC;IAC3B;;;;;;;OAOG;IACH,cAAc,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;IAChD,eAAe,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,8EAA8E;IAC9E,SAAS,CAAC,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;CACxC;AAED,iFAAiF;AACjF,MAAM,MAAM,eAAe,GAAG,CAC5B,SAAS,EAAE,aAAa,EACxB,IAAI,EAAE;IACJ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,qEAAqE;IACrE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,KACE,iBAAiB,CAAC;AAEvB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,cAAc;IAYvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,2EAA2E;IAC3E,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;IAbnC;;;;OAIG;IACH,OAAO,CAAC,wBAAwB,CAAsC;IAEtE,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAGzB,MAAM,EAAE,aAAa;IACtC,2EAA2E;IAC1D,eAAe,CAAC,EAAE,eAAe,YAAA;IAOpD,4EAA4E;IAC5E,OAAO,CAAC,SAAS;IAIjB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAejB,GAAG,IAAI,OAAO,CAAC,UAAU,CAAC;IA4JhC;;;;OAIG;YACW,UAAU;YAyBV,MAAM;IAapB,OAAO,CAAC,GAAG;IAIX;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;6EAEyE;YAC3D,QAAQ;IA6FtB;;;;;;OAMG;IACH,OAAO,CAAC,uBAAuB;IAoB/B;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,aAAa;IA2CrB;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;IAuD/B,+EAA+E;IAC/E,OAAO,CAAC,sBAAsB;IAM9B;;;;;;;;;;;;OAYG;YACW,gBAAgB;IA2E9B;oFACgF;YAClE,OAAO;IAmDrB;;;;OAIG;YACW,cAAc;YAyDd,QAAQ;IAwNtB,OAAO,CAAC,YAAY;IAepB,OAAO,CAAC,SAAS;YA2LH,iBAAiB;CAehC;AAED,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAqBjE;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAmC/D;AAED,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,aAAa,EACrB,eAAe,CAAC,EAAE,eAAe,GAChC,cAAc,CAEhB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAC;CAC9B;AAyBD;;;;;;;;GAQG;AACH,qBAAa,wBAAyB,YAAW,aAAa;IAE1D,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBADR,KAAK,EAAE,aAAa,EACpB,QAAQ,EAAE,eAAe,EAAE;IAG9C,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAyBrF;AAED;;;;GAIG;AACH,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAFhB,UAAU,EAAE,aAAa,EACzB,IAAI,EAAE,WAAW,EAAE,EACnB,eAAe,CAAC,EAAE,eAAe,YAAA;IAG9C,GAAG,IAAI,OAAO,CAAC;QACnB,IAAI,EAAE,UAAU,CAAC;QACjB,SAAS,EAAE,KAAK,CAAC;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,UAAU,CAAA;SAAE,CAAC,CAAC;KACzD,CAAC;CAmDH;AAED,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,aAAa,EACzB,IAAI,EAAE,WAAW,EAAE,EACnB,eAAe,CAAC,EAAE,eAAe,GAChC,kBAAkB,CAEpB"}