cognitive-core 0.2.5 → 0.3.1

package/dist/eval/live/live-runner.js

@@ -0,0 +1,1409 @@
+import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createAtlasWithAgents } from '../../atlas.js';
+import { inferPlaybookSkillLevel } from '../../types/playbook.js';
+import { createEmbeddingProvider, embeddingConfigFromEnv } from '../../embeddings/provider.js';
+import { createClaudeCliLLMProvider } from './claude-cli-llm-provider.js';
+import { createPlaybook, createTask } from '../../types/index.js';
+import { aggregateCondition, ciDisjoint, normalizedGain } from '../metrics.js';
+import { seededRng } from '../rng.js';
+import { ClaudeBenchmarkDelegate } from './claude-delegate.js';
+import { EnvBridgeError } from './env-bridge.js';
+import { createTraceWriter } from './trace.js';
+import { coldRowsMatchStream } from './cold-reuse.js';
+import { loadResumeState } from './run-resume.js';
+import { evaluateStopping, poolArmCounts } from './stopping.js';
+import { selectDiscriminativeTasks } from './difficulty-registry.js';
+import { diagnosticsToMechanismTraceEvents } from '../mechanism-trace-adapter.js';
+import { emitMechanismTraceAsLiveTraceEntry, } from '../../learning/mechanism-trace.js';
+import { readReplayDescriptor } from '../../learning/replay-types.js';
+/**
+ * Default per-window cap on live-replay rollouts for `--live-replay-gating`.
+ * The single-parent gate scores parent + child across a minibatch per proposal,
+ * and a scaled run (24–40 tasks) fires several proposals across its dreams; the
+ * old default (20) was quietly exhausted, silently displacing the real-verifier
+ * signal with the LLM-judge fallback. 64 comfortably covers a scaled run's
+ * replayable minibatch while still bounding runaway rollout spend. Rolling
+ * window is 1h (BudgetedReplayJudge default), so long runs refresh naturally.
+ */
+export const DEFAULT_LIVE_REPLAY_ROLLOUT_BUDGET = 64;
+/**
+ * Honest replayability predicate for the live-replay gate: an experience is
+ * faithfully replayable ONLY when the solve path stamped a reconstructable
+ * ReplayDescriptor carrying a stable `taskId`. Without it there is no way to
+ * re-run the CORRECT task, so the gate must fall back to the LLM judge rather
+ * than silently re-executing the last-solved task. Exported for testing.
+ */
+export function isTaskIdReplayable(experience) {
+    const d = readReplayDescriptor(experience);
+    return !!d && d.reconstructable && typeof d.envHandle?.taskId === 'string';
+}
+/**
+ * Rebuild the {@link Task} for a replay rollout from an experience, re-stamping
+ * the stable task handle (`taskId`/`family`/`difficulty`) captured at solve time
+ * onto `task.metadata` so a stateful benchmark delegate drives the EXPERIENCE's
+ * task. Exported for testing.
+ */
+export function reconstructTaskFromReplayDescriptor(experience) {
+    const h = readReplayDescriptor(experience)?.envHandle ?? {};
+    return createTask({
+        description: experience.taskInput,
+        domain: experience.domain,
+        metadata: {
+            taskId: h.taskId,
+            ...(h.family !== undefined ? { family: h.family } : {}),
+            ...(h.difficulty !== undefined ? { difficulty: h.difficulty } : {}),
+        },
+    });
+}
+/**
+ * Recursively merge `patch` into `base`. Plain objects are merged recursively;
+ * arrays and primitives replace the existing value. Neither argument is mutated.
+ */
+function deepMerge(base, patch) {
+    const result = { ...base };
+    for (const key of Object.keys(patch)) {
+        const bv = result[key];
+        const pv = patch[key];
+        if (pv !== null &&
+            typeof pv === 'object' &&
+            !Array.isArray(pv) &&
+            bv !== null &&
+            typeof bv === 'object' &&
+            !Array.isArray(bv)) {
+            result[key] = deepMerge(bv, pv);
+        }
+        else {
+            result[key] = pv;
+        }
+    }
+    return result;
+}
+/**
+ * Resolve which curated-playbook dirs a given arm seeds. `seedPlaybookDirs`
+ * apply to BOTH arms (recipe cold-start); `learningOnlySeedDirs` apply ONLY to
+ * the learning arm (oracle smoke). Seeding the cold arm with the latter would
+ * hand the baseline the answer and collapse the measured delta — the bug this
+ * isolation fixes. Exported for unit testing.
+ */
+export function resolveSeedDirs(seedPlaybookDirs, learningOnlySeedDirs, isLearningArm) {
+    return [...(seedPlaybookDirs ?? []), ...(isLearningArm ? (learningOnlySeedDirs ?? []) : [])];
+}
+/**
+ * Run `fn` over `items` with at most `limit` concurrent invocations, returning
+ * results in INPUT order (not completion order). `limit <= 1` runs sequentially.
+ * A worker-pool design: `limit` workers pull the next index until exhausted.
+ * If any `fn` rejects, the whole pool rejects (callers handle per-item errors
+ * inside `fn` so this only fires on programmer error).
+ */
+export async function mapPool(items, limit, fn) {
+    const results = new Array(items.length);
+    const n = items.length;
+    const workers = Math.max(1, Math.min(limit, n));
+    let next = 0;
+    async function worker() {
+        while (next < n) {
+            const i = next++;
+            results[i] = await fn(items[i], i);
+        }
+    }
+    await Promise.all(Array.from({ length: workers }, () => worker()));
+    return results;
+}
+/** Whether an error thrown by the agent/CLI represents an infrastructure failure
+ * rather than a legitimate agent failure. */
+function isEnvError(err) {
+    // EnvBridgeError is always an infra failure.
+    if (err instanceof EnvBridgeError)
+        return true;
+    const msg = err instanceof Error ? err.message : String(err);
+    // API rate-limit / overload
+    if (/429|529|rate.?limit|too many requests|overloaded/i.test(msg))
+        return true;
+    // Timeout
+    if (/timed? out|timeout/i.test(msg))
+        return true;
+    // CLI spawn or non-zero exit with no output
+    if (/Failed to spawn|exited [1-9]\d*(?:\b|:)|not found in PATH/i.test(msg))
+        return true;
+    // Bridge process died
+    if (/env bridge|bridge.*exit|Could not start env bridge/i.test(msg))
+        return true;
+    return false;
+}
+function previewOutcome(outcome, maxLength = 2000) {
+    const value = outcome.solution ?? outcome.errorInfo;
+    if (value === undefined || value === null)
+        return undefined;
+    const text = typeof value === 'string' ? value : JSON.stringify(value);
+    return text.length > maxLength ? `${text.slice(0, maxLength)}...` : text;
+}
+/**
+ * Deterministically partition a task stream into disjoint TRAIN and held-out
+ * EVAL splits, PER FAMILY, so both splits share families (the precondition for
+ * measuring transfer: trained on task A's oracle → helps held-out task B in the
+ * same family). Seeded, so both arms of a seed see the identical partition.
+ *
+ * Rules:
+ *   - Group by `family` (fallback `skillTag`), preserving stream order.
+ *   - A singleton family cannot be held out → its lone task goes to TRAIN.
+ *   - Otherwise trainCount = round(k * trainRatio), clamped to [1, k-1] so both
+ *     sub-splits are non-empty for every multi-task family.
+ *   - Selection uses a per-family seeded shuffle so train/eval both span the
+ *     family's difficulty range rather than a clean early/late cut.
+ *
+ * Exported for unit testing.
+ */
+export function partitionTrainEval(stream, trainRatio, splitSeed) {
+    const groups = new Map();
+    for (const item of stream) {
+        const key = item.family ?? item.skillTag ?? 'default';
+        const arr = groups.get(key) ?? [];
+        arr.push(item);
+        groups.set(key, arr);
+    }
+    const train = [];
+    const evalHeldOut = [];
+    for (const [family, items] of groups) {
+        if (items.length === 1) {
+            train.push(items[0]);
+            continue;
+        }
+        const rng = seededRng(`train-eval:${family}`, splitSeed);
+        const order = items.map((_, i) => i);
+        for (let i = order.length - 1; i > 0; i--) {
+            const j = Math.floor(rng() * (i + 1));
+            [order[i], order[j]] = [order[j], order[i]];
+        }
+        let trainCount = Math.round(items.length * trainRatio);
+        trainCount = Math.max(1, Math.min(items.length - 1, trainCount));
+        const trainIdx = new Set(order.slice(0, trainCount));
+        items.forEach((it, i) => (trainIdx.has(i) ? train : evalHeldOut).push(it));
+    }
+    return { train, evalHeldOut };
+}
+/**
+ * Live learning-curve runner.
+ *
+ * Same two-arm design as the mock runner, but every task is solved by a REAL
+ * agent (Claude CLI) and scored against ground truth:
+ *   - "learning": one Atlas carries memory across the stream — accumulated
+ *     experiences/playbooks are injected into the agent's context.
+ *   - "no-learning": a fresh cold Atlas per task — the agent solves each task
+ *     with no injected memory (the stateless baseline).
+ *
+ * The delta answers the real question the mock can only simulate: does
+ * cognitive-core's accumulated memory actually improve task success?
+ *
+ * Single-turn benchmarks (GSM8K) use the ClaudeBenchmarkDelegate. Interactive
+ * benchmarks (ALFWorld) supply their own multi-turn delegate via
+ * `delegateFactory`.
+ */
+export class LiveEvalRunner {
+    config;
+    delegateFactory;
+    /**
+     * Mutation/merge activity accumulated across learning-arm seeds (summed).
+     * Populated after each learning-arm Atlas closes; attached to the learning
+     * ConditionResult so run reports prove whether the mutation path executed.
+     */
+    learningMutationActivity = null;
+    /** Adopted per-(arm, seed) results from a prior aborted run, or null. */
+    resume;
+    constructor(config, 
+    /** Override for interactive benchmarks that need a multi-turn delegate. */
+    delegateFactory) {
+        this.config = config;
+        this.delegateFactory = delegateFactory;
+        // Load the resume checkpoint eagerly (before any fresh checkpoint write) so
+        // adopted rows reflect the aborted run, not this run's in-progress state.
+        this.resume = config.resumeFrom ? loadResumeState(config.resumeFrom) : null;
+    }
+    /** Completed results for (arm, seed) from the resume checkpoint, if any. */
+    completed(arm, seed) {
+        return this.resume?.completedFor(arm, seed);
+    }
+    /**
+     * Resolve the learning/train Atlas baseDir for a seed. With `runWorkDir`, this
+     * is a STABLE, deterministic subdir that survives process death (persistent =
+     * true → the caller must not delete it in `finally`). Without it, an ephemeral
+     * temp dir (persistent = false → deleted in `finally` as before).
+     */
+    learningBaseDir(kind, seed) {
+        if (this.config.runWorkDir) {
+            const dir = join(this.config.runWorkDir, `${kind}-seed-${seed}`);
+            mkdirSync(dir, { recursive: true });
+            return { dir, persistent: true };
+        }
+        return { dir: mkdtempSync(join(tmpdir(), `cc-live-${kind}-${seed}-`)), persistent: false };
+    }
+    async run() {
+        const start = Date.now();
+        const evalConfig = {
+            backend: 'live',
+            taskCount: this.config.taskCount,
+            skillCount: 0,
+            seeds: this.config.seeds,
+            rollingWindow: this.config.rollingWindow,
+            coldProb: 0,
+            warmProb: 0,
+            forceBatchEvery: 0,
+        };
+        // Single-arm (ablation/attribution): learning only, no cold baseline.
+        if (this.config.singleArm) {
+            const learning = await this.runArm('learning', true);
+            this.attachMutationActivity(learning);
+            return {
+                config: evalConfig,
+                benchmarkName: this.config.benchmark.name,
+                singleArm: true,
+                learning,
+                noLearning: learning, // placeholder; not rendered in single-arm mode
+                successRateDelta: 0,
+                normalizedGain: 0,
+                significant: false,
+                ciDisjointAdvisory: false,
+                durationMs: Date.now() - start,
+            };
+        }
+        if (this.config.interleaveArms && this.config.seeds.length <= 1) {
+            this.log('[live] --interleave-arms: single seed — flag is a no-op');
+        }
+        // interleaveArms: per-seed arm-order alternation.
+        // Odd seeds run learning then cold (default); even seeds run cold then learning.
+        // We collect per-seed results into separate arrays and aggregate after all seeds.
+        const learningPerSeed = [];
+        const coldPerSeed = [];
+        for (let seedIdx = 0; seedIdx < this.config.seeds.length; seedIdx++) {
+            const seed = this.config.seeds[seedIdx];
+            const stream = await this.loadStream(seed);
+            // Train/eval isolation path: learner trains on the TRAIN split (oracle
+            // on) then is FROZEN and scored on the disjoint held-out EVAL split
+            // (oracle off). Only the EVAL split is aggregated. The cold baseline runs
+            // over the same held-out EVAL split. This is disjoint from the default
+            // interleaved path — reuse/interleave/stopping do not apply here.
+            if (this.config.trainEvalSplit?.enabled) {
+                const trainRatio = this.config.trainEvalSplit.trainRatio ?? 0.5;
+                const splitSeed = this.config.trainEvalSplit.splitSeed ?? 0;
+                const { train, evalHeldOut } = partitionTrainEval(stream, trainRatio, splitSeed + seed);
+                this.log(`[live] seed=${seed}: train/eval split — ${train.length} train (oracle on) / ` +
+                    `${evalHeldOut.length} held-out eval (oracle off), trainRatio=${trainRatio}`);
+                if (evalHeldOut.length === 0) {
+                    this.log(`[live] seed=${seed}: WARNING — held-out eval split is empty ` +
+                        `(too few tasks per family for trainRatio=${trainRatio}); skipping seed`);
+                    continue;
+                }
+                const learn = await this.runTrainThenEval(train, evalHeldOut, seed);
+                const cold = await this.runCold(evalHeldOut, seed);
+                learningPerSeed.push(learn);
+                coldPerSeed.push(cold);
+                continue;
+            }
+            const reusedCold = this.reusableColdRows(stream, seed);
+            if (reusedCold) {
+                this.log(`[live] seed=${seed}: reusing ${reusedCold.length} cold-arm results from prior run ` +
+                    `(skipping cold arm)`);
+                const learn = await this.runCarry(stream, seed);
+                learningPerSeed.push(learn);
+                coldPerSeed.push(reusedCold);
+            }
+            else {
+                const coldFirst = this.config.interleaveArms && seed % 2 === 0;
+                if (coldFirst) {
+                    this.log(`[live] seed=${seed}: cold arm first (interleave-arms)`);
+                    const cold = await this.runCold(stream, seed);
+                    const learn = await this.runCarry(stream, seed);
+                    coldPerSeed.push(cold);
+                    learningPerSeed.push(learn);
+                }
+                else {
+                    this.log(`[live] seed=${seed}: learning arm first`);
+                    const learn = await this.runCarry(stream, seed);
+                    const cold = await this.runCold(stream, seed);
+                    learningPerSeed.push(learn);
+                    coldPerSeed.push(cold);
+                }
+            }
+            // Sequential stopping at seed boundaries (only when more seeds remain):
+            // pool completed episodes, ask whether the remaining budget still buys
+            // information, and skip the rest of the seeds when it does not.
+            const remainingSeeds = this.config.seeds.length - seedIdx - 1;
+            if (this.config.stopping?.enabled && remainingSeeds > 0) {
+                const verdict = evaluateStopping({
+                    learning: poolArmCounts(learningPerSeed),
+                    cold: poolArmCounts(coldPerSeed),
+                    remainingEpisodesPerArm: remainingSeeds * this.config.taskCount,
+                }, this.config.stopping);
+                this.log(`[live] stopping check after seed ${seed}: ${verdict.decision}` +
+                    `${verdict.rule ? ` (${verdict.rule})` : ''} — ${verdict.reason}`);
+                if (verdict.decision === 'stop') {
+                    this.log(`[live] adaptive stopping: skipping ${remainingSeeds} remaining seed(s) ` +
+                        `(${remainingSeeds * this.config.taskCount * 2} episodes saved)`);
+                    break;
+                }
+            }
+        }
+        // Aggregate over the seeds actually run (adaptive stopping may truncate).
+        const seedsRun = this.config.seeds.slice(0, learningPerSeed.length);
+        const learning = aggregateCondition('learning', learningPerSeed, this.config.rollingWindow, seedsRun);
+        this.attachMutationActivity(learning);
+        const noLearning = aggregateCondition('no-learning', coldPerSeed, this.config.rollingWindow, seedsRun);
+        const advisory = ciDisjoint(learning.successRateCI, noLearning.successRateCI);
+        return {
+            config: evalConfig,
+            benchmarkName: this.config.benchmark.name,
+            learning,
+            noLearning,
+            successRateDelta: learning.successRate - noLearning.successRate,
+            normalizedGain: normalizedGain(noLearning.lateSuccessRate, learning.lateSuccessRate),
+            significant: advisory,
+            ciDisjointAdvisory: advisory,
+            durationMs: Date.now() - start,
+        };
+    }
+    /**
+     * Load the task stream for a seed. With discriminative task selection,
+     * oversample the pool and down-select toward the p≈0.5 band using the
+     * difficulty registry; otherwise load exactly taskCount tasks.
+     */
+    async loadStream(seed) {
+        const selection = this.config.taskSelection;
+        if (!selection) {
+            return this.config.benchmark.loadTasks({ count: this.config.taskCount, seed });
+        }
+        const oversample = selection.oversample ?? 3;
+        const pool = await this.config.benchmark.loadTasks({
+            count: this.config.taskCount * oversample,
+            seed,
+        });
+        const stream = selectDiscriminativeTasks(pool, this.config.taskCount, selection.registry, this.config.benchmark.name, seed, selection.minRuns);
+        this.log(`[live] seed=${seed}: discriminative selection — ${stream.length}/${pool.length} tasks ` +
+            `(families: ${[...new Set(stream.map((t) => t.skillTag))].join(', ')})`);
+        return stream;
+    }
+    async runArm(label, carryMemory) {
+        const perSeed = [];
+        for (const seed of this.config.seeds) {
+            const stream = await this.loadStream(seed);
+            this.log(`[live] arm=${label} seed=${seed}: ${stream.length} tasks`);
+            const results = carryMemory
+                ? await this.runCarry(stream, seed)
+                : await this.runCold(stream, seed);
+            perSeed.push(results);
+        }
+        return aggregateCondition(label, perSeed, this.config.rollingWindow, this.config.seeds);
+    }
+    log(msg) {
+        if (!this.config.quiet)
+            console.error(msg);
+    }
+    /**
+     * Prior cold-arm rows for this seed, or undefined when absent/mismatched.
+     * Mismatch (different task count or taskIds) means the prior run used a
+     * different config — adopting it would compare arms across different task
+     * sets, so we warn and fall back to actually running the cold arm.
+     */
+    reusableColdRows(stream, seed) {
+        const rows = this.config.reuseColdBySeed?.[seed];
+        if (!rows)
+            return undefined;
+        const check = coldRowsMatchStream(rows, stream.map((s) => s.task.id));
+        if (!check.ok) {
+            this.log(`[live] seed=${seed}: NOT reusing cold arm — prior run does not match ` +
+                `this task stream (${check.reason}); running cold arm normally`);
+            return undefined;
+        }
+        return rows;
+    }
+    /** Learning arm: one Atlas + delegate carries memory across the stream.
+     * env_error tasks are retried once; if they still env_error, they are
+     * recorded with status='env_error' and excluded from learning writes. */
+    async runCarry(stream, seed) {
+        // `frozen` disables the dream queue, so the learning arm never
+        // consolidates playbooks mid-run — it silently degenerates into a cold arm
+        // with an experience store. Fine for retrieval-only smokes with seeded
+        // playbooks; a bug for learning-efficacy runs. Warn loudly.
+        if ((this.config.mutationMode ?? 'active') === 'frozen') {
+            this.log(`[live] WARNING: learning arm running with mutationMode=frozen — ` +
+                `dream-queue writes are disabled, so NO playbooks will be extracted ` +
+                `mid-run. For learning-efficacy runs use --mutation-mode active (the default).`);
+        }
+        // Resume: adopt already-completed learning rows for this seed. A
+        // fully-complete seed skips Atlas construction entirely (no reopen cost) —
+        // it just re-emits its rows through the checkpoint hook.
+        const completed = this.completed('learning', seed);
+        if (completed && stream.every((item) => completed.has(item.index))) {
+            this.log(`[live] seed=${seed}: learning arm fully complete in checkpoint ` +
+                `(${stream.length} tasks) — adopting rows, skipping arm`);
+            const results = stream.map((item) => completed.get(item.index));
+            for (const item of stream) {
+                await this.config.onTaskBoundary?.({
+                    arm: 'learning',
+                    seed,
+                    taskIndex: item.index,
+                    taskCount: stream.length,
+                    result: completed.get(item.index),
+                });
+            }
+            return results;
+        }
+        const { dir: baseDir, persistent } = this.learningBaseDir('learn', seed);
+        const delegate = this.makeDelegate();
+        const atlas = this.makeAtlas(baseDir, delegate, true);
+        // Reopening a persisted baseDir restores accumulated playbooks/experiences
+        // (PlaybookLibrary.init reindexes the SQLite rows), so resumed later tasks
+        // see the memory the aborted run had learned.
+        await atlas.init();
+        await this.loadMetaPlaybooks(atlas, true);
+        // Wire learning-arm-only levers (distill / mutation gating). The cold arm
+        // (runCold) must NOT receive a learning provider — it is stateless by design.
+        this.wireLearningArmMutation(atlas);
+        delegate.setMemory?.(atlas.getMemory());
+        if (completed && completed.size > 0) {
+            this.log(`[live] seed=${seed}: resuming learning arm — adopting ${completed.size} ` +
+                `completed task(s), reopened ${baseDir}`);
+        }
+        const results = [];
+        try {
+            for (const item of stream) {
+                // Adopt an already-completed task instead of re-solving it. Memory for
+                // this task is already reflected in the reopened baseDir, so we also
+                // skip its maybeDream.
+                const prior = completed?.get(item.index);
+                const result = prior ??
+                    (await this.solveWithRetry(atlas, delegate, item, 'learning', seed, stream.length));
+                results.push(result);
+                await this.config.onTaskBoundary?.({
+                    arm: 'learning',
+                    seed,
+                    taskIndex: item.index,
+                    taskCount: stream.length,
+                    result,
+                });
+                // env_error: do NOT feed the trajectory to learning (skip maybeDream
+                // so the infra failure does not corrupt the memory state). Adopted rows
+                // are skipped too (their dream already ran in the aborted generation).
+                if (!prior && result.status !== 'env_error') {
+                    // Decoupled modeling: consolidate between tasks (same as the mock
+                    // runner). With extraction off (minTrajectories) this only replays the
+                    // deferred analytics; if minTrajectories is lowered it also extracts
+                    // playbooks mid-stream so later tasks can benefit.
+                    await atlas.getConsolidationCoordinator()?.maybeDream();
+                }
+            }
+        }
+        finally {
+            // close() drains leftover dream rows (which can trigger a final
+            // maintenance cycle), so capture the mutation counters AFTER close.
+            await atlas.close();
+            this.captureMutationActivity(atlas, seed);
+            // Persistent dirs survive for --resume; the caller cleans up on success.
+            if (!persistent)
+                rmSync(baseDir, { recursive: true, force: true });
+        }
+        return results;
+    }
+    /**
+     * Wire learning-arm-only mutation levers onto a freshly-built Atlas:
+     * bind the learning LLM provider when `distill` or `mutationGating` is on
+     * (installs the LLMJudgeReplay gate judge + reflective generator), and —
+     * for `mutationGating` — make the maintenance scheduler fire after every
+     * dream so short eval streams reach the mutation tasks at all.
+     */
+    wireLearningArmMutation(atlas) {
+        if (this.config.distill || this.config.mutationGating) {
+            atlas.setLearningProvider(this.config.learningLlmProvider ??
+                createClaudeCliLLMProvider({
+                    model: this.config.model,
+                    provider: this.config.provider,
+                    awsRegion: this.config.awsRegion,
+                    timeoutMs: this.config.timeoutMs,
+                }));
+        }
+        if (this.config.mutationGating) {
+            // batchInterval 1 is the production default too, but pin it here so
+            // eval mutation runs stay reachable even if the default drifts.
+            atlas.getLearning().getMaintenanceScheduler().updateConfig({ batchInterval: 1 });
+            this.wireGateJudge(atlas);
+        }
+    }
+    /**
+     * FIX 1 — pick the mutation-gate judge and log the choice explicitly.
+     *
+     * With `--live-replay-gating`, install a {@link ConditionalReplayJudge}
+     * (via `atlas.enableLiveReplayGating`) so faithfully-replayable experiences
+     * are gated on the REAL verifier (re-execution outcome — the same
+     * ground-truth signal the harness computes) and only non-replayable ones
+     * fall back to the LLM judge. Otherwise the LLM judge stays — but the
+     * decision is always logged so a run's gating signal is never ambiguous.
+     *
+     * FAITHFUL REPLAY (see docs/DESIGN-meta-learning-validation.md §8): the gate
+     * is only honest if the rollout re-runs the CORRECT task. We route by the
+     * ReplayDescriptor the solve path stamps onto each experience — an experience
+     * is replayable ONLY when it carries a stable `taskId` — and reconstruct that
+     * task (with its `taskId`/`family`/`difficulty` metadata) so the stateful
+     * benchmark delegate drives the right task. Experiences without a descriptor
+     * (no taskId) honestly fall back to the LLM judge instead of silently
+     * re-running the last-solved task. The rollout itself is isolated from the
+     * solve path's cost/current-task state (see `AgentSpawnConfig.replay`).
+     */
+    wireGateJudge(atlas) {
+        if (!this.config.liveReplayGating) {
+            this.log(`[live] mutation gating: LLM-judge gate (haiku proxy). Pass ` +
+                `--live-replay-gating to gate on the real task verifier instead.`);
+            return;
+        }
+        // Honest replayability: only experiences carrying a stamped, reconstructable
+        // taskId descriptor are re-run for real; everything else falls back to the
+        // LLM judge (never a rollout of the wrong task). reconstruct() returns an
+        // empty env — the delegate owns the sandbox; the taskId reaches it via
+        // reconstructTask + spawnViaDelegate's setCurrentTask.
+        const provider = {
+            isReplayable: isTaskIdReplayable,
+            reconstruct: async () => ({}),
+        };
+        const budget = this.config.liveReplayMaxRolloutsPerWindow ?? DEFAULT_LIVE_REPLAY_ROLLOUT_BUDGET;
+        try {
+            atlas.enableLiveReplayGating({
+                timeoutMs: this.config.timeoutMs,
+                maxRolloutsPerWindow: budget,
+                environmentProvider: provider,
+                // Re-stamp the stable task handle onto the reconstructed task so the
+                // stateful delegate re-runs the EXPERIENCE's task, not the last solved.
+                reconstructTask: reconstructTaskFromReplayDescriptor,
+            });
+            this.log(`[live] mutation gating: LIVE replay gate enabled — experiences with a ` +
+                `stamped taskId scored by real re-execution + verifier (budget ` +
+                `${budget}/window), LLM judge fallback for the rest.`);
+        }
+        catch (e) {
+            // enableLiveReplayGating throws only when no AgentManager is wired.
+            // Keep the LLM judge (already installed by setLearningProvider) and
+            // make the degraded choice explicit.
+            this.log(`[live] mutation gating: could not enable live replay gating ` +
+                `(${e instanceof Error ? e.message : String(e)}); keeping LLM-judge gate.`);
+        }
+    }
+    /**
+     * Read the learning-arm pipeline's accumulated mutation counters and fold
+     * them into the run-level accumulator (summed across seeds). Reads
+     * in-memory state only, so it is safe after `atlas.close()`.
+     */
+    captureMutationActivity(atlas, seed) {
+        let activity;
+        try {
+            activity = atlas.getLearning().getMutationActivity();
+        }
+        catch {
+            return;
+        }
+        const acc = (this.learningMutationActivity ??= {
+            maintenanceCyclesRun: 0,
+            reflective: {
+                runs: 0,
+                skipped: 0,
+                proposalsConsidered: 0,
+                proposalsAccepted: 0,
+                proposalsRejected: 0,
+                reasonCounts: {},
+            },
+            merge: {
+                runs: 0,
+                skipped: 0,
+                proposalsConsidered: 0,
+                proposalsAccepted: 0,
+                proposalsRejected: 0,
+                reasonCounts: {},
+            },
+        });
+        acc.maintenanceCyclesRun += activity.maintenanceCyclesRun;
+        for (const key of ['reflective', 'merge']) {
+            const from = activity[key];
+            const to = acc[key];
+            to.runs += from.runs;
+            to.skipped += from.skipped;
+            if (from.lastSkipReason)
+                to.lastSkipReason = from.lastSkipReason;
+            to.proposalsConsidered += from.proposalsConsidered;
+            to.proposalsAccepted += from.proposalsAccepted;
+            to.proposalsRejected += from.proposalsRejected;
+            for (const [reason, count] of Object.entries(from.reasonCounts)) {
+                if (typeof count !== 'number')
+                    continue;
+                const rk = reason;
+                to.reasonCounts[rk] = (to.reasonCounts[rk] ?? 0) + count;
+            }
+        }
+        if (this.config.mutationGating) {
+            this.log(`[live] seed=${seed} mutation activity: maintenanceCycles=${activity.maintenanceCyclesRun} ` +
+                `reflective(considered=${activity.reflective.proposalsConsidered} ` +
+                `accepted=${activity.reflective.proposalsAccepted} skipped=${activity.reflective.skipped}` +
+                `${activity.reflective.lastSkipReason ? ` lastSkip="${activity.reflective.lastSkipReason}"` : ''}) ` +
+                `merge(considered=${activity.merge.proposalsConsidered} ` +
+                `accepted=${activity.merge.proposalsAccepted} skipped=${activity.merge.skipped}` +
+                `${activity.merge.lastSkipReason ? ` lastSkip="${activity.merge.lastSkipReason}"` : ''})`);
+        }
+    }
+    /** Attach the accumulated learning-arm mutation counters to the report row. */
+    attachMutationActivity(learning) {
+        if (this.learningMutationActivity) {
+            learning.mutationActivity = this.learningMutationActivity;
+        }
+    }
+    /**
+     * Train/eval isolation arm: one Atlas trains on the TRAIN split (oracle on,
+     * playbooks extracted mid-stream), is flushed + FROZEN, then scored on the
+     * disjoint held-out EVAL split with the trained memory served read-only.
+     * Returns ONLY the held-out eval results (the measured set).
+     *
+     * The freeze (`setValidationMutationMode('frozen')`) is what makes the
+     * isolation complete: during eval, `processTrajectory` is a no-op, so eval
+     * trajectories are neither stored nor extracted — the learner never trains on
+     * a task it is measured on. Retrieval/serving stay live, so trained playbooks
+     * are still injected. See docs/DESIGN-meta-learning-validation.md §
+     * "Train-time oracle signal vs solve-time firewall".
+     */
+    async runTrainThenEval(train, evalHeldOut, seed) {
+        if ((this.config.mutationMode ?? 'active') === 'frozen') {
+            this.log(`[live] WARNING: train/eval split with mutationMode=frozen — the TRAIN ` +
+                `phase will not extract playbooks; the eval measures a cold baseline.`);
+        }
+        const { dir: baseDir, persistent } = this.learningBaseDir('train', seed);
+        const delegate = this.makeDelegate();
+        const atlas = this.makeAtlas(baseDir, delegate, true);
+        await atlas.init();
+        await this.loadMetaPlaybooks(atlas, true);
+        this.wireLearningArmMutation(atlas);
+        delegate.setMemory?.(atlas.getMemory());
+        const completedTrain = this.completed('train', seed);
+        const completedEval = this.completed('learning', seed);
+        const evalResults = [];
+        try {
+            // TRAIN phase — oracle on; the learner extracts playbooks mid-stream.
+            for (const item of train) {
+                const priorTrain = completedTrain?.get(item.index);
+                const result = priorTrain ??
+                    (await this.solveWithRetry(atlas, delegate, item, 'train', seed, train.length));
+                await this.config.onTaskBoundary?.({
+                    arm: 'train',
+                    seed,
+                    taskIndex: item.index,
+                    taskCount: train.length,
+                    result,
+                });
+                if (!priorTrain && result.status !== 'env_error') {
+                    await atlas.getConsolidationCoordinator()?.maybeDream();
+                }
+            }
+            // Flush the dream queue so ALL train trajectories are consolidated into
+            // playbooks BEFORE the eval phase reads memory.
+            await atlas.dream({ drainAll: true });
+            // FREEZE — eval trajectories must not train the learner (firewall).
+            atlas.setValidationMutationMode('frozen');
+            this.log(`[live] seed=${seed}: train phase complete (${train.length} tasks) — ` +
+                `frozen for held-out eval (${evalHeldOut.length} tasks)`);
+            // EVAL phase — scored against trained (read-only) memory; no learning.
+            for (const item of evalHeldOut) {
+                const priorEval = completedEval?.get(item.index);
+                const result = priorEval ??
+                    (await this.solveWithRetry(atlas, delegate, item, 'learning', seed, evalHeldOut.length));
+                evalResults.push(result);
+                await this.config.onTaskBoundary?.({
+                    arm: 'learning',
+                    seed,
+                    taskIndex: item.index,
+                    taskCount: evalHeldOut.length,
+                    result,
+                });
+            }
+        }
+        finally {
+            await atlas.close();
+            this.captureMutationActivity(atlas, seed);
+            // Persistent dirs survive for --resume; the caller cleans up on success.
+            if (!persistent)
+                rmSync(baseDir, { recursive: true, force: true });
+        }
+        return evalResults;
+    }
+    /** No-learning arm: a fresh cold Atlas + delegate per task.
+     * env_error tasks are retried once; on second failure recorded as env_error. */
+    async runCold(stream, seed) {
+        // Cold tasks are fully independent (fresh Atlas + delegate + bridge each),
+        // so they parallelize cleanly. `concurrency` caps in-flight episodes; each
+        // builds and tears down its own resources. Results stay in stream order.
+        // Clamp to the benchmark's max (ALFWorld's bridge isn't concurrency-safe).
+        const benchMax = this.config.benchmark.maxConcurrency ?? Infinity;
+        const limit = Math.max(1, Math.min(this.config.concurrency ?? 1, benchMax));
+        const completed = this.completed('no-learning', seed);
+        return mapPool(stream, limit, async (item) => {
+            // Resume: adopt an already-completed cold episode (paid for in the aborted
+            // run) instead of re-running it. Cold tasks are stateless, so no reopen.
+            const prior = completed?.get(item.index);
+            if (prior) {
+                await this.config.onTaskBoundary?.({
+                    arm: 'no-learning',
+                    seed,
+                    taskIndex: item.index,
+                    taskCount: stream.length,
+                    result: prior,
+                });
+                return prior;
+            }
+            const baseDir = mkdtempSync(join(tmpdir(), `cc-live-cold-${seed}-${item.index}-`));
+            const delegate = this.makeDelegate();
+            const atlas = this.makeAtlas(baseDir, delegate);
+            await atlas.init();
+            delegate.setMemory?.(atlas.getMemory());
+            try {
+                const result = await this.solveWithRetry(atlas, delegate, item, 'no-learning', seed, stream.length);
+                await this.config.onTaskBoundary?.({
+                    arm: 'no-learning',
+                    seed,
+                    taskIndex: item.index,
+                    taskCount: stream.length,
+                    result,
+                });
+                return result;
+            }
+            finally {
+                await atlas.close();
+                rmSync(baseDir, { recursive: true, force: true });
+            }
+        });
+    }
+    /**
+     * Solve a single task with one automatic retry on env_error.
+     * On second env_error, return a result with status='env_error'.
+     * env_error results are excluded from success-rate aggregates and learning.
+     */
+    async solveWithRetry(atlas, delegate, item, arm, seed, taskCount) {
+        try {
+            return await this.solveOne(atlas, delegate, item, arm, taskCount);
+        }
+        catch (err) {
+            if (isEnvError(err)) {
+                if (this.config.failFast) {
+                    this.log(`[live]   task ${item.index + 1}/${taskCount} env_error (arm=${arm} seed=${seed}): ` +
+                        `${err.message.slice(0, 200)} — fail-fast, recording as env_error`);
+                    return {
+                        index: item.index,
+                        taskId: item.task.id,
+                        skillTag: item.skillTag,
+                        success: false,
+                        status: 'env_error',
+                        stepCount: 0,
+                        knowledgeSurfaced: false,
+                        priorSkillCoverage: 0,
+                        wallTimeMs: 0,
+                    };
+                }
+                this.log(`[live]   task ${item.index + 1}/${taskCount} env_error (arm=${arm} seed=${seed}): ` +
+                    `${err.message.slice(0, 200)} — retrying once`);
+                try {
+                    return await this.solveOne(atlas, delegate, item, arm, taskCount);
+                }
+                catch (err2) {
+                    this.log(`[live]   task ${item.index + 1}/${taskCount} env_error again (arm=${arm} seed=${seed}): ` +
+                        `${err2.message.slice(0, 200)} — recording as env_error`);
+                    return {
+                        index: item.index,
+                        taskId: item.task.id,
+                        skillTag: item.skillTag,
+                        success: false,
+                        status: 'env_error',
+                        stepCount: 0,
+                        knowledgeSurfaced: false,
+                        priorSkillCoverage: 0,
+                        wallTimeMs: 0,
+                    };
+                }
+            }
+            throw err;
+        }
+    }
+    async solveOne(atlas, delegate, item, arm, taskCount) {
+        delegate.setCurrentTask?.(item.task);
+        const costBefore = delegate.getTotalCostUsd?.() ?? 0;
+        const t0 = Date.now();
+        // --- Diagnostics: attach collector to AgentManager before the solve ---
+        const agentManager = atlas.getAgentManager();
+        let spawnDiag;
+        if (agentManager && this.config.traceDir) {
+            agentManager.setDiagnosticsCollector((d) => {
+                spawnDiag = d;
+            });
+        }
+        const result = await atlas.solve(item.task);
+        // Detach collector immediately after solve.
+        if (agentManager && this.config.traceDir) {
+            agentManager.setDiagnosticsCollector(undefined);
+        }
+        const wallTimeMs = Date.now() - t0;
+        const costAfter = delegate.getTotalCostUsd?.() ?? 0;
+        const costUsd = costAfter - costBefore;
+        const success = result.trajectory.outcome.success;
+        this.log(`[live]   task ${item.index + 1}/${taskCount} ` +
+            `${success ? 'PASS' : 'fail'} ` +
+            `(${(wallTimeMs / 1000).toFixed(1)}s` +
+            `${costUsd > 0 ? `, $${costUsd.toFixed(4)}` : ''})`);
+        const ctx = result.routing?.memoryContext;
+        const knowledgeSurfaced = !!ctx && ctx.experiences.length + ctx.playbooks.length > 0;
+        const solution = result.trajectory.outcome.solution;
+        const partialScore = typeof solution?.partialScore === 'number' ? solution.partialScore : undefined;
+        // Larger budget so a multi-step agent transcript (incl. failures) is captured
+        // in the trace for behaviour/adoption debugging, not just the first ~2 steps.
+        const outputPreview = previewOutcome(result.trajectory.outcome, 12000);
+        // --- Diagnostics: influence sampling (learning arm only) ---
+        let influenceDiag;
+        const sampleEvery = this.config.influenceSampleEvery ?? 0;
+        if (arm === 'learning' && sampleEvery > 0 && agentManager && item.index % sampleEvery === 0) {
+            try {
+                const cmp = await agentManager.compareWithKnowledge({
+                    agentType: 'delegate',
+                    task: item.task,
+                });
+                influenceDiag = {
+                    sampled: true,
+                    changedOutput: cmp.withKnowledge.session.result !== cmp.baseline.session.result,
+                };
+            }
+            catch {
+                influenceDiag = { sampled: true, changedOutput: false };
+            }
+        }
+        // --- Diagnostics: emit trace entry if traceDir is configured ---
+        if (this.config.traceDir && spawnDiag !== undefined) {
+            const threshold = spawnDiag.knowledge.retrievalDiagnostics?.playbookMinScore ?? 0;
+            const diagnosticStrip = new Set(this.config.diagnosticStripChannels ?? []);
+            let deliveredKnowledgeBlob = spawnDiag.formattedKnowledge;
+            for (const ch of diagnosticStrip) {
+                deliveredKnowledgeBlob = stripMarkdownSection(deliveredKnowledgeBlob, CHANNEL_SECTION_HEADERS[ch]);
+            }
+            const deliveredPlaybooks = diagnosticStrip.has('playbooks')
+                ? []
+                : spawnDiag.knowledge.playbooks.slice(0, spawnDiag.playbookCount).map((p) => ({
+                    id: p.playbook.id,
+                    name: p.playbook.name,
+                    score: p.score,
+                    retrievalMethod: p.retrievalMethod,
+                    extractionMethod: p.playbook.provenance?.extractionMethod,
+                    sourceFile: p.playbook.provenance?.sourceFile,
+                    skillLevel: p.playbook.skillLevel ?? inferPlaybookSkillLevel(p.playbook),
+                    applicabilityDecision: p.applicabilityDecision?.decision,
+                    applicabilityReason: p.applicabilityDecision?.reason,
+                }));
+            const candidates = [
+                ...spawnDiag.knowledge.experiences.map((e) => ({
+                    id: e.experience.id,
+                    type: 'experience',
+                    score: e.score,
+                })),
+                ...spawnDiag.knowledge.playbooks.map((p) => ({
+                    id: p.playbook.id,
+                    type: 'playbook',
+                    score: p.score,
+                    retrievalMethod: p.retrievalMethod,
+                    applicabilityDecision: p.applicabilityDecision?.decision,
+                    applicabilityReason: p.applicabilityDecision?.reason,
+                })),
+            ];
+            const hitAtThreshold = candidates.some((c) => c.score >= threshold);
+            // Phase-0: map retrieval-time candidate decisions into full audit rows,
+            // stamping task identity and whether the candidate was ultimately
+            // injected. The injected set is the actually-served playbooks (strip-
+            // independent), so a candidate dropped by the gate shows injected=false
+            // with its rejection reason intact.
+            const injectedIds = new Set(spawnDiag.knowledge.playbooks.slice(0, spawnDiag.playbookCount).map((p) => p.playbook.id));
+            const candidateDecisions = spawnDiag.knowledge.retrievalDiagnostics?.candidateDecisions?.map((c) => {
+                const injected = injectedIds.has(c.playbookId);
+                return {
+                    taskId: item.task.id,
+                    family: item.family,
+                    query: item.task.description,
+                    candidateRank: c.candidateRank,
+                    playbookId: c.playbookId,
+                    playbookName: c.playbookName,
+                    skillLevel: c.skillLevel,
+                    retrievalMethod: c.retrievalMethod,
+                    score: c.score,
+                    domainFallbackUsed: c.domainFallbackUsed,
+                    scoreFloorPassed: c.scoreFloorPassed,
+                    rawDomains: c.rawDomains,
+                    normalizedDomains: c.normalizedDomains,
+                    taskDomains: c.taskDomains,
+                    domainAliasOverlap: c.domainAliasOverlap,
+                    cueHits: c.cueHits,
+                    antiPatternHits: c.antiPatternHits,
+                    decision: c.decision,
+                    reason: c.reason,
+                    gateEnforced: c.gateEnforced,
+                    injected,
+                    incorporationMode: injected
+                        ? c.decision === 'refine'
+                            ? 'refined-prose'
+                            : 'prose'
+                        : undefined,
+                };
+            });
+            const diag = {
+                taskIndex: item.index,
+                arm,
+                taskId: item.task.id,
+                family: item.family,
+                difficulty: item.difficulty,
+                skillTag: item.skillTag,
+                extraction: result.extractionYield ?? { itemsStored: 0, byType: {} },
+                retrieval: {
+                    query: item.task.description,
+                    candidates,
+                    candidateDecisions,
+                    hitAtThreshold,
+                    threshold,
+                    playbookRetriever: spawnDiag.knowledge.retrievalDiagnostics?.playbookRetriever,
+                    playbookRawCount: spawnDiag.knowledge.retrievalDiagnostics?.playbookRawCount,
+                    playbookExcludedByScoreFloor: spawnDiag.knowledge.retrievalDiagnostics?.playbookExcludedByScoreFloor,
+                    playbookExcludedByApplicability: spawnDiag.knowledge.retrievalDiagnostics?.playbookExcludedByApplicability,
+                    playbookDomainFallbackUsed: spawnDiag.knowledge.retrievalDiagnostics?.playbookDomainFallbackUsed,
+                    playbookApplicabilityGateEnabled: spawnDiag.knowledge.retrievalDiagnostics?.playbookApplicabilityGateEnabled,
+                },
+                injection: {
+                    blob: deliveredKnowledgeBlob,
+                    playbookCount: diagnosticStrip.has('playbooks') ? 0 : spawnDiag.playbookCount,
+                    experienceCount: diagnosticStrip.has('experiences') ? 0 : spawnDiag.experienceCount,
+                    playbooks: deliveredPlaybooks,
+                },
+                influence: influenceDiag,
+                outcome: {
+                    success,
+                    status: success ? 'success' : 'failure',
+                    stepCount: result.trajectory.steps.length,
+                    wallTimeMs,
+                    costUsd: costUsd > 0 ? costUsd : undefined,
+                    outputPreview,
+                },
+            };
+            const trace = createTraceWriter(this.config.traceDir, item.task.id);
+            trace({ type: 'diagnostics', ...diag });
+            const collectMechanismTrace = emitMechanismTraceAsLiveTraceEntry(trace);
+            for (const event of diagnosticsToMechanismTraceEvents(diag, {
+                benchmark: this.config.benchmark.name,
+                source: 'LiveEvalRunner.runTask',
+            })) {
+                collectMechanismTrace(event);
+            }
+        }
+        return {
+            index: item.index,
+            taskId: item.task.id,
+            skillTag: item.skillTag,
+            success,
+            status: success ? 'success' : 'failure',
+            stepCount: result.trajectory.steps.length,
+            knowledgeSurfaced,
+            priorSkillCoverage: 0,
+            wallTimeMs,
+            costUsd: costUsd > 0 ? costUsd : undefined,
+            partialScore,
+        };
+    }
+    makeDelegate() {
+        const opts = {
+            model: this.config.model,
+            timeoutMs: this.config.timeoutMs,
+            provider: this.config.provider,
+            awsRegion: this.config.awsRegion,
+            refine: this.config.refine,
+            // failFast: one attempt per model call — a hung call costs one timeout,
+            // not the 2-3x compounding that turns doomed episodes into hours.
+            ...(this.config.failFast ? { cliRetries: 0 } : {}),
+        };
+        if (this.delegateFactory)
+            return this.delegateFactory(this.config.benchmark, opts);
+        return new ClaudeBenchmarkDelegate(this.config.benchmark, opts);
+    }
+    makeAtlas(baseDir, delegate, isLearningArm = false) {
+        if (this.config.proceduralMemoryDir) {
+            const targetDir = join(baseDir, 'playbooks');
+            if (existsSync(this.config.proceduralMemoryDir)) {
+                rmSync(targetDir, { recursive: true, force: true });
+                cpSync(this.config.proceduralMemoryDir, targetDir, { recursive: true });
+            }
+        }
+        // Skill-learning benchmarks opt into playbook extraction via
+        // `playbookMinTrajectories`; others leave it off (extraction disabled). When
+        // on, the mid-run dream extracts playbooks at a cadence clamped to the
+        // stream length so even short benchmarks consolidate before the run ends.
+        //
+        // When `distill` or `analysisBackends` is enabled on the learning arm and
+        // the benchmark has no explicit playbookMinTrajectories, lower the threshold
+        // to distillMinTrajectories (default 3) so the mid-run dream fires and the
+        // non-heuristic extractor actually runs before the run ends.
+        const pbMin = this.config.benchmark.playbookMinTrajectories ??
+            (isLearningArm &&
+                (this.config.distill ||
+                    this.config.mutationGating ||
+                    (this.config.analysisBackends?.length ?? 0) > 0)
+                ? (this.config.distillMinTrajectories ?? 3)
+                : undefined);
+        const learnsPlaybooks = typeof pbMin === 'number' && pbMin > 0;
+        const minTrajectories = learnsPlaybooks ? pbMin : 1_000_000;
+        const dreamCadence = learnsPlaybooks
+            ? Math.max(1, Math.min(pbMin, this.config.taskCount))
+            : undefined;
+        const config = {
+            storage: { baseDir, dbName: 'eval.db' },
+            // No bundled seed pack / skill-tree publishing / agent-refinement confound.
+            // B1: seedPlaybookDirs seeds BOTH arms (recipe cold-start, lifts absolute
+            // performance equally). learningOnlySeedDirs seeds ONLY the learning arm
+            // (oracle smoke) — seeding the cold arm there would hand it the answer.
+            curatedPlaybooks: {
+                // bundledSeeds: the Phase-1 seed-lift arm — committed seed channels ON
+                // in both arms (see LiveRunConfig.bundledSeeds for the A/B protocol).
+                loadBundledSeedPack: this.config.bundledSeeds ?? false,
+                additionalDirs: resolveSeedDirs(this.config.seedPlaybookDirs, this.config.learningOnlySeedDirs, isLearningArm),
+            },
+            skillTree: { enabled: false },
+            // Disable within-task refinement entirely: the delegate's execute() returns
+            // ground-truth pass/fail from the test oracle, so Atlas retrying on failure
+            // leaks the oracle signal into the agent prompt — an unfair advantage that
+            // inflates scores. maxIterations:0 prevents any retry spawns.
+            refinement: { useAgentEvaluation: false, maxIterations: 0 },
+            // Playbook extraction: ON for skill-learning benchmarks (small
+            // minTrajectories), OFF otherwise (1_000_000). Either way the immediate
+            // loop stores experiences + bumps playbook confidence on the hot path.
+            learning: { minTrajectories },
+            // Model the shipped system: decoupled dreaming. When extraction is on, the
+            // mid-run maybeDream() (in runCarry) consolidates at `dreamCadence` so
+            // later tasks reuse playbooks learned earlier in the stream. With
+            // analysisBackends configured, playbooks use the workspace abstractor;
+            // otherwise evals keep the legacy no-nested-agent heuristic path.
+            dream: {
+                ...(dreamCadence !== undefined
+                    ? { countThreshold: dreamCadence, maxStalenessMs: 1_000_000_000 }
+                    : {}),
+            },
+            // Success-biased exemplars: only inject trajectories that worked. Raw
+            // failed trajectories injected as exemplars anchor the agent to bad
+            // approaches (the ALFWorld poisoning we diagnosed). Failures are still
+            // stored and learned from via reflexion / anti-patterns / causal edges.
+            memory: {
+                // Seed-free by default: warmth must come from learned memory or the
+                // explicit seed levers (seedPlaybookDirs / metaPlaybookDirs / the
+                // bundledSeeds seed-lift arm), never implicitly.
+                seedMetaPlaybooks: this.config.bundledSeeds ?? false,
+                seedMetaStrategies: this.config.bundledSeeds ?? false,
+                injectFailedExperiences: false,
+                playbookRetrieval: {
+                    // skill-tree-hybrid by default: the legacy lexical retriever's
+                    // textSimilarity between a long task description and short
+                    // trigger/situation strings tops out ~0.02-0.10, so the 0.1 floor
+                    // gated out virtually every mid-run learned playbook (observed in
+                    // the 2026-07-01 multi-seed traces; a replay of the same playbooks/
+                    // query scored 0.58/0.38 under hybrid vs 0.096 max under legacy).
+                    // The roadmap experiments already standardize on hybrid.
+                    mode: this.config.playbookRetriever ?? 'skill-tree-hybrid',
+                    minScore: this.config.minPlaybookScore ?? 0.1,
+                    fusion: 'rrf',
+                    incorporation: this.config.playbookIncorporation ?? 'full',
+                },
+                playbookApplicability: {
+                    enabled: this.config.playbookApplicabilityGate ?? false,
+                },
+                ...(this.config.proceduralMemorySource || this.config.proceduralMemoryDir
+                    ? {
+                        proceduralMemory: {
+                            source: this.config.proceduralMemorySource ?? 'filesystem',
+                            dir: 'playbooks',
+                            ...(this.config.proceduralMemoryStrictStartup !== undefined
+                                ? { strictStartup: this.config.proceduralMemoryStrictStartup }
+                                : {}),
+                            ...(this.config.proceduralMemoryPruneSqliteOnly !== undefined
+                                ? { pruneSqliteOnly: this.config.proceduralMemoryPruneSqliteOnly }
+                                : {}),
+                        },
+                    }
+                    : {}),
+            },
+            validation: {
+                enabled: true,
+                mutationMode: this.config.mutationMode ?? 'active',
+                // Without this, applyValidationControls would silently strip the
+                // seed channels the bundledSeeds arm is trying to measure.
+                allowSeeds: this.config.bundledSeeds ?? false,
+            },
+            // ADDITIVE LEARNING A/B lever (config default ON). When explicitly set
+            // (true OR false) we write the resolved boolean so `--no-additive`
+            // genuinely disables it, overriding the config default; when undefined
+            // (library caller) we leave features unset so the config default applies.
+            // FeaturesConfigSchema fills every other feature with its default.
+            ...(this.config.additive !== undefined
+                ? { features: { additiveConsolidation: this.config.additive } }
+                : {}),
+        };
+        if (this.config.configPatch) {
+            // Deep merge so nested keys (e.g. memory.injectFailedExperiences) are
+            // preserved when the patch only specifies siblings.
+            const merged = deepMerge(config, this.config.configPatch);
+            Object.keys(merged).forEach((k) => {
+                config[k] = merged[k];
+            });
+        }
+        // No registered backends — the delegate handles all execution.
+        const atlas = createAtlasWithAgents([], config);
+        atlas.setDelegate(delegate);
+        if (this.config.traceDir) {
+            const learningTrace = createTraceWriter(this.config.traceDir, 'learning');
+            atlas.setLearningDiagnosticsCollector((event) => {
+                learningTrace({ type: 'learning-diagnostics', ...event });
+            });
+            // FIX 3: make the GEPA-style mutation loop observable — write one
+            // `mutation` mechanism-trace event per considered proposal so a fired
+            // (or suppressed) mutation is recoverable after the run.
+            const mutationTrace = createTraceWriter(this.config.traceDir, 'mutations');
+            const emitMutation = emitMechanismTraceAsLiveTraceEntry(mutationTrace);
+            atlas.getLearning().setMechanismTraceCollector((event) => {
+                if (event.family === 'mutation')
+                    emitMutation(event);
+            });
+        }
+        if (this.config.analysisBackends && this.config.analysisBackends.length > 0) {
+            atlas.setAnalysisAgentManager(this.config.analysisBackends);
+        }
+        // Semantic retrieval: resolve provider type.
+        // embedProvider takes precedence; legacy `embed: true` defaults to 'local'.
+        // 'local' = LocalEmbeddingProvider (all-MiniLM-L6-v2, 384-dim, no API key).
+        // 'hash'  = HashEmbeddingProvider  (zero-dep lexical baseline, 256-dim).
+        // 'none'  = no embedding; text-similarity scoring (default).
+        //
+        // Hot-path note: LocalEmbeddingProvider.embed() is async and blocks on the
+        // first model load (~seconds). We set embedOnStore=false so experiences are
+        // NOT embedded during InstantLoop.process (the <200ms hot path). Embeddings
+        // are computed lazily on first retrieval in findSimilar / findMatching.
+        // Playbooks are embedded at add()-time in the dream phase (off hot path).
+        const providerType = this.config.embedProvider ?? (this.config.embed ? 'local' : 'none');
+        if (providerType !== 'none') {
+            const provider = createEmbeddingProvider(embeddingConfigFromEnv(providerType));
+            if (provider) {
+                // Pass embedOnStore=false for local to avoid blocking the InstantLoop
+                // hot path; hash is sync so it's fine either way but we keep consistent.
+                atlas.getMemory().setEmbeddingProvider(provider, { embedOnStore: false });
+            }
+        }
+        // Each solve must be a SINGLE benchmark agent call. If a separate analysis
+        // manager exists, preserve it for dream-phase extraction and only disable
+        // solve-time usage/meta analysis; otherwise clear all workspace analysis.
+        if (this.config.analysisBackends && this.config.analysisBackends.length > 0) {
+            atlas.disableAgenticHotPathAnalysis();
+        }
+        else {
+            atlas.disableAgenticAnalysis();
+        }
+        return atlas;
+    }
+    async loadMetaPlaybooks(atlas, isLearningArm) {
+        if (!isLearningArm || !this.config.metaPlaybookDirs?.length)
+            return;
+        const memory = atlas.getMemory();
+        let loaded = 0;
+        for (const inputPath of this.config.metaPlaybookDirs) {
+            for (const file of findPlaybookJsonFiles(inputPath)) {
+                const raw = JSON.parse(readFileSync(file, 'utf8'));
+                await memory.metaPlaybooks.add(revivePlaybook(raw));
+                loaded++;
+            }
+        }
+        if (loaded > 0) {
+            this.log(`[live] seeded ${loaded} meta-playbook(s) into learning-analysis bank`);
+        }
+    }
+}
+export function findPlaybookJsonFiles(inputPath) {
+    if (!existsSync(inputPath)) {
+        throw new Error(`Meta-playbook path does not exist: ${inputPath}`);
+    }
+    const stat = statSync(inputPath);
+    if (stat.isFile()) {
+        return inputPath.endsWith('.json') ? [inputPath] : [];
+    }
+    if (!stat.isDirectory())
+        return [];
+    const files = [];
+    for (const entry of readdirSync(inputPath).sort()) {
+        const child = join(inputPath, entry);
+        const childStat = statSync(child);
+        if (childStat.isDirectory()) {
+            files.push(...findPlaybookJsonFiles(child));
+        }
+        else if (childStat.isFile() && child.endsWith('.json')) {
+            files.push(child);
+        }
+    }
+    return files;
+}
+export function revivePlaybook(raw) {
+    if (!raw.name || !raw.applicability || !raw.guidance) {
+        throw new Error('Meta-playbook JSON must include name, applicability, and guidance');
+    }
+    const provenance = raw.provenance
+        ? {
+            ...raw.provenance,
+            recordedAt: new Date(raw.provenance.recordedAt),
+        }
+        : undefined;
+    const evolution = raw.evolution
+        ? {
+            ...raw.evolution,
+            lastUsed: raw.evolution.lastUsed ? new Date(raw.evolution.lastUsed) : undefined,
+            failures: raw.evolution.failures.map((failure) => ({
+                ...failure,
+                timestamp: new Date(failure.timestamp),
+            })),
+            refinements: raw.evolution.refinements.map((refinement) => ({
+                ...refinement,
+                addedAt: new Date(refinement.addedAt),
+            })),
+        }
+        : undefined;
+    return createPlaybook({
+        ...raw,
+        name: raw.name,
+        applicability: raw.applicability,
+        guidance: raw.guidance,
+        provenance,
+        evolution,
+        createdAt: raw.createdAt ? new Date(raw.createdAt) : undefined,
+        updatedAt: raw.updatedAt ? new Date(raw.updatedAt) : undefined,
+    });
+}
+export function createLiveEvalRunner(config, delegateFactory) {
+    return new LiveEvalRunner(config, delegateFactory);
+}
+/**
+ * Section headers emitted by DefaultKnowledgeInjector.formatMarkdown for each
+ * ablatable channel (see src/runtime/manager.ts). Kept in sync by the
+ * channel-stripping unit test, which formats real knowledge and asserts the
+ * stripped systemContext no longer contains them.
+ */
+const CHANNEL_SECTION_HEADERS = {
+    experiences: '## Similar Past Experiences',
+    playbooks: '## Candidate Playbooks',
+};
+/**
+ * Remove a `## <header>` markdown section (header through the next `## ` or
+ * end of string) from formatted knowledge text.
+ */
+function stripMarkdownSection(text, header) {
+    const start = text.indexOf(header);
+    if (start === -1)
+        return text;
+    const next = text.indexOf('\n## ', start + header.length);
+    const removed = next === -1 ? text.slice(0, start) : text.slice(0, start) + text.slice(next + 1);
+    return removed;
+}
+/**
+ * Delegate wrapper that strips chosen knowledge channels before forwarding to
+ * the inner delegate. Mirrors AblatingBackend for the live path.
+ *
+ * IMPORTANT: live delegates consume the pre-formatted `systemContext` STRING
+ * (built by AgentManager.spawnViaDelegate before the delegate is called), not
+ * the structured `injectedKnowledge` object — so both must be stripped.
+ * Stripping only the object would be a silent no-op on the live path.
+ */
+export class ChannelStrippingDelegate {
+    inner;
+    channels;
+    constructor(inner, channels) {
+        this.inner = inner;
+        this.channels = channels;
+    }
+    execute(prompt, options) {
+        const knowledge = options.injectedKnowledge;
+        if (this.channels.length === 0) {
+            return this.inner.execute(prompt, options);
+        }
+        const strip = new Set(this.channels);
+        const stripped = knowledge
+            ? {
+                ...knowledge,
+                experiences: strip.has('experiences') ? [] : knowledge.experiences,
+                playbooks: strip.has('playbooks') ? [] : knowledge.playbooks,
+            }
+            : knowledge;
+        let systemContext = options.systemContext;
+        if (systemContext) {
+            for (const ch of this.channels) {
+                systemContext = stripMarkdownSection(systemContext, CHANNEL_SECTION_HEADERS[ch]);
+            }
+        }
+        return this.inner.execute(prompt, {
+            ...options,
+            injectedKnowledge: stripped,
+            systemContext,
+        });
+    }
+}
+/**
+ * Live ablation runner: runs the learning arm multiple times, each time
+ * disabling one component, to measure each component's contribution.
+ * Results are single-arm ConditionResults that can be compared to a full run.
+ */
+export class LiveAblationRunner {
+    baseConfig;
+    arms;
+    delegateFactory;
+    constructor(baseConfig, arms, delegateFactory) {
+        this.baseConfig = baseConfig;
+        this.arms = arms;
+        this.delegateFactory = delegateFactory;
+    }
+    async run() {
+        const traceDirFor = (label) => {
+            if (!this.baseConfig.traceDir)
+                return undefined;
+            const safeLabel = label.replace(/[^a-z0-9._-]+/gi, '-').toLowerCase();
+            return join(this.baseConfig.traceDir, safeLabel);
+        };
+        const fullRunner = new LiveEvalRunner({ ...this.baseConfig, singleArm: true, traceDir: traceDirFor('full') }, this.delegateFactory);
+        const full = await fullRunner.run();
+        const ablations = [];
+        for (const arm of this.arms) {
+            // For channel-type ablations, wrap the delegate factory to strip channels.
+            const wrappedFactory = arm.channels?.length
+                ? (benchmark, opts) => {
+                    const inner = this.delegateFactory
+                        ? this.delegateFactory(benchmark, opts)
+                        : new ClaudeBenchmarkDelegate(benchmark, opts);
+                    const channels = arm.channels;
+                    // Return a proxy that strips channels but preserves all delegate hooks
+                    // (setCurrentTask, getTotalCostUsd, setMemory).
+                    const stripping = new ChannelStrippingDelegate(inner, channels);
+                    const bm = inner;
+                    return Object.assign(stripping, {
+                        setCurrentTask: bm.setCurrentTask?.bind(bm),
+                        getTotalCostUsd: bm.getTotalCostUsd?.bind(bm),
+                        setMemory: bm.setMemory?.bind(bm),
+                    });
+                }
+                : this.delegateFactory;
+            const patchedConfig = {
+                ...this.baseConfig,
+                singleArm: true,
+                traceDir: traceDirFor(arm.label),
+                diagnosticStripChannels: arm.channels,
+                configPatch: arm.configPatch && Object.keys(arm.configPatch).length > 0
+                    ? { ...this.baseConfig.configPatch, ...arm.configPatch }
+                    : this.baseConfig.configPatch,
+            };
+            const runner = new LiveEvalRunner(patchedConfig, wrappedFactory);
+            const result = await runner.run();
+            ablations.push({ label: arm.label, result });
+        }
+        return { full, ablations };
+    }
+}
+export function createLiveAblationRunner(baseConfig, arms, delegateFactory) {
+    return new LiveAblationRunner(baseConfig, arms, delegateFactory);
+}
+//# sourceMappingURL=live-runner.js.map