cognitive-core 0.2.5 → 0.3.1

package/dist/bin/commands/eval.js

@@ -0,0 +1,1582 @@
+/**
+ * Evaluation CLI — runs the eval harness and prints a report. Parses its own
+ * flags (the shared parser doesn't know eval options).
+ *
+ * Subcommands / modes:
+ *   cogcore eval                       Sequential learning-curve eval (mock)
+ *   cogcore eval ablation              Per-component ablation (mock)
+ *   cogcore eval --backend live --benchmark <name>   Real-agent live arm
+ *   cogcore eval ablation --backend live --benchmark clbench   Live memory ablation
+ *
+ * Flags:
+ *   --backend mock|live   --benchmark <name>   --model haiku
+ *   --tasks N   --skills 6   --seeds 1,2,3   --window 10   --max-steps 30
+ *   --cold 0.35   --warm 0.9   --batch-every 0   --components a,b,c (ablation)
+ *   --out report.md   --json
+ */
+import { existsSync, rmSync, writeFileSync } from 'node:fs';
+import { join, dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createSequentialEvalRunner, createAblationRunner, createLiveEvalRunner, createLiveAblationRunner, createRunCheckpoint, checkpointPathFor, workdirPathFor, loadColdArmResults, loadDifficultyRegistry, saveDifficultyRegistry, ingestRunRows, extractIngestRows, estimateFamilySuccess, ingestLiftRows, extractLiftIngestRows, estimateLift, mergeLiftRegistries, resolveRegistryStore, migrateJsonRegistryToSqlite, SqliteRegistryStore, ClaudeWorkspaceBackend, createGsm8kBenchmark, createAlfworldBenchmark, createSkillsBenchBenchmark, createSkillFlowBenchmark, createClBenchBenchmark, createSkillLearnBenchBenchmark, createLifelongAgentBenchmark, alfworldSeedDir, ABLATION_COMPONENTS, renderJson, renderMarkdown, renderAblationJson, renderAblationMarkdown, renderDiagnosticsReport, DEFAULT_EVAL_CONFIG, getCiSmokeProfile, } from '../../eval/index.js';
+import { renderSwarmkitTraceReport } from '../../eval/swarmkit-report.js';
+import { renderPlaybookUsageReport } from '../../eval/playbook-usage-report.js';
+import { renderPlaybookCalibrationReport } from '../../eval/playbook-calibration-report.js';
+import { renderPlaybookCandidateDecisionsReport } from '../../eval/playbook-candidate-decisions-report.js';
+import { validateAblationPatch } from '../../eval/ablation-validate.js';
+import { loadOracleManifest, findPlaceholderFiles, evaluateOracleCriteria, } from '../../eval/oracle-smoke.js';
+import { canonicalizeSolver } from '../../utils/solver-canonicalizer.js';
+export async function evalCommand(rawArgs) {
+    if (rawArgs[0] === 'eval') {
+        rawArgs = rawArgs.slice(1);
+    }
+    if (rawArgs.includes('--help') || rawArgs.includes('-h')) {
+        printUsage();
+        return;
+    }
+    // eval report --diagnostics <traceDir>  — post-run stage-funnel report
+    if (rawArgs[0] === 'report') {
+        await runReport(rawArgs.slice(1));
+        return;
+    }
+    // eval ingest-difficulty --registry <file> <artifacts...>  — backfill the
+    // difficulty registry from existing result JSONs / checkpoint files.
+    if (rawArgs[0] === 'ingest-difficulty') {
+        await runIngestDifficulty(rawArgs.slice(1));
+        return;
+    }
+    // eval ingest-lift [--registry-db <db> | --registry <file>] <artifacts...>
+    // — backfill the lift registry (guided vs unguided per family × solver).
+    if (rawArgs[0] === 'ingest-lift') {
+        await runIngestLift(rawArgs.slice(1));
+        return;
+    }
+    // eval lift-report — merged eval + production lift scorecard.
+    if (rawArgs[0] === 'lift-report') {
+        await runLiftReport(rawArgs.slice(1));
+        return;
+    }
+    // eval registry migrate --from <file.json> --to <db>  — one-shot JSON → SQLite.
+    if (rawArgs[0] === 'registry' && rawArgs[1] === 'migrate') {
+        await runRegistryMigrate(rawArgs.slice(2));
+        return;
+    }
+    const flags = parseEvalFlags(rawArgs);
+    // Oracle smoke gate: --smoke oracle [--backend mock|live]
+    if (flags['smoke'] === 'oracle') {
+        await runOracleSmoke(flags);
+        return;
+    }
+    if (flags.backend === 'live') {
+        if (rawArgs.includes('ablation')) {
+            await runLiveAblation(flags);
+        }
+        else {
+            await runLive(flags);
+        }
+        return;
+    }
+    const isAblation = rawArgs.includes('ablation');
+    const config = {};
+    if (flags.backend)
+        config.backend = flags.backend;
+    if (flags.tasks !== undefined)
+        config.taskCount = flags.tasks;
+    if (flags.skills !== undefined)
+        config.skillCount = flags.skills;
+    if (flags.seeds)
+        config.seeds = flags.seeds;
+    if (flags.window !== undefined)
+        config.rollingWindow = flags.window;
+    if (flags.cold !== undefined)
+        config.coldProb = flags.cold;
+    if (flags.warm !== undefined)
+        config.warmProb = flags.warm;
+    if (flags['batch-every'] !== undefined)
+        config.forceBatchEvery = flags['batch-every'];
+    const merged = { ...DEFAULT_EVAL_CONFIG, ...config };
+    if (isAblation)
+        await runAblation(config, merged, flags);
+    else
+        await runCurve(config, merged, flags);
+}
+async function runCurve(config, merged, flags) {
+    if (!flags.json) {
+        console.error(`Running sequential learning-curve eval ` +
+            `(backend=${merged.backend}, tasks=${merged.taskCount}, ` +
+            `skills=${merged.skillCount}, seeds=${merged.seeds.join(',')})...`);
+    }
+    const result = await createSequentialEvalRunner(config).run();
+    const markdown = renderMarkdown(result);
+    if (flags.out) {
+        writeFileSync(flags.out, flags.out.endsWith('.json') ? renderJson(result) : markdown);
+        console.error(`Wrote report to ${flags.out}`);
+    }
+    if (flags.json)
+        console.log(renderJson(result));
+    else
+        console.log('\n' + markdown);
+    if (result.successRateDelta <= 0)
+        process.exitCode = 2;
+}
+async function runAblation(config, merged, flags) {
+    if (config.forceBatchEvery === undefined) {
+        config.forceBatchEvery = Math.max(5, merged.skillCount);
+    }
+    const components = flags.components
+        ? ABLATION_COMPONENTS.filter((c) => flags.components.includes(c.key))
+        : undefined;
+    if (!flags.json) {
+        console.error(`Running component ablation (tasks=${merged.taskCount}, seeds=${merged.seeds.join(',')}, ` +
+            `components=${components ? components.map((c) => c.key).join(',') : 'all'})...`);
+    }
+    const result = await createAblationRunner(config, components).run();
+    const markdown = renderAblationMarkdown(result);
+    if (flags.out) {
+        writeFileSync(flags.out, flags.out.endsWith('.json') ? renderAblationJson(result) : markdown);
+        console.error(`Wrote report to ${flags.out}`);
+    }
+    if (flags.json)
+        console.log(renderAblationJson(result));
+    else
+        console.log('\n' + markdown);
+}
+async function runLive(flags) {
+    const benchmarkName = flags.benchmark ?? 'gsm8k';
+    if (flags['ci-smoke']) {
+        const profile = getCiSmokeProfile(benchmarkName);
+        if (profile) {
+            flags.tasks = flags.tasks ?? profile.taskCount;
+            flags.seeds = flags.seeds ?? profile.seeds;
+            flags['timeout'] = flags['timeout'] ?? profile.timeoutMinutes * 60;
+        }
+    }
+    const taskCount = flags.tasks ?? 12;
+    const seeds = flags.seeds ?? [1];
+    const model = flags.model ?? 'haiku';
+    const rollingWindow = flags.window ?? Math.max(3, Math.ceil(taskCount / 3));
+    const seedPlaybookDirs = [];
+    const resolved = resolveBenchmark(benchmarkName, flags);
+    if (!resolved) {
+        process.exitCode = 1;
+        return;
+    }
+    const { benchmark, delegateFactory } = resolved;
+    if (benchmarkName === 'alfworld' && flags['seed-playbooks']) {
+        seedPlaybookDirs.push(alfworldSeedDir());
+    }
+    if (!flags.json) {
+        console.error(`Running LIVE eval on ${benchmarkName} ` +
+            `(model=${model}, tasks=${taskCount}, seeds=${seeds.join(',')}). ` +
+            `This makes ~${taskCount * (flags['single-arm'] ? 1 : 2) * seeds.length} real agent calls — be patient.`);
+    }
+    // Incremental checkpointing: with --out, write a running partial-result file
+    // (<out>.checkpoint.json) after every completed task so an aborted run keeps
+    // its per-task results instead of losing hours of paid episodes. Removed
+    // once the final report is written.
+    // Resumability: with --out, the learning-arm Atlas persists to a stable
+    // <out>.workdir/ (next to the checkpoint) instead of temp dirs, so a run that
+    // dies can be resumed. --resume auto-loads <out>.checkpoint.json (if present)
+    // and skips every task already completed there — reopening the persisted
+    // learning dirs so accumulated memory carries forward.
+    const runWorkDir = flags.out ? workdirPathFor(flags.out) : undefined;
+    let resumeFrom;
+    if (flags['resume']) {
+        if (!flags.out) {
+            console.error('--resume requires --out (it reads <out>.checkpoint.json)');
+            process.exitCode = 1;
+            return;
+        }
+        const cpPath = checkpointPathFor(flags.out);
+        if (existsSync(cpPath)) {
+            resumeFrom = cpPath;
+            if (!flags.json)
+                console.error(`Resuming from checkpoint ${cpPath}`);
+        }
+        else if (!flags.json) {
+            console.error(`--resume: no checkpoint at ${cpPath} — starting a fresh run`);
+        }
+    }
+    // Load the resume checkpoint's cold rows BEFORE the fresh checkpoint's first
+    // write would overwrite the file — createRunCheckpoint defers writing until
+    // the first record(), and the runner (constructed below) loads resumeFrom in
+    // its constructor, so the read still sees the aborted run's rows.
+    const checkpoint = flags.out
+        ? createRunCheckpoint(flags.out, {
+            benchmark: benchmarkName,
+            model,
+            taskCount,
+            seeds,
+            bundledSeeds: flags['bundled-seeds'] ?? false,
+            singleArm: flags['single-arm'] ?? false,
+        })
+        : undefined;
+    // Cold-arm reuse: load per-seed cold results from a prior run's result JSON
+    // or checkpoint file. Fail loudly on a bad path/artifact — silently running
+    // without the control would waste the whole run.
+    let reuseColdBySeed;
+    if (flags['reuse-cold-from']) {
+        reuseColdBySeed = loadColdArmResults(flags['reuse-cold-from']);
+        if (!flags.json) {
+            console.error(`Reusing cold-arm results from ${flags['reuse-cold-from']} ` +
+                `(seeds: ${Object.keys(reuseColdBySeed).join(',')})`);
+        }
+    }
+    // Registries: loaded up front (difficulty drives discriminative task
+    // selection when requested), updated with this run's results afterwards.
+    // --difficulty-registry <file.json> keeps its original JSON contract
+    // unchanged; --registry-db <path> is the SQLite backend and additionally
+    // receives this run's lift rows (guided vs unguided per family × solver).
+    let registryStore;
+    if (flags['registry-db']) {
+        registryStore = new SqliteRegistryStore(flags['registry-db']);
+    }
+    let difficultyRegistry;
+    if (flags['difficulty-registry']) {
+        difficultyRegistry = loadDifficultyRegistry(flags['difficulty-registry']);
+    }
+    else if (registryStore) {
+        difficultyRegistry = registryStore.loadDifficulty();
+    }
+    if (flags['task-selection'] === 'discriminative' && !difficultyRegistry) {
+        console.error('--task-selection discriminative requires --difficulty-registry <file> or --registry-db <path>');
+        process.exitCode = 1;
+        return;
+    }
+    const extended = benchmark;
+    try {
+        if (extended.setup)
+            await extended.setup();
+        const runner = createLiveEvalRunner({
+            benchmark,
+            taskCount,
+            seeds,
+            rollingWindow,
+            model,
+            seedPlaybookDirs,
+            onTaskBoundary: checkpoint?.record,
+            runWorkDir,
+            resumeFrom,
+            reuseColdBySeed,
+            failFast: flags['fail-fast'],
+            ...(flags['task-selection'] === 'discriminative' && difficultyRegistry
+                ? { taskSelection: { mode: 'discriminative', registry: difficultyRegistry } }
+                : {}),
+            ...(flags['adaptive-stopping']
+                ? {
+                    stopping: {
+                        enabled: true,
+                        ...(flags['stopping-threshold'] !== undefined
+                            ? { decisionThreshold: flags['stopping-threshold'] }
+                            : {}),
+                    },
+                }
+                : {}),
+            embed: flags['embeddings'],
+            embedProvider: flags['embed-provider'],
+            singleArm: flags['single-arm'],
+            interleaveArms: flags['interleave-arms'],
+            provider: flags['provider'],
+            awsRegion: flags['aws-region'],
+            timeoutMs: flags['timeout'] ? flags['timeout'] * 1000 : undefined,
+            distill: flags['distill'],
+            ...(flags['train-eval-split']
+                ? {
+                    trainEvalSplit: {
+                        enabled: true,
+                        ...(flags['train-ratio'] !== undefined ? { trainRatio: flags['train-ratio'] } : {}),
+                        ...(flags['split-seed'] !== undefined ? { splitSeed: flags['split-seed'] } : {}),
+                    },
+                }
+                : {}),
+            concurrency: flags['concurrency'],
+            traceDir: flags['trace-dir'],
+            metaPlaybookDirs: flags['meta-playbook-dir'],
+            playbookRetriever: flags['playbook-retriever'],
+            minPlaybookScore: flags['min-playbook-score'],
+            playbookApplicabilityGate: flags['playbook-applicability-gate'],
+            playbookIncorporation: flags['playbook-incorporation'],
+            mutationMode: flags['mutation-mode'],
+            mutationGating: resolveMutationGating(flags),
+            liveReplayGating: flags['live-replay-gating'],
+            bundledSeeds: flags['bundled-seeds'],
+            additive: resolveAdditive(flags),
+            ...liveProceduralMemoryConfig(flags),
+            learningOnlySeedDirs: flags['seed-playbook-dir'],
+            analysisBackends: createAnalysisBackends(flags, model, benchmark),
+        }, delegateFactory);
+        const result = await runner.run();
+        const markdown = renderMarkdown(result);
+        if (flags.out) {
+            writeFileSync(flags.out, flags.out.endsWith('.json') ? renderJson(result) : markdown);
+            console.error(`Wrote report to ${flags.out}`);
+            // Final report is safely on disk — the partial checkpoint and the
+            // resumable learning workdir are now redundant.
+            checkpoint?.finalize();
+            if (runWorkDir)
+                rmSync(runWorkDir, { recursive: true, force: true });
+        }
+        if (flags.json)
+            console.log(renderJson(result));
+        else
+            console.log('\n' + markdown);
+        // Fold this run's per-task outcomes into the difficulty registry so future
+        // discriminative runs sample from richer priors. Done after the report is
+        // safely written; ingestion failures must not eat a completed run.
+        if (flags['difficulty-registry'] && difficultyRegistry) {
+            try {
+                const { rows } = extractIngestRows(result);
+                ingestRunRows(difficultyRegistry, benchmarkName, rows);
+                saveDifficultyRegistry(flags['difficulty-registry'], difficultyRegistry);
+                console.error(`Ingested ${rows.length} rows into difficulty registry ${flags['difficulty-registry']}`);
+            }
+            catch (err) {
+                console.error(`difficulty registry update failed: ${err.message}`);
+            }
+        }
+        // SQLite registry (--registry-db): ingest difficulty AND lift rows. The
+        // lift solver is this run's --model, canonicalized so evidence pools
+        // across aliases of the same model but never across different solvers.
+        if (registryStore) {
+            try {
+                const raw = result;
+                const { rows } = extractIngestRows(raw);
+                const difficulty = registryStore.loadDifficulty();
+                ingestRunRows(difficulty, benchmarkName, rows);
+                registryStore.saveDifficulty(difficulty);
+                const { rows: liftRows } = extractLiftIngestRows(raw);
+                const lift = registryStore.loadLift();
+                ingestLiftRows(lift, benchmarkName, canonicalizeSolver(model), liftRows);
+                registryStore.saveLift(lift);
+                console.error(`Ingested ${rows.length} difficulty rows + ${liftRows.length} lift rows into ${registryStore.description}`);
+            }
+            catch (err) {
+                console.error(`registry-db update failed: ${err.message}`);
+            }
+        }
+    }
+    finally {
+        registryStore?.close();
+        if (extended.teardown)
+            await extended.teardown();
+        await benchmark.close?.();
+    }
+}
+/**
+ * Backfill the difficulty registry from existing run artifacts:
+ *   cogcore eval ingest-difficulty --registry <file> [--benchmark <name>] <artifacts...>
+ * Artifacts are result JSONs (with perSeed rows) or <out>.checkpoint.json
+ * partials — the same shapes cold-arm reuse accepts.
+ */
+async function runIngestDifficulty(args) {
+    let registryPath;
+    let benchmarkOverride;
+    const artifacts = [];
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--registry')
+            registryPath = args[++i];
+        else if (args[i] === '--benchmark')
+            benchmarkOverride = args[++i];
+        else if (!args[i].startsWith('--'))
+            artifacts.push(args[i]);
+    }
+    if (!registryPath || artifacts.length === 0) {
+        console.error('Usage: cogcore eval ingest-difficulty --registry <file> [--benchmark <name>] <artifacts...>');
+        process.exitCode = 1;
+        return;
+    }
+    const registry = loadDifficultyRegistry(registryPath);
+    const { readFileSync } = await import('node:fs');
+    for (const artifact of artifacts) {
+        const raw = JSON.parse(readFileSync(artifact, 'utf8'));
+        const { benchmark, rows } = extractIngestRows(raw);
+        const target = benchmarkOverride ?? benchmark;
+        if (!target) {
+            console.error(`${artifact}: no benchmark name in artifact — pass --benchmark <name> to override`);
+            process.exitCode = 1;
+            return;
+        }
+        if (rows.length === 0) {
+            console.error(`${artifact}: no ingestible task rows found, skipping`);
+            continue;
+        }
+        ingestRunRows(registry, target, rows);
+        console.error(`${artifact}: ingested ${rows.length} rows into '${target}'`);
+    }
+    saveDifficultyRegistry(registryPath, registry);
+    // Summary: per-benchmark family difficulty estimates.
+    for (const [bench, { families }] of Object.entries(registry.benchmarks)) {
+        console.log(`\n${bench}:`);
+        for (const [family, stats] of Object.entries(families).sort()) {
+            const p = estimateFamilySuccess(stats);
+            console.log(`  ${family}: p̂=${p.toFixed(2)} ` +
+                `(${stats.successes}/${stats.attempts} pooled, ` +
+                `${stats.coldSuccesses}/${stats.coldAttempts} cold, ${stats.runs} runs)`);
+        }
+    }
+    console.error(`\nWrote ${registryPath}`);
+}
+/**
+ * Backfill the lift registry from existing run artifacts:
+ *   cogcore eval ingest-lift [--registry-db <db> | --registry <file.json>]
+ *     [--benchmark <name>] [--solver <model>] <artifacts...>
+ * Learning-arm rows map to guided, cold-arm rows to unguided; family is the
+ * row's skillTag. The solver comes from a checkpoint's meta.model when
+ * present, else --solver (final result JSONs carry no model, so --solver is
+ * required for those). Default store: SQLite at eval-traces/registry.db.
+ */
+async function runIngestLift(args) {
+    let registryDb;
+    let registryJson;
+    let benchmarkOverride;
+    let solverFlag;
+    const artifacts = [];
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--registry-db')
+            registryDb = args[++i];
+        else if (args[i] === '--registry')
+            registryJson = args[++i];
+        else if (args[i] === '--benchmark')
+            benchmarkOverride = args[++i];
+        else if (args[i] === '--solver')
+            solverFlag = args[++i];
+        else if (!args[i].startsWith('--'))
+            artifacts.push(args[i]);
+    }
+    if (artifacts.length === 0) {
+        console.error('Usage: cogcore eval ingest-lift [--registry-db <db> | --registry <file.json>] ' +
+            '[--benchmark <name>] [--solver <model>] <artifacts...>');
+        process.exitCode = 1;
+        return;
+    }
+    const store = resolveRegistryStore({ registryDb, registryJson });
+    try {
+        const lift = store.loadLift();
+        const { readFileSync } = await import('node:fs');
+        for (const artifact of artifacts) {
+            const raw = JSON.parse(readFileSync(artifact, 'utf8'));
+            const { benchmark, solver: artifactSolver, rows } = extractLiftIngestRows(raw);
+            const scope = benchmarkOverride ?? benchmark;
+            if (!scope) {
+                console.error(`${artifact}: no benchmark name in artifact — pass --benchmark <name> to override`);
+                process.exitCode = 1;
+                return;
+            }
+            const rawSolver = artifactSolver ?? solverFlag;
+            if (!rawSolver) {
+                console.error(`${artifact}: no solver model in artifact (final result JSONs carry none) — ` +
+                    `pass --solver <model> so cells never pool across solvers`);
+                process.exitCode = 1;
+                return;
+            }
+            if (rows.length === 0) {
+                console.error(`${artifact}: no ingestible task rows found, skipping`);
+                continue;
+            }
+            const solver = canonicalizeSolver(rawSolver);
+            ingestLiftRows(lift, scope, solver, rows);
+            console.error(`${artifact}: ingested ${rows.length} rows into '${scope}' (solver: ${solver})`);
+        }
+        store.saveLift(lift);
+        printLiftTable(lift);
+        console.error(`\nWrote ${store.description}`);
+    }
+    finally {
+        store.close();
+    }
+}
+/**
+ * Merged lift scorecard — "where does memory help?":
+ *   cogcore eval lift-report [--registry-db <db> | --registry <file.json>]
+ *     [--atlas-dir <dir>] [--json]
+ * Reads the eval-side registry store (default eval-traces/registry.db) and
+ * optionally a production .atlas instance DB (its `lift_cells` table,
+ * scope 'production'), printing per-cell counts, delta, CI and decided state.
+ */
+async function runLiftReport(args) {
+    let registryDb;
+    let registryJson;
+    let atlasDir;
+    let json = false;
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--registry-db')
+            registryDb = args[++i];
+        else if (args[i] === '--registry')
+            registryJson = args[++i];
+        else if (args[i] === '--atlas-dir')
+            atlasDir = args[++i];
+        else if (args[i] === '--json')
+            json = true;
+    }
+    const store = resolveRegistryStore({ registryDb, registryJson });
+    let lift;
+    try {
+        lift = store.loadLift();
+    }
+    finally {
+        store.close();
+    }
+    if (atlasDir) {
+        const { existsSync } = await import('node:fs');
+        const dbPath = join(atlasDir, 'cognitive-core.db');
+        if (!existsSync(dbPath)) {
+            console.error(`--atlas-dir: no cognitive-core.db found at ${dbPath}`);
+            process.exitCode = 1;
+            return;
+        }
+        const instance = new SqliteRegistryStore(dbPath, { readonly: true });
+        try {
+            mergeLiftRegistries(lift, instance.loadLift());
+        }
+        finally {
+            instance.close();
+        }
+    }
+    if (json) {
+        const cells = [];
+        for (const [scope, { families }] of Object.entries(lift.scopes)) {
+            for (const [family, solvers] of Object.entries(families)) {
+                for (const [solver, cell] of Object.entries(solvers)) {
+                    cells.push({ scope, family, solver, ...cell, estimate: estimateLift(cell) });
+                }
+            }
+        }
+        console.log(JSON.stringify(cells, null, 2));
+        return;
+    }
+    if (Object.keys(lift.scopes).length === 0) {
+        console.log('Lift registry is empty — run evals with --registry-db or use eval ingest-lift.');
+        return;
+    }
+    printLiftTable(lift);
+}
+/** Per-cell lift summary shared by ingest-lift and lift-report. */
+function printLiftTable(lift) {
+    for (const [scope, { families }] of Object.entries(lift.scopes).sort()) {
+        console.log(`\n${scope}:`);
+        for (const [family, solvers] of Object.entries(families).sort()) {
+            for (const [solver, cell] of Object.entries(solvers).sort()) {
+                const { delta, ci95, decided } = estimateLift(cell);
+                const sign = delta >= 0 ? '+' : '';
+                console.log(`  ${family} × ${solver}: ` +
+                    `guided ${cell.guidedSuccesses}/${cell.guidedAttempts}, ` +
+                    `unguided ${cell.unguidedSuccesses}/${cell.unguidedAttempts} ` +
+                    `(${cell.runs} runs) — Δ=${sign}${delta.toFixed(2)} ` +
+                    `CI[${ci95[0].toFixed(2)}, ${ci95[1].toFixed(2)}] ${decided.toUpperCase()}`);
+            }
+        }
+    }
+}
+/**
+ * One-shot registry migration:
+ *   cogcore eval registry migrate --from <file.json> --to <db>
+ * Copies difficulty + lift cells from a JSON registry file into a SQLite
+ * registry DB. Idempotent (cells are overwritten with the same values).
+ */
+async function runRegistryMigrate(args) {
+    let from;
+    let to;
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--from')
+            from = args[++i];
+        else if (args[i] === '--to')
+            to = args[++i];
+    }
+    if (!from || !to) {
+        console.error('Usage: cogcore eval registry migrate --from <file.json> --to <db>');
+        process.exitCode = 1;
+        return;
+    }
+    const { difficultyCells, liftCells } = migrateJsonRegistryToSqlite(from, to);
+    console.error(`Migrated ${difficultyCells} difficulty cell(s) + ${liftCells} lift cell(s) from ${from} to ${to}`);
+}
+async function runLiveAblation(flags) {
+    const benchmarkName = flags.benchmark ?? 'clbench';
+    const taskCount = flags.tasks ?? 12;
+    const seeds = flags.seeds ?? [1];
+    const model = flags.model ?? 'haiku';
+    const rollingWindow = flags.window ?? Math.max(3, Math.ceil(taskCount / 3));
+    const resolved = resolveBenchmark(benchmarkName, flags);
+    if (!resolved) {
+        process.exitCode = 1;
+        return;
+    }
+    const { benchmark, delegateFactory } = resolved;
+    // Filter ABLATION_COMPONENTS by --components flag (or run all).
+    const selectedSpecs = flags.components
+        ? ABLATION_COMPONENTS.filter((c) => flags.components.includes(c.key))
+        : ABLATION_COMPONENTS;
+    if (flags.components) {
+        const unknown = flags.components.filter((k) => !ABLATION_COMPONENTS.some((c) => c.key === k));
+        if (unknown.length > 0) {
+            console.error(`Unknown ablation component keys: ${unknown.join(', ')}. ` +
+                `Valid keys: ${ABLATION_COMPONENTS.map((c) => c.key).join(', ')}`);
+            process.exitCode = 1;
+            return;
+        }
+    }
+    // Build AblationArm[] from the registry. Strict-validate each configPatch so
+    // unknown keys (e.g. old memory.disableExperiences) hard-error instead of
+    // being silently stripped by Zod, which would make the arm run the full system.
+    const arms = [];
+    for (const spec of selectedSpecs) {
+        const patch = spec.configPatch ?? {};
+        try {
+            validateAblationPatch(patch, `no-${spec.key}`);
+        }
+        catch (err) {
+            console.error(err.message);
+            process.exitCode = 1;
+            return;
+        }
+        arms.push({
+            label: `no-${spec.key}`,
+            configPatch: patch,
+            channels: spec.channels,
+        });
+    }
+    if (!flags.json) {
+        console.error(`Running LIVE ablation on ${benchmarkName} ` +
+            `(model=${model}, tasks=${taskCount}, components=${selectedSpecs.map((s) => s.key).join(',')}).`);
+    }
+    const extended = benchmark;
+    try {
+        if (extended.setup)
+            await extended.setup();
+        const runner = createLiveAblationRunner({
+            benchmark,
+            taskCount,
+            seeds,
+            rollingWindow,
+            model,
+            provider: flags['provider'],
+            awsRegion: flags['aws-region'],
+            timeoutMs: flags['timeout'] ? flags['timeout'] * 1000 : undefined,
+            concurrency: flags['concurrency'],
+            traceDir: flags['trace-dir'],
+            metaPlaybookDirs: flags['meta-playbook-dir'],
+            playbookRetriever: flags['playbook-retriever'],
+            minPlaybookScore: flags['min-playbook-score'],
+            playbookIncorporation: flags['playbook-incorporation'],
+            mutationMode: flags['mutation-mode'],
+            mutationGating: resolveMutationGating(flags),
+            liveReplayGating: flags['live-replay-gating'],
+            bundledSeeds: flags['bundled-seeds'],
+            additive: resolveAdditive(flags),
+            ...liveProceduralMemoryConfig(flags),
+            embed: flags['embeddings'],
+            embedProvider: flags['embed-provider'],
+            distill: flags['distill'],
+            analysisBackends: createAnalysisBackends(flags, model, benchmark),
+        }, arms, delegateFactory);
+        const result = await runner.run();
+        const lines = [
+            `# Live Ablation: ${benchmarkName}`,
+            '',
+            `**Full system:** ${(result.full.learning.successRate * 100).toFixed(1)}%`,
+            '',
+            '| Component disabled | Success rate | Contribution |',
+            '|--------------------|-------------|-------------|',
+        ];
+        for (const a of result.ablations) {
+            const rate = (a.result.learning.successRate * 100).toFixed(1);
+            const contrib = ((result.full.learning.successRate - a.result.learning.successRate) *
+                100).toFixed(1);
+            lines.push(`| ${a.label} | ${rate}% | ${Number(contrib) >= 0 ? '+' : ''}${contrib}pp |`);
+        }
+        const report = lines.join('\n');
+        if (flags.out) {
+            writeFileSync(flags.out, report);
+            console.error(`Wrote report to ${flags.out}`);
+        }
+        if (flags.json) {
+            console.log(JSON.stringify({ full: result.full, ablations: result.ablations }, null, 2));
+        }
+        else {
+            console.log('\n' + report);
+        }
+    }
+    finally {
+        if (extended.teardown)
+            await extended.teardown();
+        await benchmark.close?.();
+    }
+}
+/**
+ * Oracle smoke harness (--smoke oracle).
+ *
+ * Live path: runs two-arm ALFWorld on manifest.games with the learning arm seeded
+ * from the fixture oracle playbooks, then asserts the pre-registered criteria.
+ *
+ * Mock path (--backend mock): uses a recording delegate that captures the
+ * injected systemContext for each task. Assertion: for each task, the seeded
+ * oracle playbook's strategy text appears in the captured systemContext, proving
+ * seed → retrieve → inject plumbing works.
+ */
+async function runOracleSmoke(flags) {
+    const here = dirname(fileURLToPath(import.meta.url));
+    const fixtureDir = join(here, '../../../eval-fixtures/oracle/alfworld');
+    // Load and validate manifest.
+    let manifest;
+    try {
+        manifest = loadOracleManifest(fixtureDir);
+    }
+    catch (err) {
+        console.error(`oracle smoke: ${err.message}`);
+        process.exitCode = 1;
+        return;
+    }
+    // Refuse to run if any oracle playbook still has placeholder content.
+    const placeholders = findPlaceholderFiles(fixtureDir);
+    if (placeholders.length > 0) {
+        console.error(`oracle smoke: refusing to run — the following oracle playbooks still contain ` +
+            `__FILL_FROM_COLD_RUN__ placeholders:\n` +
+            placeholders.map((f) => `  ${f}`).join('\n') +
+            `\n\nRun a cold pass first to select the real action sequences, then fill them in.`);
+        process.exitCode = 1;
+        return;
+    }
+    const backend = flags.backend ?? 'live';
+    if (backend === 'mock') {
+        // Mock CI variant: verify the seed → retrieve → inject plumbing.
+        // We use a recording delegate that captures systemContext per task and assert
+        // the oracle playbook strategy text appears in it.
+        console.error(`oracle smoke (mock): verifying seed → retrieve → inject plumbing...`);
+        const { createLiveEvalRunner: _r } = await import('../../eval/index.js');
+        const bench = createAlfworldBenchmark({
+            maxSteps: 1,
+            reinject: true,
+            gameIndices: manifest.games,
+        });
+        // Recording delegate: captures systemContext for each execute() call.
+        const captured = [];
+        let taskIdx = 0;
+        const recordingDelegateFactory = (_benchmark, _opts) => {
+            return {
+                setCurrentTask: () => {
+                    taskIdx++;
+                },
+                execute: async (_prompt, options) => {
+                    captured.push({ taskIdx, systemContext: options.systemContext ?? '' });
+                    // Return success so learning arm records a trajectory.
+                    return { success: true, output: 'mock-oracle-pass' };
+                },
+            };
+        };
+        const runner = createLiveEvalRunner({
+            benchmark: bench,
+            taskCount: manifest.games.length,
+            seeds: [1],
+            rollingWindow: manifest.games.length,
+            model: 'mock',
+            learningOnlySeedDirs: [fixtureDir],
+            singleArm: false,
+            quiet: flags.json,
+        }, recordingDelegateFactory);
+        let result;
+        try {
+            result = await runner.run();
+        }
+        catch (err) {
+            console.error(`oracle smoke (mock): runner failed: ${err.message}`);
+            process.exitCode = 1;
+            return;
+        }
+        finally {
+            await bench.close?.();
+        }
+        // Assert plumbing: every captured systemContext that has a non-empty context
+        // should contain at least one of the oracle playbook names (strategy-level signal).
+        // This proves the seeded playbooks were retrieved and injected.
+        const withContext = captured.filter((c) => c.systemContext.length > 0);
+        const hasInjection = withContext.length > 0;
+        if (!hasInjection) {
+            console.error(`oracle smoke (mock): FAIL — no systemContext was injected into any task.`);
+            console.error(`  This means the seed → retrieve → inject plumbing is broken.`);
+            process.exitCode = 1;
+            return;
+        }
+        console.log(`oracle smoke (mock): PASS`);
+        console.log(`  Injected context in ${withContext.length}/${captured.length} tasks`);
+        console.log(`  Learning success rate: ${(result.learning.successRate * 100).toFixed(0)}%`);
+        return;
+    }
+    // Live path: real ALFWorld two-arm run.
+    const model = flags.model ?? 'haiku';
+    const seeds = flags.seeds ?? [1];
+    const bench = createAlfworldBenchmark({
+        maxSteps: flags['max-steps'] ?? 30,
+        reinject: !flags['no-reinject'],
+        gameIndices: manifest.games,
+    });
+    console.error(`oracle smoke (live): running ${manifest.games.length} games × 2 arms × ${seeds.length} seed(s) ` +
+        `(games: ${manifest.games.join(',')})`);
+    const extended = bench;
+    let learningResults = [];
+    let coldResults = [];
+    try {
+        if (extended.setup)
+            await extended.setup();
+        const runner = createLiveEvalRunner({
+            benchmark: bench,
+            taskCount: manifest.games.length,
+            seeds,
+            rollingWindow: manifest.games.length,
+            model,
+            learningOnlySeedDirs: [fixtureDir],
+            provider: flags['provider'],
+            awsRegion: flags['aws-region'],
+            timeoutMs: flags['timeout'] ? flags['timeout'] * 1000 : undefined,
+            quiet: flags.json,
+        }, bench.delegateFactory);
+        const result = await runner.run();
+        learningResults = result.learning.perSeed.flat().map((r) => r.success);
+        coldResults = result.noLearning.perSeed.flat().map((r) => r.success);
+    }
+    finally {
+        if (extended.teardown)
+            await extended.teardown();
+        await bench.close?.();
+    }
+    const smokeResult = evaluateOracleCriteria(learningResults, coldResults, manifest.preRegistered);
+    for (const line of smokeResult.lines) {
+        console.log(line);
+    }
+    if (flags.out) {
+        writeFileSync(flags.out, smokeResult.lines.join('\n') + '\n');
+        console.error(`Wrote oracle smoke report to ${flags.out}`);
+    }
+    process.exitCode = smokeResult.pass ? 0 : 1;
+}
+function resolveBenchmark(name, flags) {
+    switch (name) {
+        case 'gsm8k':
+            return { benchmark: createGsm8kBenchmark() };
+        case 'alfworld': {
+            const curriculum = flags['curriculum'];
+            const bench = createAlfworldBenchmark({
+                maxSteps: flags['max-steps'] ?? 30,
+                reinject: !flags['no-reinject'],
+                stratified: flags['stratified'],
+                refine: flags['refine'],
+                curriculum: curriculum === 'recurrence-ladder' ? 'recurrence-ladder' : undefined,
+                gameIndices: flags['games'],
+            });
+            return { benchmark: bench, delegateFactory: bench.delegateFactory };
+        }
+        case 'skillsbench': {
+            const bench = createSkillsBenchBenchmark({
+                repoPath: flags['benchmark-repo'],
+                timeoutMs: flags['timeout'] ? flags['timeout'] * 1000 : undefined,
+            });
+            return { benchmark: bench, delegateFactory: bench.delegateFactory };
+        }
+        case 'skillflow': {
+            const bench = createSkillFlowBenchmark({
+                maxSteps: flags['max-steps'] ?? 25,
+                reinject: !flags['no-reinject'],
+                historyWindow: flags['history-window'],
+                families: flags['families'],
+                runtime: flags['skillflow-runtime'],
+                embeddedWorkbookBundle: flags['skillflow-embedded-workbook-bundle'],
+            });
+            return { benchmark: bench, delegateFactory: bench.delegateFactory };
+        }
+        case 'clbench': {
+            const bench = createClBenchBenchmark({
+                maxSteps: flags['max-steps'] ?? 30,
+                reinject: !flags['no-reinject'],
+                historyWindow: flags['history-window'],
+            });
+            return { benchmark: bench, delegateFactory: bench.delegateFactory };
+        }
+        case 'skilllearnbench': {
+            const bench = createSkillLearnBenchBenchmark({
+                iterations: flags['iterations'],
+            });
+            return { benchmark: bench, delegateFactory: bench.delegateFactory };
+        }
+        case 'lifelongagent': {
+            const bench = createLifelongAgentBenchmark({
+                maxSteps: flags['max-steps'] ?? 20,
+                historyWindow: flags['history-window'],
+            });
+            return { benchmark: bench, delegateFactory: bench.delegateFactory };
+        }
+        default:
+            console.error(`Unknown benchmark '${name}'. Use: gsm8k | alfworld | skillsbench | skillflow | clbench | skilllearnbench | lifelongagent`);
+            return null;
+    }
+}
+function parseEvalFlags(args) {
+    const flags = {};
+    for (let i = 0; i < args.length; i++) {
+        const a = args[i];
+        switch (a) {
+            case '--backend':
+                flags.backend = args[++i];
+                break;
+            case '--benchmark':
+                flags.benchmark = args[++i];
+                break;
+            case '--model':
+                flags.model = args[++i];
+                break;
+            case '--tasks':
+            case '--limit':
+                flags.tasks = parseInt(args[++i], 10);
+                break;
+            case '--skills':
+                flags.skills = parseInt(args[++i], 10);
+                break;
+            case '--seeds':
+                flags.seeds = args[++i]
+                    .split(',')
+                    .map((s) => parseInt(s.trim(), 10))
+                    .filter((n) => !Number.isNaN(n));
+                break;
+            case '--window':
+                flags.window = parseInt(args[++i], 10);
+                break;
+            case '--cold':
+                flags.cold = parseFloat(args[++i]);
+                break;
+            case '--warm':
+                flags.warm = parseFloat(args[++i]);
+                break;
+            case '--batch-every':
+                flags['batch-every'] = parseInt(args[++i], 10);
+                break;
+            case '--max-steps':
+                flags['max-steps'] = parseInt(args[++i], 10);
+                break;
+            case '--seed-playbooks':
+                flags['seed-playbooks'] = true;
+                break;
+            case '--bundled-seeds':
+                flags['bundled-seeds'] = true;
+                break;
+            case '--additive':
+                flags['additive'] = true;
+                break;
+            case '--no-additive':
+                flags['additive'] = false;
+                break;
+            case '--reuse-cold-from':
+                flags['reuse-cold-from'] = args[++i];
+                break;
+            case '--resume':
+                flags['resume'] = true;
+                break;
+            case '--fail-fast':
+                flags['fail-fast'] = true;
+                break;
+            case '--adaptive-stopping':
+                flags['adaptive-stopping'] = true;
+                break;
+            case '--stopping-threshold':
+                flags['stopping-threshold'] = parseFloat(args[++i]);
+                break;
+            case '--difficulty-registry':
+                flags['difficulty-registry'] = args[++i];
+                break;
+            case '--registry-db':
+                flags['registry-db'] = args[++i];
+                break;
+            case '--task-selection': {
+                const value = args[++i];
+                if (value === 'fixed' || value === 'discriminative') {
+                    flags['task-selection'] = value;
+                }
+                else {
+                    throw new Error(`Invalid --task-selection value: ${value}`);
+                }
+                break;
+            }
+            case '--no-reinject':
+                flags['no-reinject'] = true;
+                break;
+            case '--embeddings':
+                flags['embeddings'] = true;
+                break;
+            case '--embed-provider':
+                flags['embed-provider'] = args[++i];
+                flags['embeddings'] = true;
+                break;
+            case '--single-arm':
+                flags['single-arm'] = true;
+                break;
+            case '--concurrency':
+                flags['concurrency'] = parseInt(args[++i], 10);
+                break;
+            case '--interleave-arms':
+                flags['interleave-arms'] = true;
+                break;
+            case '--stratified':
+                flags['stratified'] = true;
+                break;
+            case '--refine':
+                flags['refine'] = true;
+                break;
+            case '--ci-smoke':
+                flags['ci-smoke'] = true;
+                break;
+            case '--components':
+                flags.components = args[++i].split(',').map((s) => s.trim());
+                break;
+            case '--benchmark-repo':
+                flags['benchmark-repo'] = args[++i];
+                break;
+            case '--iterations':
+                flags['iterations'] = parseInt(args[++i], 10);
+                break;
+            case '--history-window':
+                flags['history-window'] = parseInt(args[++i], 10);
+                break;
+            case '--timeout':
+                flags['timeout'] = parseInt(args[++i], 10);
+                break;
+            case '--provider':
+                flags['provider'] = args[++i];
+                break;
+            case '--aws-region':
+                flags['aws-region'] = args[++i];
+                break;
+            case '--analysis-provider':
+                flags['analysis-provider'] = args[++i];
+                break;
+            case '--analysis-model':
+                flags['analysis-model'] = args[++i];
+                break;
+            case '--analysis-aws-region':
+                flags['analysis-aws-region'] = args[++i];
+                break;
+            case '--smoke':
+                flags['smoke'] = args[++i];
+                break;
+            case '--curriculum':
+                flags['curriculum'] = args[++i];
+                break;
+            case '--games':
+                flags['games'] = args[++i]
+                    .split(',')
+                    .map((s) => parseInt(s.trim(), 10))
+                    .filter((n) => !Number.isNaN(n));
+                break;
+            case '--families':
+                flags['families'] = args[++i]
+                    .split(',')
+                    .map((s) => s.trim())
+                    .filter(Boolean);
+                break;
+            case '--skillflow-runtime':
+                flags['skillflow-runtime'] = args[++i];
+                break;
+            case '--skillflow-embedded-workbook-bundle':
+                flags['skillflow-embedded-workbook-bundle'] = true;
+                break;
+            case '--distill':
+                flags['distill'] = true;
+                break;
+            case '--train-eval-split':
+                flags['train-eval-split'] = true;
+                break;
+            case '--train-ratio':
+                flags['train-ratio'] = Number(args[++i]);
+                break;
+            case '--split-seed':
+                flags['split-seed'] = Number(args[++i]);
+                break;
+            case '--agentic-playbooks':
+                flags['agentic-playbooks'] = true;
+                break;
+            case '--no-agentic-playbooks':
+                flags['no-agentic-playbooks'] = true;
+                break;
+            case '--playbook-applicability-gate':
+                flags['playbook-applicability-gate'] = true;
+                break;
+            case '--playbook-incorporation':
+                flags['playbook-incorporation'] = args[++i];
+                break;
+            case '--mutation-mode': {
+                const value = args[++i];
+                if (value === 'frozen' || value === 'active' || value === 'candidate') {
+                    flags['mutation-mode'] = value;
+                }
+                else {
+                    throw new Error(`Invalid --mutation-mode value: ${value}`);
+                }
+                break;
+            }
+            case '--mutation-gating':
+                flags['mutation-gating'] = true;
+                break;
+            case '--no-mutation-gating':
+                flags['mutation-gating'] = false;
+                break;
+            case '--live-replay-gating':
+                flags['live-replay-gating'] = true;
+                break;
+            case '--procedural-memory-source': {
+                const value = args[++i];
+                if (value === 'sqlite' || value === 'filesystem') {
+                    flags['procedural-memory-source'] = value;
+                }
+                else {
+                    throw new Error(`Invalid --procedural-memory-source value: ${value}`);
+                }
+                break;
+            }
+            case '--procedural-memory-dir':
+                flags['procedural-memory-dir'] = args[++i];
+                break;
+            case '--procedural-memory-strict':
+                flags['procedural-memory-strict'] = true;
+                break;
+            case '--procedural-memory-prune-sqlite-only':
+                flags['procedural-memory-prune-sqlite-only'] = true;
+                break;
+            case '--trace-dir':
+                flags['trace-dir'] = args[++i];
+                break;
+            case '--playbook-retriever': {
+                const value = args[++i];
+                if (value === 'legacy' || value === 'skill-tree-hybrid') {
+                    flags['playbook-retriever'] = value;
+                }
+                else {
+                    throw new Error(`Invalid --playbook-retriever value: ${value}`);
+                }
+                break;
+            }
+            case '--min-playbook-score': {
+                const value = parseFloat(args[++i]);
+                if (!Number.isFinite(value) || value < 0 || value > 1) {
+                    throw new Error(`Invalid --min-playbook-score value: ${args[i]}`);
+                }
+                flags['min-playbook-score'] = value;
+                break;
+            }
+            case '--meta-playbook-dir':
+                flags['meta-playbook-dir'] = [
+                    ...(flags['meta-playbook-dir'] ?? []),
+                    ...args[++i]
+                        .split(',')
+                        .map((s) => s.trim())
+                        .filter(Boolean),
+                ];
+                break;
+            case '--seed-playbook-dir':
+                flags['seed-playbook-dir'] = [
+                    ...(flags['seed-playbook-dir'] ?? []),
+                    ...args[++i]
+                        .split(',')
+                        .map((s) => s.trim())
+                        .filter(Boolean),
+                ];
+                break;
+            case '--out':
+                flags.out = args[++i];
+                break;
+            case '--json':
+                flags.json = true;
+                break;
+            default:
+                break;
+        }
+    }
+    return flags;
+}
+/**
+ * Resolve the mutation-gating default for live eval runs: ON unless the run
+ * is frozen (retrieval-only smokes have nothing to mutate) or the user opts
+ * out with `--no-mutation-gating`. This makes the full learning loop —
+ * extraction + reflective-mutation/pareto-merge gating — the default
+ * measured system, so "learning arm" means the whole pipeline, not just
+ * extraction.
+ */
+export function resolveMutationGating(flags) {
+    return flags['mutation-gating'] ?? (flags['mutation-mode'] ?? 'active') !== 'frozen';
+}
+/**
+ * Resolve additive-consolidation for live eval runs. Default ON (matching the
+ * `features.additiveConsolidation` config default): `--additive` sets it true,
+ * `--no-additive` sets it false, neither leaves it at the default (true). This
+ * mirrors `resolveMutationGating` — both `--additive` and `--no-additive` write
+ * the same `additive` flag key.
+ */
+export function resolveAdditive(flags) {
+    return flags['additive'] ?? true;
+}
+export function liveProceduralMemoryConfig(flags) {
+    if (!flags['procedural-memory-source'] &&
+        !flags['procedural-memory-dir'] &&
+        !flags['procedural-memory-strict'] &&
+        !flags['procedural-memory-prune-sqlite-only']) {
+        return {};
+    }
+    return {
+        proceduralMemorySource: flags['procedural-memory-source'] ?? 'filesystem',
+        ...(flags['procedural-memory-dir']
+            ? { proceduralMemoryDir: resolve(flags['procedural-memory-dir']) }
+            : {}),
+        ...(flags['procedural-memory-strict'] !== undefined
+            ? { proceduralMemoryStrictStartup: flags['procedural-memory-strict'] }
+            : {}),
+        ...(flags['procedural-memory-prune-sqlite-only'] !== undefined
+            ? { proceduralMemoryPruneSqliteOnly: flags['procedural-memory-prune-sqlite-only'] }
+            : {}),
+    };
+}
+export function shouldUseAgenticPlaybooks(flags, benchmark) {
+    if (flags['no-agentic-playbooks'])
+        return false;
+    if (flags['agentic-playbooks'])
+        return true;
+    return (typeof benchmark?.playbookMinTrajectories === 'number' && benchmark.playbookMinTrajectories > 0);
+}
+function createAnalysisBackends(flags, solveModel, benchmark) {
+    if (!shouldUseAgenticPlaybooks(flags, benchmark))
+        return undefined;
+    const provider = flags['analysis-provider'] ??
+        (isResponsesProviderName(flags.provider) ? 'bedrock' : flags.provider);
+    const model = flags['analysis-model'] ?? (provider && provider !== flags.provider ? 'haiku' : solveModel);
+    return [
+        new ClaudeWorkspaceBackend({
+            model,
+            timeoutMs: Math.max(flags['timeout'] ? flags['timeout'] * 1000 : 0, 180_000),
+            provider,
+            awsRegion: flags['analysis-aws-region'] ?? flags['aws-region'],
+        }),
+    ];
+}
+function isResponsesProviderName(provider) {
+    return provider === 'openai' || provider === 'azure' || provider === 'azure-openai';
+}
+async function runReport(args) {
+    let diagnostics = false;
+    let swarmkitLearning = false;
+    let playbookUsage = false;
+    let playbookCalibration = false;
+    let candidateDecisions = false;
+    let traceDir;
+    let outFile;
+    let learningArm;
+    let baselineArm;
+    let model;
+    let window;
+    let postExtractionStartIndex;
+    let thresholds;
+    const truePositiveNames = [];
+    for (let i = 0; i < args.length; i++) {
+        switch (args[i]) {
+            case '--diagnostics':
+                diagnostics = true;
+                break;
+            case '--swarmkit-learning':
+                swarmkitLearning = true;
+                break;
+            case '--playbook-usage':
+                playbookUsage = true;
+                break;
+            case '--playbook-calibration':
+                playbookCalibration = true;
+                break;
+            case '--candidate-decisions':
+                candidateDecisions = true;
+                break;
+            case '--trace-dir':
+                traceDir = args[++i];
+                break;
+            case '--out':
+                outFile = args[++i];
+                break;
+            case '--learning-arm':
+                learningArm = args[++i];
+                break;
+            case '--baseline-arm':
+                baselineArm = args[++i];
+                break;
+            case '--model':
+                model = args[++i];
+                break;
+            case '--window':
+                window = parseInt(args[++i], 10);
+                break;
+            case '--post-extraction-start':
+                postExtractionStartIndex = parseInt(args[++i], 10);
+                break;
+            case '--thresholds':
+                thresholds = parseNumberList(args[++i]);
+                break;
+            case '--true-positive':
+                truePositiveNames.push(args[++i]);
+                break;
+            default:
+                break;
+        }
+    }
+    if (!diagnostics &&
+        !swarmkitLearning &&
+        !playbookUsage &&
+        !playbookCalibration &&
+        !candidateDecisions) {
+        console.error('eval report: unknown mode. Currently supported: --diagnostics, --swarmkit-learning, --playbook-usage, --playbook-calibration, or --candidate-decisions with --trace-dir <dir>');
+        process.exitCode = 1;
+        return;
+    }
+    if (!traceDir) {
+        console.error('eval report requires --trace-dir <dir>');
+        process.exitCode = 1;
+        return;
+    }
+    const report = swarmkitLearning
+        ? renderSwarmkitTraceReport(traceDir, {
+            learningArm,
+            baselineArm,
+            model,
+            window,
+            postExtractionStartIndex,
+        })
+        : playbookUsage
+            ? renderPlaybookUsageReport(traceDir, {
+                learningArm,
+                baselineArm,
+                postExtractionStartIndex,
+            })
+            : playbookCalibration
+                ? renderPlaybookCalibrationReport(traceDir, {
+                    learningArm,
+                    baselineArm,
+                    thresholds,
+                })
+                : candidateDecisions
+                    ? renderPlaybookCandidateDecisionsReport(traceDir, {
+                        learningArm,
+                        baselineArm,
+                        truePositives: truePositiveNames.map((playbookName) => ({ playbookName })),
+                    })
+                    : renderDiagnosticsReport(traceDir);
+    if (outFile) {
+        writeFileSync(outFile, report + '\n');
+        console.error(`Wrote diagnostics report to ${outFile}`);
+    }
+    else {
+        console.log('\n' + report);
+    }
+}
+function parseNumberList(value) {
+    return (value ?? '')
+        .split(',')
+        .map((part) => Number(part.trim()))
+        .filter((n) => Number.isFinite(n));
+}
+function printUsage() {
+    console.log(`Usage: cogcore eval [ablation] [options]
+
+Modes:
+  cogcore eval                              Sequential learning-curve eval (mock)
+  cogcore eval ablation                     Per-component ablation (mock)
+  cogcore eval --backend live --benchmark <name>    Real-agent live arm
+  cogcore eval ablation --backend live --benchmark clbench   Live memory ablation
+
+The learning-curve eval solves an ordered task stream with learning enabled
+(memory carried across tasks) vs. a no-learning baseline (fresh memory per task),
+reporting success-rate delta, normalized gain, and learning curves.
+
+Benchmarks:
+  gsm8k            Grade-school math (single-turn, deterministic)
+  alfworld         Embodied household tasks (multi-turn, needs Python env)
+  skillsbench      84 tasks, 11 domains — one-shot skill quality gate (Docker)
+  skillflow        166 tasks, 5 domains — cumulative learning curve (Docker + bridge)
+  clbench          ~30 tasks, 6 domains — memory hierarchy vs naive ICL (bridge)
+  skilllearnbench  20 tasks — iterative skill generation quality (bridge)
+  lifelongagent    ~50 tasks, 3 envs (DB/OS/KG) — sequential experience reuse (bridge)
+
+Options:
+  --backend <mock|live>   Execution backend (default: mock)
+  --benchmark <name>      Live benchmark (see list above)
+  --model <name>          Model for live runs (Claude CLI or Responses API; default: haiku)
+  --tasks/--limit <n>     Tasks per arm (mock default 60, live default 12)
+  --skills <n>            (mock) distinct skill families (default: 6)
+  --seeds <a,b,c>         Seeds (mock default 1,2,3; live default 1)
+  --window <n>            Rolling-window size for the curve
+  --max-steps <n>         Max env steps per episode (default varies by benchmark)
+  --seed-playbooks        (alfworld) seed curated recipe playbooks (B1 cold-start)
+  --bundled-seeds         (live) enable committed cold-start seeds (bundled packs +
+                          meta seeds + routing priors) in BOTH arms. Seed-lift A/B:
+                          run once without, once with, and compare per arm.
+  --additive              (live) ADDITIVE LEARNING (default ON): refine/reuse
+                          existing playbooks (inline match-and-refine + injection-
+                          aligned frontier accounting) instead of minting a unique
+                          distilled-<uuid> per trajectory. On by default as of
+                          2026-07-06; this flag is now a no-op explicit opt-IN.
+  --no-additive           (live) Opt OUT of additive learning: restore the legacy
+                          proliferating behavior (a new unique distilled-<uuid>
+                          per trajectory). A/B: run once with --no-additive and
+                          once without to compare additive vs proliferating.
+  --no-reinject           Inject knowledge on turn 1 only, not every step
+  --embeddings            Semantic (hash-embedding) experience retrieval
+  --single-arm            Run only the learning arm (no cold baseline)
+  --concurrency <n>       Max concurrent cold-arm episodes (default 1). The
+                          harness is I/O-bound on the model, so n=4-8 gives a
+                          large speedup on the no-learning arm at ~0 CPU cost.
+  --interleave-arms       Per-seed arm-order alternation (even seeds: cold first)
+  --stratified            (alfworld) sample games evenly across task types
+  --curriculum <name>     (alfworld) curriculum ordering: recurrence-ladder = Curriculum-1
+                          block→interleave→probe phases (Section 6.2)
+  --families <a,b>        (skillflow) exact task families, preserving listed family order
+  --skillflow-runtime <r> (skillflow) docker or e2b (default: docker / SKILLFLOW_RUNTIME)
+  --skillflow-embedded-workbook-bundle
+                          (skillflow) Install the embedded workbook helper bundle in EDR tasks.
+  --refine                (alfworld) B3 in-episode think-act-refine loop
+  --ci-smoke              Apply budget-capped CI smoke profile for the benchmark
+  --benchmark-repo <path> Path to benchmark repo (skillsbench)
+  --iterations <n>        Iterations per task (skilllearnbench, default 5)
+  --history-window <n>    History window for multi-turn prompts
+  --timeout <seconds>     Per-task timeout in seconds
+  --provider <name>       API provider (bedrock, vertex, openai, azure-openai). Default: ambient auth.
+                          bedrock routes calls via CLAUDE_CODE_USE_BEDROCK env routing (fixed
+                          2026-07-02, 6798326 — before that it silently used the Anthropic
+                          subscription); needs ~/.aws creds with bedrock:InvokeModel for the
+                          BEDROCK_MODELS inference profiles (src/eval/live/claude-cli.ts).
+                          Verify via CloudWatch AWS/Bedrock Invocations for the model id.
+  --aws-region <region>   AWS region for Bedrock (default us-west-1)
+  --analysis-provider <p> Provider for workspace-agent extraction. Defaults to the solve
+                          provider, except OpenAI/Azure solve calls default analysis to bedrock.
+  --analysis-model <m>    Model for workspace-agent extraction (default follows provider)
+  --analysis-aws-region <r>
+                          AWS region for analysis provider when using Bedrock
+  --distill               Wire a real LLM into the learning arm (distiller + convention-distiller).
+                          Learning arm only; cold arm is unaffected. On ALFWorld (no explicit
+                          playbookMinTrajectories), also lowers extraction threshold to 3 so the
+                          distiller fires mid-stream. Use with --backend live. (G3 arm C)
+  --train-eval-split      Complete train/eval isolation: the learner trains on a per-family
+                          TRAIN split (oracle on) then is FROZEN and scored on a disjoint
+                          held-out EVAL split (oracle off, solver firewall intact). Only the
+                          held-out tasks are counted. Cold baseline runs on the same EVAL split.
+  --train-ratio <0..1>    Fraction of each family's tasks used for TRAINING (default 0.5).
+                          Requires --train-eval-split.
+  --split-seed <n>        Deterministic seed offset for the train/eval partition (default 0).
+                          Requires --train-eval-split.
+  --agentic-playbooks     Use a separate Claude workspace agent for dream-phase playbook extraction.
+                          Default for live benchmarks with playbookMinTrajectories; opt-in for others.
+                          Keeps benchmark solve calls single-delegate while exercising the abstractor.
+  --no-agentic-playbooks  Force legacy heuristic dream-phase playbook extraction.
+  --trace-dir <dir>       Write live per-task diagnostics JSONL for eval report --diagnostics.
+  --playbook-retriever <legacy|skill-tree-hybrid>
+                          Select solver-facing playbook retrieval implementation
+                          (default skill-tree-hybrid; legacy lexical scores are too
+                          sparse to clear the 0.1 injection floor).
+  --min-playbook-score <n>
+                          Minimum playbook relevance score for injection (default 0.1).
+  --playbook-applicability-gate
+                          Drop retrieved playbooks that lack task-specific applicability cues.
+  --mutation-mode <mode>  Validation mutation policy for live eval Atlas instances:
+                          active (default — learning arm learns), frozen (retrieval-only,
+                          no learned writes), or candidate.
+  --mutation-gating       Wire the playbook mutation/merge gating stack on the learning
+                          arm: binds a learning LLM provider (LLMJudgeReplay gate judge +
+                          reflective generator, no --distill needed) and runs maintenance
+                          after every dream so reflective-mutation / pareto-merge-proposing
+                          actually execute in short runs. Reported as mutationActivity in
+                          the JSON summary. DEFAULT ON whenever --mutation-mode is not
+                          frozen; disable with --no-mutation-gating.
+  --no-mutation-gating    Opt out of the default mutation/merge gating stack on the
+                          learning arm (extraction-only learning arm).
+  --live-replay-gating    Gate playbook mutations on the REAL task verifier for replayable
+                          experiences (re-execution outcome) instead of the haiku LLM-judge
+                          proxy, falling back to the LLM judge only when live replay isn't
+                          possible. Rollouts are budget-capped. Off by default (opt-in);
+                          requires --mutation-gating.
+  --procedural-memory-source <sqlite|filesystem>
+                          Procedural-memory authority for live eval Atlas instances.
+                          Use filesystem to validate migrated .atlas/playbooks/SKILL.md artifacts.
+  --procedural-memory-dir <p>
+                          Canonical SKILL.md playbook dir for live filesystem procedural memory.
+                          Relative paths are resolved against the current working directory.
+  --procedural-memory-strict
+                          Fail live eval startup on invalid filesystem procedural memory.
+  --meta-playbook-dir <p> Seed extractor-guidance meta-playbooks from a JSON file or directory.
+                          Repeat or comma-separate paths. Used only by learning-analysis agents.
+  report --swarmkit-learning --trace-dir <dir>
+                          Render swarmkit-eval ordered-learning report from ablation traces.
+  report --playbook-usage --trace-dir <dir>
+                          Render playbook surfacing/outcome audit from ablation traces.
+  report --playbook-calibration --trace-dir <dir>
+                          Render score/utility calibration from paired or single-arm traces.
+  report --candidate-decisions --trace-dir <dir>
+                          Audit every retrieved playbook candidate (incl. dropped): domain-fallback
+                          rate, rejected-candidate reasons. --true-positive <name> (repeatable)
+                          flags known true-positive playbooks dropped by the gate.
+  --cold <p> --warm <p>   (mock) cold/warm success probabilities
+  --batch-every <n>       Force batch learning every N tasks
+  --components <a,b,c>    (ablation) subset of components. Keys:
+                          ${ABLATION_COMPONENTS.map((c) => c.key).join(', ')}
+  --out <file>            Write report (.json → JSON, else Markdown). Live runs
+                          also checkpoint per-task partial results to
+                          <out>.checkpoint.json (deleted on successful finish)
+  --reuse-cold-from <f>   (live) Adopt cold-arm results from a prior run's
+                          result JSON or checkpoint file instead of re-running
+                          the arm. Requires same benchmark/tasks/model; rows
+                          are validated against this run's task stream
+  --resume                (live) Resume an aborted run from <out>.checkpoint.json
+                          (same --out): skip every already-completed (arm, seed,
+                          task) and reopen the persisted <out>.workdir/ learning
+                          dirs so accumulated memory carries forward. The final
+                          aggregated report is identical to an uninterrupted run.
+                          Requires --out; a no-op (fresh run) if no checkpoint
+                          exists. See scripts/run-eval-resilient.sh for a
+                          tmux-based auto-restart wrapper
+  --fail-fast             (live) One attempt per model call, no env_error
+                          episode retry — halves worst-case wall time on
+                          failure-heavy runs at some resilience cost
+  --adaptive-stopping     (live) Sequential stopping: after each seed, stop the
+                          run early if the learning-vs-cold question is already
+                          answered (threshold) or provably unanswerable within
+                          the remaining budget (futility)
+  --stopping-threshold <p> Posterior probability to declare early (default 0.98)
+  --difficulty-registry <f>
+                          (live) Per-family difficulty registry JSON. This run's
+                          per-task outcomes are ingested into it on completion;
+                          with --task-selection discriminative it also drives
+                          task sampling
+  --registry-db <path>    (live) SQLite registry (difficulty + lift cells) —
+                          the canonical backend. Ingests this run's difficulty
+                          rows AND guided/unguided lift rows (solver = --model,
+                          canonicalized) on completion; also usable as the
+                          --task-selection discriminative source. Lift
+                          subcommands default to eval-traces/registry.db when
+                          no registry flag is given
+  --task-selection <fixed|discriminative>
+                          (live) discriminative oversamples the benchmark pool
+                          and weights families toward the p≈0.5 band using the
+                          registry (advisory until a family has ≥3 runs).
+                          Requires --difficulty-registry or --registry-db.
+                          CAUTION: changes the estimand — never compare
+                          discriminative numbers against fixed-stream numbers
+  ingest-difficulty --registry <f> [--benchmark <n>] <artifacts...>
+                          Backfill the registry from existing result JSONs or
+                          checkpoint files
+  ingest-lift [--registry-db <db> | --registry <f.json>] [--benchmark <n>]
+              [--solver <model>] <artifacts...>
+                          Backfill the lift registry (guided = learning arm,
+                          unguided = cold arm, family = skillTag). Solver comes
+                          from a checkpoint's meta.model, else --solver is
+                          required (result JSONs carry no model). Cells never
+                          pool across solvers
+  lift-report [--registry-db <db> | --registry <f.json>] [--atlas-dir <dir>] [--json]
+                          "Where does memory help?" — per-(scope × family ×
+                          solver) guided/unguided counts, posterior Δ, 95% CI,
+                          and decided state (POSITIVE: CI lower bound > 0;
+                          NULL: CI upper bound < 0.05; else UNDECIDED).
+                          --atlas-dir merges a production instance DB's
+                          lift_cells (scope 'production')
+  registry migrate --from <file.json> --to <db>
+                          One-shot migration of a JSON registry (difficulty +
+                          lift) into a SQLite registry DB
+  --json                  Print JSON to stdout
+
+Exit codes (mock curve): 0 = learning helped, 2 = no improvement (CI gate).`);
+}
+//# sourceMappingURL=eval.js.map