cognitive-core 0.2.4 → 0.3.1

package/dist/eval/live/lifelongagent.js

@@ -0,0 +1,216 @@
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createTask, createTrajectory, createStep, successOutcome, failureOutcome, } from '../../types/index.js';
+import { EnvBridge } from './env-bridge.js';
+import { runClaude } from './claude-cli.js';
+import { seededRng } from '../rng.js';
+export class LifelongAgentBenchmark {
+    name = 'lifelongagent';
+    interactive = true;
+    preserveOrder = true;
+    playbookMinTrajectories = 3; // lifelong learning — extract & reuse playbooks
+    bridge = null;
+    maxSteps;
+    historyWindow;
+    constructor(opts = {}) {
+        this.maxSteps = opts.maxSteps ?? 20;
+        this.historyWindow = opts.historyWindow ?? 8;
+    }
+    async setup() {
+        await this.getBridge();
+    }
+    async teardown() {
+        await this.bridge?.close();
+        this.bridge = null;
+    }
+    async getBridge() {
+        if (this.bridge)
+            return this.bridge;
+        const { command, args } = resolveBridgeCommand();
+        const bridge = new EnvBridge(command, args);
+        try {
+            await bridge.start();
+        }
+        catch (err) {
+            throw new Error(`LifelongAgentBench bridge failed to start (${err.message}).\n` +
+                `Install the LifelongAgentBench package and set LIFELONGAGENT_BRIDGE\n` +
+                `or LIFELONGAGENT_REPO. See docs/DESIGN-evaluation-benchmarks.md.`);
+        }
+        this.bridge = bridge;
+        return bridge;
+    }
+    async freshBridge() {
+        await this.bridge?.close();
+        this.bridge = null;
+        return this.getBridge();
+    }
+    async loadTasks(opts) {
+        const bridge = await this.getBridge();
+        const response = await bridge.send({ cmd: 'list' });
+        // Seeded shuffle so different seeds produce different orderings.
+        const allTasks = [...response.tasks];
+        const rng = seededRng('lifelongagent-loadtasks', opts.seed);
+        for (let i = allTasks.length - 1; i > 0; i--) {
+            const j = Math.floor(rng() * (i + 1));
+            [allTasks[i], allTasks[j]] = [allTasks[j], allTasks[i]];
+        }
+        const selected = allTasks.slice(0, opts.count);
+        return selected.map((info, index) => ({
+            task: createTask({
+                id: `lifelong-${info.id}`,
+                domain: info.environment,
+                description: info.description,
+                metadata: {
+                    skillTag: info.environment,
+                    taskId: info.id,
+                    environment: info.environment,
+                    difficulty: info.difficulty,
+                    evalIndex: index,
+                },
+            }),
+            skillTag: info.environment,
+            index,
+            coldProb: 0,
+            warmProb: 0,
+            family: info.environment,
+            difficulty: info.difficulty,
+        }));
+    }
+    verify(_task, _output) {
+        return { correct: false };
+    }
+    delegateFactory = (_benchmark, opts) => {
+        return new LifelongAgentDelegate(this, {
+            model: opts?.model,
+            timeoutMs: opts?.timeoutMs,
+            maxSteps: this.maxSteps,
+            historyWindow: this.historyWindow,
+        });
+    };
+}
+export function createLifelongAgentBenchmark(opts) {
+    return new LifelongAgentBenchmark(opts);
+}
+/**
+ * Multi-turn delegate for LifelongAgentBench tasks.
+ *
+ * Environment-specific prompt formatting: DB tasks get SQL-focused prompts,
+ * OS tasks get shell-focused prompts, KG tasks get query-focused prompts.
+ */
+class LifelongAgentDelegate {
+    benchmark;
+    opts;
+    currentTask = null;
+    totalCostUsd = 0;
+    constructor(benchmark, opts) {
+        this.benchmark = benchmark;
+        this.opts = opts;
+    }
+    setCurrentTask(task) {
+        this.currentTask = task;
+    }
+    getTotalCostUsd() {
+        return this.totalCostUsd;
+    }
+    async execute(prompt, options) {
+        const task = this.currentTask;
+        const taskId = String(task?.metadata?.taskId ?? '');
+        const environment = task?.metadata?.environment ?? 'db';
+        const knowledge = options.systemContext;
+        const bridge = await this.benchmark.freshBridge();
+        const resetResult = await bridge.send({
+            cmd: 'reset',
+            task: taskId,
+        });
+        const steps = [createStep({ action: 'reset', observation: resetResult.obs })];
+        const history = [];
+        let reward = 0;
+        let done = resetResult.done;
+        let obs = resetResult.obs;
+        let transcript = `[Reset] env=${environment} task=${taskId}\n${resetResult.obs}`;
+        for (let i = 0; i < this.opts.maxSteps && !done; i++) {
+            const stepPrompt = buildLifelongPrompt({
+                knowledge,
+                instruction: prompt,
+                environment,
+                history: history.slice(-this.opts.historyWindow),
+                obs,
+            });
+            const res = await runClaude(stepPrompt, {
+                model: this.opts.model,
+                timeoutMs: this.opts.timeoutMs,
+            });
+            this.totalCostUsd += res.costUsd ?? 0;
+            const action = extractAction(res.text, environment);
+            const stepResult = await bridge.send({
+                cmd: 'step',
+                action,
+            });
+            reward = stepResult.reward;
+            done = stepResult.done;
+            obs = stepResult.obs;
+            history.push({ action, obs });
+            steps.push(createStep({ action, observation: obs }));
+            transcript += `\n> ${action}\n${obs}`;
+        }
+        const success = reward >= 1;
+        const trajectory = createTrajectory({
+            task: task ?? createTask({ domain: environment, description: taskId }),
+            steps,
+            outcome: success
+                ? successOutcome(transcript)
+                : failureOutcome(`LifelongAgentBench task failed (reward=${reward})`),
+            agentId: 'lifelongagent-delegate',
+        });
+        return {
+            success,
+            output: transcript,
+            trajectory,
+            structured: { reward, costUsd: this.totalCostUsd },
+        };
+    }
+}
+const ENV_DESCRIPTIONS = {
+    db: 'You are working with a database. Respond with SQL queries or database commands.',
+    os: 'You are working in an operating system shell. Respond with shell commands.',
+    kg: 'You are querying a knowledge graph. Respond with graph queries or commands.',
+};
+export function buildLifelongPrompt(p) {
+    const sections = [
+        ENV_DESCRIPTIONS[p.environment] + ' Respond with EXACTLY ONE command.',
+    ];
+    if (p.knowledge) {
+        sections.push(`## Relevant experience\n${p.knowledge}`);
+    }
+    sections.push(`## Task\n${p.instruction}`);
+    if (p.history.length) {
+        const lines = p.history.map((h) => `> ${h.action}\n${h.obs}`).join('\n');
+        sections.push(`## History (most recent last)\n${lines}`);
+    }
+    sections.push(`## Current observation\n${p.obs}\n\nYour command:`);
+    return sections.join('\n\n');
+}
+function extractAction(text, env) {
+    const fenced = text.match(/```(?:sql|bash|sh|sparql|cypher)?\n([\s\S]*?)```/);
+    if (fenced)
+        return fenced[1].trim();
+    const line = text
+        .split('\n')
+        .map((l) => l.trim())
+        .find((l) => l.length > 0 && !l.startsWith('#') && !l.startsWith('//'));
+    return (line ?? (env === 'db' ? 'SELECT 1' : 'echo ok'))
+        .replace(/^(action:|command:|>\s*)\s*/i, '')
+        .trim();
+}
+function resolveBridgeCommand() {
+    const env = process.env.LIFELONGAGENT_BRIDGE;
+    if (env && env.trim()) {
+        const parts = env.trim().split(/\s+/);
+        return { command: parts[0], args: parts.slice(1) };
+    }
+    const here = dirname(fileURLToPath(import.meta.url));
+    const srcDir = here.includes('/dist/') ? here.replace('/dist/', '/src/') : here;
+    const script = join(srcDir, 'bridges', 'lifelongagent_bridge.py');
+    return { command: process.env.PYTHON ?? 'python3', args: [script] };
+}
+//# sourceMappingURL=lifelongagent.js.map

package/dist/eval/live/lifelongagent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lifelongagent.js","sourceRoot":"","sources":["../../../src/eval/live/lifelongagent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,cAAc,GAEf,MAAM,sBAAsB,CAAC;AAK9B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAkCtC,MAAM,OAAO,sBAAsB;IACxB,IAAI,GAAG,eAAe,CAAC;IACvB,WAAW,GAAG,IAAI,CAAC;IACnB,aAAa,GAAG,IAAI,CAAC;IACrB,uBAAuB,GAAG,CAAC,CAAC,CAAC,gDAAgD;IAE9E,MAAM,GAAqB,IAAI,CAAC;IACvB,QAAQ,CAAS;IACjB,aAAa,CAAS;IAEvC,YAAY,OAAsD,EAAE;QAClE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QACpC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,8CAA+C,GAAa,CAAC,OAAO,MAAM;gBACxE,uEAAuE;gBACvE,kEAAkE,CACrE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAqC;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAgC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QAEnF,iEAAiE;QACjE,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,SAAS,CAAC,yBAAyB,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAE/C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,EAAE,UAAU,CAAC;gBACf,EAAE,EAAE,YAAY,IAAI,CAAC,EAAE,EAAE;gBACzB,MAAM,EAAE,IAAI,CAAC,WAAW;gBACxB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,QAAQ,EAAE;oBACR,QAAQ,EAAE,IAAI,CAAC,WAAW;oBAC1B,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,SAAS,EAAE,KAAK;iBACjB;aACF,CAAC;YACF,QAAQ,EAAE,IAAI,CAAC,WAAW;YAC1B,KAAK;YACL,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,IAAI,CAAC,WAAW;YACxB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,CAAC,KAAW,EAAE,OAAe;QACjC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,eAAe,GAAG,CAChB,UAAiC,EACjC,IAA6C,EAC1B,EAAE;QACrB,OAAO,IAAI,qBAAqB,CAAC,IAAI,EAAE;YACrC,KAAK,EAAE,IAAI,EAAE,KAAK;YAClB,SAAS,EAAE,IAAI,EAAE,SAAS;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC,CAAC;IACL,CAAC,CAAC;CACH;AAED,MAAM,UAAU,4BAA4B,CAAC,IAG5C;IACC,OAAO,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;GAKG;AACH,MAAM,qBAAqB;IAKN;IACA;IALX,WAAW,GAAgB,IAAI,CAAC;IAChC,YAAY,GAAG,CAAC,CAAC;IAEzB,YACmB,SAAiC,EACjC,IAKhB;QANgB,cAAS,GAAT,SAAS,CAAwB;QACjC,SAAI,GAAJ,IAAI,CAKpB;IACA,CAAC;IAEJ,cAAc,CAAC,IAAU;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAA6B;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QACpD,MAAM,WAAW,GAAI,IAAI,EAAE,QAAQ,EAAE,WAA+B,IAAI,IAAI,CAAC;QAC7E,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC;QAExC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;QAElD,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAiC;YACpE,GAAG,EAAE,OAAO;YACZ,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC9E,MAAM,OAAO,GAA2C,EAAE,CAAC;QAE3D,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;QAC5B,IAAI,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC;QAC1B,IAAI,UAAU,GAAG,eAAe,WAAW,SAAS,MAAM,KAAK,WAAW,CAAC,GAAG,EAAE,CAAC;QAEjF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrD,MAAM,UAAU,GAAG,mBAAmB,CAAC;gBACrC,SAAS;gBACT,WAAW,EAAE,MAAM;gBACnB,WAAW;gBACX,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC;gBAChD,GAAG;aACJ,CAAC,CAAC;YAEH,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE;gBACtC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;gBACtB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;aAC/B,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,IAAI,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;YAEtC,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YACpD,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,IAAI,CAAuB;gBACzD,GAAG,EAAE,MAAM;gBACX,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;YAC3B,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;YACvB,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACrD,UAAU,IAAI,OAAO,MAAM,KAAK,GAAG,EAAE,CAAC;QACxC,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,CAAC;QAC5B,MAAM,UAAU,GAAG,gBAAgB,CAAC;YAClC,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;YACtE,KAAK;YACL,OAAO,EAAE,OAAO;gBACd,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC;gBAC5B,CAAC,CAAC,cAAc,CAAC,0CAA0C,MAAM,GAAG,CAAC;YACvE,OAAO,EAAE,wBAAwB;SAClC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO;YACP,MAAM,EAAE,UAAU;YAClB,UAAU;YACV,UAAU,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE;SACnD,CAAC;IACJ,CAAC;CACF;AAED,MAAM,gBAAgB,GAAoC;IACxD,EAAE,EAAE,iFAAiF;IACrF,EAAE,EAAE,4EAA4E;IAChF,EAAE,EAAE,6EAA6E;CAClF,CAAC;AAEF,MAAM,UAAU,mBAAmB,CAAC,CAMnC;IACC,MAAM,QAAQ,GAAa;QACzB,gBAAgB,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,oCAAoC;KACvE,CAAC;IAEF,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;QAChB,QAAQ,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAE3C,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzE,QAAQ,CAAC,IAAI,CAAC,kCAAkC,KAAK,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,GAAG,mBAAmB,CAAC,CAAC;IAEnE,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,GAAoB;IACvD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;IAC9E,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI;SACd,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,OAAO,CAAC,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;SACrD,OAAO,CAAC,8BAA8B,EAAE,EAAE,CAAC;SAC3C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,oBAAoB;IAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAC7C,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACrD,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,yBAAyB,CAAC,CAAC;IAClE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;AACtE,CAAC"}

package/dist/eval/live/lift-registry.d.ts

@@ -0,0 +1,128 @@
+/**
+ * Per-(scope × family × solver) lift registry — the treatment-effect
+ * companion to the difficulty registry (DESIGN-meta-learning-validation.md
+ * §10, lift-registry subsection).
+ *
+ * Design rationale (preserve when refactoring):
+ *   - The DIFFICULTY registry answers "how hard is family X" (base rate).
+ *   - The LIFT registry answers "does memory injection change outcomes on
+ *     family X with solver Y" (treatment effect).
+ *   - Cells must NEVER pool across solvers (a playbook that lifts haiku may
+ *     be noise for opus) or across eval/production populations (different
+ *     task distributions and exposure mechanisms). `scope` is the benchmark
+ *     name for eval rows or 'production' for dream-time production rollups.
+ *
+ * Cell math: guided vs unguided Beta(1,1)-binomial posteriors, compared via
+ * the same seeded Monte Carlo used by the stopping controller
+ * (`posteriorDelta` in stopping.ts — imported, never duplicated).
+ *
+ * Recency: cells carry a `runs` counter (ingested artifacts for eval scopes,
+ * rolled-up trajectories for production).
+ * TODO(recency): EMA/window decay of stale counts is explicitly deferred —
+ * see DESIGN-meta-learning-validation.md §10 (lift registry).
+ */
+import type { TaskResult } from '../types.js';
+/** Accumulated guided/unguided outcome counts for one (family, solver) cell. */
+export interface LiftCellStats {
+    /** Episodes with memory injection (learning arm / detected exposure). */
+    guidedAttempts: number;
+    guidedSuccesses: number;
+    /** Episodes without memory injection (cold arm / no detected exposure). */
+    unguidedAttempts: number;
+    unguidedSuccesses: number;
+    /**
+     * Distinct ingested run artifacts (eval scopes) or rolled-up trajectories
+     * (production scope) that touched this cell. Recency raw material — decay
+     * is deferred (see module TODO).
+     */
+    runs: number;
+}
+/**
+ * In-memory lift registry: scope ('production' or a benchmark name) →
+ * family (skillTag / task domain) → solver (canonical name) → stats.
+ */
+export interface LiftRegistryData {
+    scopes: Record<string, {
+        families: Record<string, Record<string, LiftCellStats>>;
+    }>;
+}
+/** One episode row extracted from a run artifact for lift ingestion. */
+export interface LiftIngestRow {
+    family: string;
+    /** True when the episode ran with memory injection (learning arm). */
+    guided: boolean;
+    success: boolean;
+    status: TaskResult['status'];
+}
+export declare function emptyLiftRegistry(): LiftRegistryData;
+export declare function emptyLiftCell(): LiftCellStats;
+/**
+ * Fold one run's episode rows into the registry under (scope, solver).
+ * env_error rows are excluded (matching the difficulty registry and the
+ * run aggregates); `runs` is incremented once per touched cell per call, so
+ * a call corresponds to one run artifact.
+ */
+export declare function ingestLiftRows(registry: LiftRegistryData, scope: string, solver: string, rows: LiftIngestRow[]): void;
+/**
+ * Extract lift-ingestible rows from a run artifact — the same two artifact
+ * shapes the difficulty registry accepts: a final result JSON
+ * (learning.perSeed + noLearning.perSeed) or a <out>.checkpoint.json partial.
+ * Learning-arm rows map to guided, cold-arm rows to unguided; family is the
+ * row's skillTag. Returns the benchmark name and the solver model found in
+ * the artifact (checkpoint `meta.model`; final result JSONs carry no model,
+ * so callers must supply --solver for those).
+ */
+export declare function extractLiftIngestRows(raw: Record<string, unknown>): {
+    benchmark: string | undefined;
+    solver: string | undefined;
+    rows: LiftIngestRow[];
+};
+/**
+ * Merge `from` into `into` (mutating `into`) by summing cell counters.
+ * Used by the lift report to combine the eval-side registry store with a
+ * production instance DB. Scopes normally don't overlap (eval scopes are
+ * benchmark names, production is 'production'), so summing is only a
+ * safety behavior for accidental overlap.
+ */
+export declare function mergeLiftRegistries(into: LiftRegistryData, from: LiftRegistryData): void;
+export interface LiftEstimateConfig {
+    /**
+     * Practical-equivalence half-width for the 'null' decision. A cell is
+     * 'null' when the CI upper bound sits below this — the data are confident
+     * there is at most a negligible positive effect (which includes confidently
+     * NEGATIVE cells: "memory does not help here"). Default 0.05 (5pp).
+     */
+    nullEpsilon?: number;
+    /**
+     * Minimum attempts required in EACH arm before any decision is made.
+     * Below this the Beta(1,1) prior dominates and the CI is meaningless
+     * prior width. Default 3.
+     */
+    minAttemptsPerArm?: number;
+    /** Monte Carlo samples for the posterior delta (default 4000). */
+    mcSamples?: number;
+    /** RNG seed for deterministic estimates (default 1). */
+    seed?: number;
+}
+export interface LiftEstimate {
+    /** Posterior-mean delta = E[pGuided] − E[pUnguided] under Beta(1,1) priors. */
+    delta: number;
+    /** 95% central credible interval on the delta (seeded Monte Carlo). */
+    ci95: [number, number];
+    /**
+     * Decision semantics:
+     *   'positive'  — CI lower bound > 0: memory injection credibly helps.
+     *   'null'      — CI upper bound < nullEpsilon: at most a negligible
+     *                 positive effect (covers confidently-harmful cells too).
+     *   'undecided' — everything else, including thin cells
+     *                 (< minAttemptsPerArm in either arm).
+     */
+    decided: 'undecided' | 'positive' | 'null';
+}
+/**
+ * Estimate the treatment effect of memory injection for one cell.
+ * Reuses the stopping controller's seeded Beta Monte Carlo (`posteriorDelta`)
+ * with guided as the "learning" arm and unguided as the "cold" arm.
+ */
+export declare function estimateLift(cell: LiftCellStats, config?: LiftEstimateConfig): LiftEstimate;
+//# sourceMappingURL=lift-registry.d.ts.map

package/dist/eval/live/lift-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lift-registry.d.ts","sourceRoot":"","sources":["../../../src/eval/live/lift-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,gFAAgF;AAChF,MAAM,WAAW,aAAa;IAC5B,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,2EAA2E;IAC3E,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAA;KAAE,CAAC,CAAC;CACrF;AAED,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;CAC9B;AAID,wBAAgB,iBAAiB,IAAI,gBAAgB,CAEpD;AAED,wBAAgB,aAAa,IAAI,aAAa,CAQ7C;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,aAAa,EAAE,GACpB,IAAI,CAmBN;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IACnE,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,IAAI,EAAE,aAAa,EAAE,CAAC;CACvB,CAoCA;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,gBAAgB,GAAG,IAAI,CAexF;AAMD,MAAM,WAAW,kBAAkB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AASD,MAAM,WAAW,YAAY;IAC3B,+EAA+E;IAC/E,KAAK,EAAE,MAAM,CAAC;IACd,uEAAuE;IACvE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvB;;;;;;;OAOG;IACH,OAAO,EAAE,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;CAC5C;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,GAAE,kBAAuB,GAAG,YAAY,CA2B/F"}

package/dist/eval/live/lift-registry.js

@@ -0,0 +1,147 @@
+import { posteriorDelta } from './stopping.js';
+const COLD_ARM_LABEL = 'no-learning';
+export function emptyLiftRegistry() {
+    return { scopes: {} };
+}
+export function emptyLiftCell() {
+    return {
+        guidedAttempts: 0,
+        guidedSuccesses: 0,
+        unguidedAttempts: 0,
+        unguidedSuccesses: 0,
+        runs: 0,
+    };
+}
+/**
+ * Fold one run's episode rows into the registry under (scope, solver).
+ * env_error rows are excluded (matching the difficulty registry and the
+ * run aggregates); `runs` is incremented once per touched cell per call, so
+ * a call corresponds to one run artifact.
+ */
+export function ingestLiftRows(registry, scope, solver, rows) {
+    const scopeEntry = (registry.scopes[scope] ??= { families: {} });
+    const touched = new Set();
+    for (const row of rows) {
+        if (row.status === 'env_error')
+            continue;
+        const solvers = (scopeEntry.families[row.family] ??= {});
+        const cell = (solvers[solver] ??= emptyLiftCell());
+        if (row.guided) {
+            cell.guidedAttempts++;
+            if (row.success)
+                cell.guidedSuccesses++;
+        }
+        else {
+            cell.unguidedAttempts++;
+            if (row.success)
+                cell.unguidedSuccesses++;
+        }
+        touched.add(row.family);
+    }
+    for (const family of touched) {
+        scopeEntry.families[family][solver].runs++;
+    }
+}
+/**
+ * Extract lift-ingestible rows from a run artifact — the same two artifact
+ * shapes the difficulty registry accepts: a final result JSON
+ * (learning.perSeed + noLearning.perSeed) or a <out>.checkpoint.json partial.
+ * Learning-arm rows map to guided, cold-arm rows to unguided; family is the
+ * row's skillTag. Returns the benchmark name and the solver model found in
+ * the artifact (checkpoint `meta.model`; final result JSONs carry no model,
+ * so callers must supply --solver for those).
+ */
+export function extractLiftIngestRows(raw) {
+    const rows = [];
+    if (raw.partial === true) {
+        const file = raw;
+        for (const t of file.tasks ?? []) {
+            rows.push({
+                family: t.skillTag,
+                guided: t.arm !== COLD_ARM_LABEL,
+                success: t.success,
+                status: t.status,
+            });
+        }
+        const meta = (raw.meta ?? {});
+        return {
+            benchmark: typeof meta.benchmark === 'string' ? meta.benchmark : undefined,
+            solver: typeof meta.model === 'string' ? meta.model : undefined,
+            rows,
+        };
+    }
+    const pullArm = (guided, condition) => {
+        const perSeed = condition?.perSeed ?? [];
+        for (const seedRows of perSeed) {
+            for (const r of seedRows) {
+                rows.push({ family: r.skillTag, guided, success: r.success, status: r.status });
+            }
+        }
+    };
+    pullArm(true, raw.learning);
+    pullArm(false, raw.noLearning);
+    return {
+        benchmark: typeof raw.benchmarkName === 'string' ? raw.benchmarkName : undefined,
+        solver: undefined,
+        rows,
+    };
+}
+/**
+ * Merge `from` into `into` (mutating `into`) by summing cell counters.
+ * Used by the lift report to combine the eval-side registry store with a
+ * production instance DB. Scopes normally don't overlap (eval scopes are
+ * benchmark names, production is 'production'), so summing is only a
+ * safety behavior for accidental overlap.
+ */
+export function mergeLiftRegistries(into, from) {
+    for (const [scope, { families }] of Object.entries(from.scopes)) {
+        const scopeEntry = (into.scopes[scope] ??= { families: {} });
+        for (const [family, solvers] of Object.entries(families)) {
+            const intoSolvers = (scopeEntry.families[family] ??= {});
+            for (const [solver, cell] of Object.entries(solvers)) {
+                const target = (intoSolvers[solver] ??= emptyLiftCell());
+                target.guidedAttempts += cell.guidedAttempts;
+                target.guidedSuccesses += cell.guidedSuccesses;
+                target.unguidedAttempts += cell.unguidedAttempts;
+                target.unguidedSuccesses += cell.unguidedSuccesses;
+                target.runs += cell.runs;
+            }
+        }
+    }
+}
+const DEFAULT_ESTIMATE_CONFIG = {
+    nullEpsilon: 0.05,
+    minAttemptsPerArm: 3,
+    mcSamples: 4000,
+    seed: 1,
+};
+/**
+ * Estimate the treatment effect of memory injection for one cell.
+ * Reuses the stopping controller's seeded Beta Monte Carlo (`posteriorDelta`)
+ * with guided as the "learning" arm and unguided as the "cold" arm.
+ */
+export function estimateLift(cell, config = {}) {
+    const cfg = { ...DEFAULT_ESTIMATE_CONFIG, ...config };
+    const guided = {
+        successes: cell.guidedSuccesses,
+        failures: cell.guidedAttempts - cell.guidedSuccesses,
+    };
+    const unguided = {
+        successes: cell.unguidedSuccesses,
+        failures: cell.unguidedAttempts - cell.unguidedSuccesses,
+    };
+    // Deterministic posterior-mean delta (no MC noise on the point estimate).
+    const delta = (guided.successes + 1) / (cell.guidedAttempts + 2) -
+        (unguided.successes + 1) / (cell.unguidedAttempts + 2);
+    const { ci } = posteriorDelta(guided, unguided, cfg.mcSamples, cfg.seed);
+    let decided = 'undecided';
+    const thin = cell.guidedAttempts < cfg.minAttemptsPerArm || cell.unguidedAttempts < cfg.minAttemptsPerArm;
+    if (!thin) {
+        if (ci[0] > 0)
+            decided = 'positive';
+        else if (ci[1] < cfg.nullEpsilon)
+            decided = 'null';
+    }
+    return { delta, ci95: ci, decided };
+}
+//# sourceMappingURL=lift-registry.js.map

package/dist/eval/live/lift-registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lift-registry.js","sourceRoot":"","sources":["../../../src/eval/live/lift-registry.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAmC/C,MAAM,cAAc,GAAG,aAAa,CAAC;AAErC,MAAM,UAAU,iBAAiB;IAC/B,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,cAAc,EAAE,CAAC;QACjB,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,iBAAiB,EAAE,CAAC;QACpB,IAAI,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC5B,QAA0B,EAC1B,KAAa,EACb,MAAc,EACd,IAAqB;IAErB,MAAM,UAAU,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACzC,MAAM,OAAO,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,aAAa,EAAE,CAAC,CAAC;QACnD,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YACf,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,GAAG,CAAC,OAAO;gBAAE,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,OAAO;gBAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC5C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7C,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAA4B;IAKhE,MAAM,IAAI,GAAoB,EAAE,CAAC;IAEjC,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,GAAgC,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC;gBACR,MAAM,EAAE,CAAC,CAAC,QAAQ;gBAClB,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,cAAc;gBAChC,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB,CAAC,CAAC;QACL,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;QACzD,OAAO;YACL,SAAS,EAAE,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YAC1E,MAAM,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YAC/D,IAAI;SACL,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,MAAe,EAAE,SAAkB,EAAQ,EAAE;QAC5D,MAAM,OAAO,GAAI,SAAsD,EAAE,OAAO,IAAI,EAAE,CAAC;QACvF,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC5B,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,OAAO;QACL,SAAS,EAAE,OAAO,GAAG,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QAChF,MAAM,EAAE,SAAS;QACjB,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAsB,EAAE,IAAsB;IAChF,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAChE,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC7D,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzD,MAAM,WAAW,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YACzD,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrD,MAAM,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,aAAa,EAAE,CAAC,CAAC;gBACzD,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC;gBAC7C,MAAM,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC;gBAC/C,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,CAAC;gBACjD,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,CAAC;gBACnD,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AA0BD,MAAM,uBAAuB,GAAiC;IAC5D,WAAW,EAAE,IAAI;IACjB,iBAAiB,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI;IACf,IAAI,EAAE,CAAC;CACR,CAAC;AAkBF;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAmB,EAAE,SAA6B,EAAE;IAC/E,MAAM,GAAG,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAAC;IACtD,MAAM,MAAM,GAAG;QACb,SAAS,EAAE,IAAI,CAAC,eAAe;QAC/B,QAAQ,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe;KACrD,CAAC;IACF,MAAM,QAAQ,GAAG;QACf,SAAS,EAAE,IAAI,CAAC,iBAAiB;QACjC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,iBAAiB;KACzD,CAAC;IAEF,0EAA0E;IAC1E,MAAM,KAAK,GACT,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QAClD,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAEzD,MAAM,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAEzE,IAAI,OAAO,GAA4B,WAAW,CAAC;IACnD,MAAM,IAAI,GACR,IAAI,CAAC,cAAc,GAAG,GAAG,CAAC,iBAAiB,IAAI,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC,iBAAiB,CAAC;IAC/F,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,UAAU,CAAC;aAC/B,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,WAAW;YAAE,OAAO,GAAG,MAAM,CAAC;IACrD,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC;AACtC,CAAC"}

package/dist/eval/live/live-benchmark.d.ts

@@ -0,0 +1,84 @@
+import type { Task } from '../../types/index.js';
+import type { EvalTask } from '../types.js';
+/**
+ * A live benchmark: a real, deterministically-verifiable task set that a real
+ * agent solves. Unlike the synthetic mock stream, success here is ground truth.
+ *
+ * Two flavors:
+ *   - **single-turn** (`interactive` falsy): the agent gets one prompt and
+ *     returns one answer (e.g. GSM8K). Driven by the ClaudeCliDelegate.
+ *   - **interactive** (`interactive: true`): the agent loops with an external
+ *     environment over many steps (e.g. ALFWorld). Driven by a multi-turn
+ *     delegate that owns the observation→action loop.
+ */
+export interface LiveBenchmark {
+    /** Stable name (CLI selector). */
+    readonly name: string;
+    /** Whether tasks require a multi-turn environment loop. */
+    readonly interactive: boolean;
+    /**
+     * Max concurrent episodes this benchmark's environment tolerates. Benchmarks
+     * with a shared/fragile in-process bridge (e.g. ALFWorld's TextWorld bridge,
+     * which exhausts multiprocessing semaphores when run N-wide) set this to 1.
+     * Docker-isolated benchmarks (each task in its own container) can leave it
+     * undefined (unbounded). The runner clamps `--concurrency` to this.
+     */
+    readonly maxConcurrency?: number;
+    /**
+     * Enable playbook extraction for this benchmark by setting the minimum number
+     * of similar trajectories required to mint a playbook. Skill-learning
+     * benchmarks (cumulative / lifelong) set a small value (e.g. 3) so the mid-run
+     * dream extracts playbooks that later tasks can reuse. Leave undefined to
+     * disable extraction entirely (the default for one-shot benchmarks like
+     * GSM8K/ALFWorld, where the learning signal is experience injection only).
+     * The eval CLI wires a separate workspace analysis backend by default for
+     * benchmarks that set this field; callers can opt out with
+     * `--no-agentic-playbooks`.
+     */
+    readonly playbookMinTrajectories?: number;
+    /**
+     * Load an ordered task slice. `seed` permutes/samples the pool so multiple
+     * runs can vary ordering; `count` caps the size. Gold answers live in
+     * `task.metadata` for the verifier to read.
+     */
+    loadTasks(opts: {
+        count: number;
+        seed: number;
+    }): Promise<EvalTask[]>;
+    /**
+     * Deterministically verify a single-turn answer against ground truth.
+     * `output` is the agent's raw final text. Returns whether it's correct plus
+     * the normalized value that was extracted (for debugging/reporting).
+     *
+     * Interactive benchmarks score from environment reward instead and may leave
+     * this as a no-op.
+     */
+    verify(task: Task, output: string): {
+        correct: boolean;
+        extracted?: string;
+    };
+}
+/**
+ * Verdict shape returned by a benchmark verifier (mirrors verify()'s return).
+ */
+export interface BenchmarkVerdict {
+    correct: boolean;
+    extracted?: string;
+}
+/**
+ * Extended benchmark interface for benchmarks needing lifecycle hooks
+ * beyond simple load+verify (e.g. Docker setup/teardown, strict ordering).
+ */
+export interface ExtendedLiveBenchmark extends LiveBenchmark {
+    /** Called once before the task stream starts. Build Docker images, etc. */
+    setup?(): Promise<void>;
+    /** Called once after the entire stream completes. Cleanup containers, etc. */
+    teardown?(): Promise<void>;
+    /**
+     * Whether task ordering must be preserved (not shuffled by seed).
+     * When true, loadTasks uses the seed for family/subset selection only,
+     * not for reordering tasks within families.
+     */
+    readonly preserveOrder?: boolean;
+}
+//# sourceMappingURL=live-benchmark.d.ts.map

package/dist/eval/live/live-benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"live-benchmark.d.ts","sourceRoot":"","sources":["../../../src/eval/live/live-benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C;;;;;;;;;;GAUG;AACH,MAAM,WAAW,aAAa;IAC5B,kCAAkC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2DAA2D;IAC3D,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B;;;;;;OAMG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAE1C;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAEtE;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,GAAG;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9E;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IAC1D,2EAA2E;IAC3E,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B;;;;OAIG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC"}

package/dist/eval/live/live-benchmark.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=live-benchmark.js.map

package/dist/eval/live/live-benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"live-benchmark.js","sourceRoot":"","sources":["../../../src/eval/live/live-benchmark.ts"],"names":[],"mappings":""}