cognitive-core 0.2.4 → 0.3.1

package/dist/eval/live/clbench.js

@@ -0,0 +1,291 @@
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createTask, createTrajectory, createStep, successOutcome, failureOutcome, } from '../../types/index.js';
+import { EnvBridge } from './env-bridge.js';
+import { runClaude } from './claude-cli.js';
+import { seededRng } from '../rng.js';
+export class ClBenchBenchmark {
+    name = 'clbench';
+    interactive = true;
+    preserveOrder = true;
+    playbookMinTrajectories = 3;
+    bridge = null;
+    maxSteps;
+    reinject;
+    historyWindow;
+    constructor(opts = {}) {
+        this.maxSteps = opts.maxSteps ?? 30;
+        this.reinject = opts.reinject ?? true;
+        this.historyWindow = opts.historyWindow ?? 10;
+    }
+    async setup() {
+        await this.getBridge();
+    }
+    async teardown() {
+        await this.bridge?.close();
+        this.bridge = null;
+    }
+    async getBridge() {
+        if (this.bridge)
+            return this.bridge;
+        const { command, args } = resolveBridgeCommand();
+        const bridge = new EnvBridge(command, args);
+        try {
+            await bridge.start();
+        }
+        catch (err) {
+            throw new Error(`CL-Bench bridge failed to start (${err.message}).\n` +
+                `Install the CL-Bench Python package and set CLBENCH_BRIDGE or\n` +
+                `CLBENCH_REPO. See docs/DESIGN-evaluation-benchmarks.md.`);
+        }
+        this.bridge = bridge;
+        return bridge;
+    }
+    async freshBridge() {
+        await this.bridge?.close();
+        this.bridge = null;
+        return this.getBridge();
+    }
+    /**
+     * Load conversations as tasks. Applies a seeded Fisher-Yates shuffle so
+     * different seeds produce different orderings (reproducible per seed).
+     */
+    async loadTasks(opts) {
+        const bridge = await this.getBridge();
+        const response = await bridge.send({ cmd: 'list' });
+        // Seeded shuffle so different seeds produce different orderings.
+        const convs = [...response.tasks];
+        const rng = seededRng('clbench-loadtasks', opts.seed);
+        for (let i = convs.length - 1; i > 0; i--) {
+            const j = Math.floor(rng() * (i + 1));
+            [convs[i], convs[j]] = [convs[j], convs[i]];
+        }
+        const tasks = [];
+        let index = 0;
+        for (const conv of convs.slice(0, opts.count)) {
+            tasks.push({
+                task: createTask({
+                    id: `clbench-${conv.id}`,
+                    domain: 'CRM',
+                    description: conv.description ??
+                        `CRM ${conv.category} conversation (${conv.complexity}, ${conv.num_turns} turns)`,
+                    metadata: {
+                        skillTag: conv.category,
+                        taskId: conv.id,
+                        category: conv.category,
+                        complexity: conv.complexity,
+                        numTurns: conv.num_turns,
+                        evalIndex: index,
+                    },
+                }),
+                skillTag: conv.category,
+                index,
+                coldProb: 0,
+                warmProb: 0,
+                family: conv.category,
+            });
+            index++;
+        }
+        return tasks;
+    }
+    verify(_task, _output) {
+        return { correct: false };
+    }
+    getBridgeInstance() {
+        return this.bridge;
+    }
+    delegateFactory = (_benchmark, opts) => {
+        return new ClBenchDelegate(this, {
+            model: opts?.model,
+            timeoutMs: opts?.timeoutMs,
+            provider: opts?.provider,
+            awsRegion: opts?.awsRegion,
+            cliRetries: opts?.cliRetries,
+            maxSteps: this.maxSteps,
+            reinject: this.reinject,
+            historyWindow: this.historyWindow,
+        });
+    };
+}
+export function createClBenchBenchmark(opts) {
+    return new ClBenchBenchmark(opts);
+}
+/**
+ * Multi-turn delegate for CL-Bench conversations.
+ *
+ * Each conversation is a sequence of CRM user requests. The agent must
+ * respond with JSON tool calls (e.g. {"tool_name": "create_contact",
+ * "arguments": {...}}). The bridge executes against MockCrmApi and returns
+ * the result plus the next user utterance.
+ *
+ * The delegate uses structured prompts with B2 re-injection to keep
+ * accumulated knowledge visible throughout the conversation.
+ */
+class ClBenchDelegate {
+    benchmark;
+    opts;
+    currentTask = null;
+    totalCostUsd = 0;
+    constructor(benchmark, opts) {
+        this.benchmark = benchmark;
+        this.opts = opts;
+    }
+    setCurrentTask(task) {
+        this.currentTask = task;
+    }
+    getTotalCostUsd() {
+        return this.totalCostUsd;
+    }
+    async execute(prompt, options) {
+        const task = this.currentTask;
+        const taskId = String(task?.metadata?.taskId ?? '');
+        const category = String(task?.metadata?.category ?? 'CRM');
+        const complexity = String(task?.metadata?.complexity ?? 'simple');
+        const knowledge = options.systemContext;
+        const bridge = await this.benchmark.freshBridge();
+        const resetResult = await bridge.send({
+            cmd: 'reset',
+            task: taskId,
+        });
+        const steps = [createStep({ action: 'reset', observation: resetResult.obs })];
+        const history = [];
+        let reward = 0;
+        let done = resetResult.done;
+        let obs = resetResult.obs;
+        let transcript = `[Reset] conversation=${taskId} (${category}, ${complexity})\n${resetResult.obs}`;
+        let lastMetrics;
+        const availableTools = resetResult.context?.available_tools ?? [];
+        for (let i = 0; i < this.opts.maxSteps && !done; i++) {
+            const stepPrompt = this.opts.reinject
+                ? buildClBenchPrompt({
+                    knowledge,
+                    instruction: prompt,
+                    category,
+                    complexity,
+                    history: history.slice(-this.opts.historyWindow),
+                    obs,
+                    availableTools,
+                    context: resetResult.context,
+                })
+                : i === 0
+                    ? [knowledge, prompt, `\nCurrent request:\n${obs}`].filter(Boolean).join('\n\n')
+                    : `Request:\n${obs}\n\nYour tool call (JSON):`;
+            const res = await runClaude(stepPrompt, {
+                model: this.opts.model,
+                timeoutMs: this.opts.timeoutMs,
+                provider: this.opts.provider,
+                awsRegion: this.opts.awsRegion,
+                retries: this.opts.cliRetries,
+            });
+            this.totalCostUsd += res.costUsd ?? 0;
+            const action = extractToolCall(res.text);
+            const stepResult = await bridge.send({
+                cmd: 'step',
+                action,
+            });
+            reward = stepResult.reward;
+            done = stepResult.done;
+            obs = stepResult.obs;
+            lastMetrics = stepResult.metrics;
+            history.push({ action, obs });
+            steps.push(createStep({ action, observation: obs }));
+            transcript += `\n> ${action}\n${obs}`;
+        }
+        const episodeResult = await bridge.send({
+            cmd: 'episode_complete',
+            task: taskId,
+        });
+        const success = reward >= 1 || episodeResult.gain_metric > 0.5;
+        const trajectory = createTrajectory({
+            task: task ?? createTask({ domain: 'CRM', description: taskId }),
+            steps,
+            outcome: success
+                ? successOutcome(transcript)
+                : failureOutcome(`CL-Bench conversation failed (gain=${episodeResult.gain_metric.toFixed(3)})`),
+            agentId: 'clbench-delegate',
+        });
+        return {
+            success,
+            output: transcript,
+            trajectory,
+            structured: {
+                gainMetric: episodeResult.gain_metric,
+                expertValidated: episodeResult.expert_validated,
+                conversationDetails: episodeResult.details,
+                turnMetrics: lastMetrics,
+                costUsd: this.totalCostUsd,
+            },
+        };
+    }
+}
+/**
+ * Structured per-step prompt for CL-Bench CRM conversations.
+ * Surfaces tool catalog, conversation context, and injected knowledge.
+ */
+export function buildClBenchPrompt(p) {
+    const sections = [
+        `You are a CRM assistant handling a ${p.complexity} ${p.category} conversation. ` +
+            'Respond with EXACTLY ONE JSON tool call: {"tool_name": "...", "arguments": {...}}',
+    ];
+    if (p.knowledge) {
+        sections.push(`## Relevant knowledge and past experience\n${p.knowledge}`);
+    }
+    if (p.availableTools && p.availableTools.length > 0) {
+        sections.push(`## Available CRM tools\n${p.availableTools.join(', ')}`);
+    }
+    if (p.context && Object.keys(p.context).length > 0) {
+        const contextStr = JSON.stringify({ ...p.context, available_tools: undefined }, null, 2).slice(0, 1500);
+        if (contextStr.length > 5) {
+            sections.push(`## Conversation context\n${contextStr}`);
+        }
+    }
+    sections.push(`## Task\n${p.instruction}`);
+    if (p.history.length) {
+        const lines = p.history.map((h) => `Tool call: ${h.action}\nResult: ${h.obs}`).join('\n\n');
+        sections.push(`## Conversation history (most recent last)\n${lines}`);
+    }
+    sections.push(`## Current user request\n${p.obs}\n\nYour tool call (JSON):`);
+    return sections.join('\n\n');
+}
+/**
+ * Extract a JSON tool call from the agent's response.
+ * Looks for JSON objects with tool_name/tool fields.
+ */
+function extractToolCall(text) {
+    const fenced = text.match(/```(?:json)?\s*\n([\s\S]*?)```/);
+    const jsonStr = fenced ? fenced[1].trim() : text.trim();
+    try {
+        const parsed = JSON.parse(jsonStr);
+        if (typeof parsed === 'object' &&
+            parsed !== null &&
+            ('tool_name' in parsed || 'tool' in parsed)) {
+            return JSON.stringify(parsed);
+        }
+    }
+    catch {
+        // Not valid JSON — try extracting from within the text
+    }
+    const jsonMatch = text.match(/\{[^{}]*"(?:tool_name|tool)"[^{}]*\}/);
+    if (jsonMatch) {
+        try {
+            JSON.parse(jsonMatch[0]);
+            return jsonMatch[0];
+        }
+        catch {
+            // Fall through
+        }
+    }
+    return JSON.stringify({ tool_name: '', arguments: {} });
+}
+function resolveBridgeCommand() {
+    const env = process.env.CLBENCH_BRIDGE;
+    if (env && env.trim()) {
+        const parts = env.trim().split(/\s+/);
+        return { command: parts[0], args: parts.slice(1) };
+    }
+    const here = dirname(fileURLToPath(import.meta.url));
+    const srcDir = here.includes('/dist/') ? here.replace('/dist/', '/src/') : here;
+    const script = join(srcDir, 'bridges', 'clbench_bridge.py');
+    return { command: process.env.PYTHON ?? 'python3', args: [script] };
+}
+//# sourceMappingURL=clbench.js.map

package/dist/eval/live/clbench.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clbench.js","sourceRoot":"","sources":["../../../src/eval/live/clbench.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,cAAc,GAEf,MAAM,sBAAsB,CAAC;AAK9B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAqEtC,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,SAAS,CAAC;IACjB,WAAW,GAAG,IAAI,CAAC;IACnB,aAAa,GAAG,IAAI,CAAC;IACrB,uBAAuB,GAAG,CAAC,CAAC;IAE7B,MAAM,GAAqB,IAAI,CAAC;IACvB,QAAQ,CAAS;IACjB,QAAQ,CAAU;IAClB,aAAa,CAAS;IAEvC,YACE,OAII,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QACtC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QACpC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,oCAAqC,GAAa,CAAC,OAAO,MAAM;gBAC9D,iEAAiE;gBACjE,yDAAyD,CAC5D,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,IAAqC;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAuC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QAE1F,iEAAiE;QACjE,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,GAAG,GAAG,SAAS,CAAC,mBAAmB,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACtD,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,KAAK,GAAe,EAAE,CAAC;QAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9C,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,UAAU,CAAC;oBACf,EAAE,EAAE,WAAW,IAAI,CAAC,EAAE,EAAE;oBACxB,MAAM,EAAE,KAAK;oBACb,WAAW,EACT,IAAI,CAAC,WAAW;wBAChB,OAAO,IAAI,CAAC,QAAQ,kBAAkB,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,SAAS,SAAS;oBACnF,QAAQ,EAAE;wBACR,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;wBAC3B,QAAQ,EAAE,IAAI,CAAC,SAAS;wBACxB,SAAS,EAAE,KAAK;qBACjB;iBACF,CAAC;gBACF,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK;gBACL,QAAQ,EAAE,CAAC;gBACX,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,IAAI,CAAC,QAAQ;aACtB,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;QACV,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,CAAC,KAAW,EAAE,OAAe;QACjC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,iBAAiB;QACf,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,eAAe,GAAG,CAChB,UAAiC,EACjC,IAMC,EACkB,EAAE;QACrB,OAAO,IAAI,eAAe,CAAC,IAAI,EAAE;YAC/B,KAAK,EAAE,IAAI,EAAE,KAAK;YAClB,SAAS,EAAE,IAAI,EAAE,SAAS;YAC1B,QAAQ,EAAE,IAAI,EAAE,QAAQ;YACxB,SAAS,EAAE,IAAI,EAAE,SAAS;YAC1B,UAAU,EAAE,IAAI,EAAE,UAAU;YAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC,CAAC;IACL,CAAC,CAAC;CACH;AAED,MAAM,UAAU,sBAAsB,CAAC,IAItC;IACC,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,eAAe;IAKA;IACA;IALX,WAAW,GAAgB,IAAI,CAAC;IAChC,YAAY,GAAG,CAAC,CAAC;IAEzB,YACmB,SAA2B,EAC3B,IAUhB;QAXgB,cAAS,GAAT,SAAS,CAAkB;QAC3B,SAAI,GAAJ,IAAI,CAUpB;IACA,CAAC;IAEJ,cAAc,CAAC,IAAU;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAA6B;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,IAAI,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC;QAExC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;QAElD,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAuB;YAC1D,GAAG,EAAE,OAAO;YACZ,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC9E,MAAM,OAAO,GAA2C,EAAE,CAAC;QAE3D,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;QAC5B,IAAI,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC;QAC1B,IAAI,UAAU,GAAG,wBAAwB,MAAM,KAAK,QAAQ,KAAK,UAAU,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;QACnG,IAAI,WAA+C,CAAC;QACpD,MAAM,cAAc,GAAI,WAAW,CAAC,OAAO,EAAE,eAA4B,IAAI,EAAE,CAAC;QAEhF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACrD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ;gBACnC,CAAC,CAAC,kBAAkB,CAAC;oBACjB,SAAS;oBACT,WAAW,EAAE,MAAM;oBACnB,QAAQ;oBACR,UAAU;oBACV,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC;oBAChD,GAAG;oBACH,cAAc;oBACd,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC;gBACJ,CAAC,CAAC,CAAC,KAAK,CAAC;oBACP,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;oBAChF,CAAC,CAAC,aAAa,GAAG,4BAA4B,CAAC;YAEnD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE;gBACtC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;gBACtB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;gBAC9B,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;gBAC5B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;gBAC9B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU;aAC9B,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,IAAI,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;YAEtC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,IAAI,CAAsB;gBACxD,GAAG,EAAE,MAAM;gBACX,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;YAC3B,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;YACvB,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC;YACrB,WAAW,GAAG,UAAU,CAAC,OAA6C,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACrD,UAAU,IAAI,OAAO,MAAM,KAAK,GAAG,EAAE,CAAC;QACxC,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,IAAI,CAAiC;YACtE,GAAG,EAAE,kBAAkB;YACvB,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,aAAa,CAAC,WAAW,GAAG,GAAG,CAAC;QAC/D,MAAM,UAAU,GAAG,gBAAgB,CAAC;YAClC,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;YAChE,KAAK;YACL,OAAO,EAAE,OAAO;gBACd,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC;gBAC5B,CAAC,CAAC,cAAc,CACZ,sCAAsC,aAAa,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC9E;YACL,OAAO,EAAE,kBAAkB;SAC5B,CAAC,CAAC;QAEH,OAAO;YACL,OAAO;YACP,MAAM,EAAE,UAAU;YAClB,UAAU;YACV,UAAU,EAAE;gBACV,UAAU,EAAE,aAAa,CAAC,WAAW;gBACrC,eAAe,EAAE,aAAa,CAAC,gBAAgB;gBAC/C,mBAAmB,EAAE,aAAa,CAAC,OAAO;gBAC1C,WAAW,EAAE,WAAW;gBACxB,OAAO,EAAE,IAAI,CAAC,YAAY;aAC3B;SACF,CAAC;IACJ,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,CASlC;IACC,MAAM,QAAQ,GAAa;QACzB,sCAAsC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,QAAQ,iBAAiB;YAC/E,mFAAmF;KACtF,CAAC;IAEF,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;QAChB,QAAQ,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpD,QAAQ,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAC5F,CAAC,EACD,IAAI,CACL,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,4BAA4B,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAE3C,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,MAAM,aAAa,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5F,QAAQ,CAAC,IAAI,CAAC,+CAA+C,KAAK,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,GAAG,4BAA4B,CAAC,CAAC;IAE7E,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAExD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IACE,OAAO,MAAM,KAAK,QAAQ;YAC1B,MAAM,KAAK,IAAI;YACf,CAAC,WAAW,IAAI,MAAM,IAAI,MAAM,IAAI,MAAM,CAAC,EAC3C,CAAC;YACD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,uDAAuD;IACzD,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACrE,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACzB,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,oBAAoB;IAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IACvC,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACrD,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,mBAAmB,CAAC,CAAC;IAC5D,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;AACtE,CAAC"}

package/dist/eval/live/cold-reuse.d.ts

@@ -0,0 +1,21 @@
+import type { TaskResult } from '../types.js';
+/** Per-seed cold-arm rows extracted from a prior run artifact. */
+export type ColdArmResultsBySeed = Record<number, TaskResult[]>;
+/**
+ * Load per-seed cold-arm results from a prior run artifact (final result JSON
+ * or checkpoint file). Throws when the artifact contains no cold-arm rows —
+ * silently returning nothing would let a mis-pointed flag quietly turn a
+ * two-arm comparison into garbage.
+ */
+export declare function loadColdArmResults(path: string): ColdArmResultsBySeed;
+/**
+ * Whether prior cold rows are adoptable for a stream: same episode count and
+ * same taskId at every position. A mismatch means the prior run used a
+ * different benchmark config — adopting its rows would compare arms across
+ * different task sets.
+ */
+export declare function coldRowsMatchStream(rows: TaskResult[], streamTaskIds: string[]): {
+    ok: boolean;
+    reason?: string;
+};
+//# sourceMappingURL=cold-reuse.d.ts.map

package/dist/eval/live/cold-reuse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cold-reuse.d.ts","sourceRoot":"","sources":["../../../src/eval/live/cold-reuse.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C,kEAAkE;AAClE,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;AAoBhE;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,CAoCrE;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,UAAU,EAAE,EAClB,aAAa,EAAE,MAAM,EAAE,GACtB;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAgBlC"}

package/dist/eval/live/cold-reuse.js

@@ -0,0 +1,102 @@
+/**
+ * Cold-arm reuse for seed-lift / config A/B runs.
+ *
+ * The no-learning (cold) arm is a property of {benchmark, model, harness
+ * version} only — it cannot see seeds, memory, or any config under test. So
+ * re-running it for every arm of an A/B is pure spend: ~half of a two-arm
+ * run's episodes measure a control that a prior run already measured.
+ *
+ * `loadColdArmResults` extracts per-seed cold-arm TaskResults from a prior
+ * run artifact so `LiveEvalRunner` can adopt them instead of re-running the
+ * arm. Two artifact shapes are supported:
+ *
+ *   1. A final result JSON written by `eval --out <file>.json`
+ *      (needs `noLearning.perSeed` — present since 2026-07-02).
+ *   2. A `<out>.checkpoint.json` partial file from an aborted run — episodes
+ *      already paid for remain reusable even when the run never finished.
+ *
+ * Validity is the caller's contract: the prior run must have used the same
+ * benchmark, task set, model, and harness version. The runner cross-checks
+ * taskIds per seed and refuses rows that don't match its own stream.
+ */
+import { readFileSync } from 'node:fs';
+const COLD_ARM_LABEL = 'no-learning';
+function toTaskResult(row) {
+    return {
+        index: row.index,
+        taskId: row.taskId,
+        skillTag: row.skillTag,
+        success: row.success,
+        status: row.status,
+        stepCount: row.stepCount ?? 0,
+        knowledgeSurfaced: row.knowledgeSurfaced ?? false,
+        priorSkillCoverage: row.priorSkillCoverage ?? 0,
+        wallTimeMs: row.wallTimeMs ?? 0,
+        ...(typeof row.costUsd === 'number' ? { costUsd: row.costUsd } : {}),
+        ...(typeof row.partialScore === 'number' ? { partialScore: row.partialScore } : {}),
+    };
+}
+/**
+ * Load per-seed cold-arm results from a prior run artifact (final result JSON
+ * or checkpoint file). Throws when the artifact contains no cold-arm rows —
+ * silently returning nothing would let a mis-pointed flag quietly turn a
+ * two-arm comparison into garbage.
+ */
+export function loadColdArmResults(path) {
+    const raw = JSON.parse(readFileSync(path, 'utf8'));
+    const bySeed = {};
+    if (raw.partial === true) {
+        // Checkpoint format: flat task rows stamped with arm + seed.
+        const tasks = raw.tasks ?? [];
+        for (const row of tasks) {
+            if (row.arm !== COLD_ARM_LABEL)
+                continue;
+            (bySeed[row.seed] ??= []).push(toTaskResult(row));
+        }
+        for (const rows of Object.values(bySeed)) {
+            rows.sort((a, b) => a.index - b.index);
+        }
+    }
+    else {
+        // Final result format: noLearning.perSeed aligned with noLearning.seeds.
+        const noLearning = raw.noLearning;
+        const seeds = noLearning?.seeds ?? [];
+        const perSeed = noLearning?.perSeed ?? [];
+        for (let i = 0; i < seeds.length; i++) {
+            const rows = perSeed[i];
+            if (rows && rows.length > 0) {
+                bySeed[seeds[i]] = rows.map(toTaskResult);
+            }
+        }
+    }
+    if (Object.keys(bySeed).length === 0) {
+        throw new Error(`No cold-arm (no-learning) task results found in ${path}. ` +
+            `Reusable artifacts are result JSONs with noLearning.perSeed ` +
+            `(written since 2026-07-02) or <out>.checkpoint.json partial files.`);
+    }
+    return bySeed;
+}
+/**
+ * Whether prior cold rows are adoptable for a stream: same episode count and
+ * same taskId at every position. A mismatch means the prior run used a
+ * different benchmark config — adopting its rows would compare arms across
+ * different task sets.
+ */
+export function coldRowsMatchStream(rows, streamTaskIds) {
+    if (rows.length !== streamTaskIds.length) {
+        return {
+            ok: false,
+            reason: `prior run has ${rows.length} cold tasks, this run has ${streamTaskIds.length}`,
+        };
+    }
+    for (let i = 0; i < rows.length; i++) {
+        if (rows[i].taskId !== streamTaskIds[i]) {
+            return {
+                ok: false,
+                reason: `task ${i} mismatch: prior=${rows[i].taskId} current=${streamTaskIds[i]}`,
+            };
+        }
+    }
+    return { ok: true };
+}
+//# sourceMappingURL=cold-reuse.js.map

package/dist/eval/live/cold-reuse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cold-reuse.js","sourceRoot":"","sources":["../../../src/eval/live/cold-reuse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAOvC,MAAM,cAAc,GAAG,aAAa,CAAC;AAErC,SAAS,YAAY,CAAC,GAA4B;IAChD,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAe;QAC1B,MAAM,EAAE,GAAG,CAAC,MAAgB;QAC5B,QAAQ,EAAE,GAAG,CAAC,QAAkB;QAChC,OAAO,EAAE,GAAG,CAAC,OAAkB;QAC/B,MAAM,EAAE,GAAG,CAAC,MAA8B;QAC1C,SAAS,EAAG,GAAG,CAAC,SAAoB,IAAI,CAAC;QACzC,iBAAiB,EAAG,GAAG,CAAC,iBAA6B,IAAI,KAAK;QAC9D,kBAAkB,EAAG,GAAG,CAAC,kBAA6B,IAAI,CAAC;QAC3D,UAAU,EAAG,GAAG,CAAC,UAAqB,IAAI,CAAC;QAC3C,GAAG,CAAC,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA4B,CAAC;IAC9E,MAAM,MAAM,GAAyB,EAAE,CAAC;IAExC,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACzB,6DAA6D;QAC7D,MAAM,KAAK,GAAI,GAAiC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC7D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,GAAG,KAAK,cAAc;gBAAE,SAAS;YACzC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAyC,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,yEAAyE;QACzE,MAAM,UAAU,GAAG,GAAG,CAAC,UACkD,CAAC;QAC1E,MAAM,KAAK,GAAG,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,UAAU,EAAE,OAAO,IAAI,EAAE,CAAC;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,mDAAmD,IAAI,IAAI;YACzD,8DAA8D;YAC9D,oEAAoE,CACvE,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CACjC,IAAkB,EAClB,aAAuB;IAEvB,IAAI,IAAI,CAAC,MAAM,KAAK,aAAa,CAAC,MAAM,EAAE,CAAC;QACzC,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,iBAAiB,IAAI,CAAC,MAAM,6BAA6B,aAAa,CAAC,MAAM,EAAE;SACxF,CAAC;IACJ,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;YACxC,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,QAAQ,CAAC,oBAAoB,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,YAAY,aAAa,CAAC,CAAC,CAAC,EAAE;aAClF,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC"}

package/dist/eval/live/difficulty-registry.d.ts

@@ -0,0 +1,68 @@
+import type { EvalTask, TaskResult } from '../types.js';
+/** Accumulated outcome counts for one task family (env_error rows excluded). */
+export interface FamilyDifficultyStats {
+    /** Episodes across all arms. */
+    attempts: number;
+    successes: number;
+    /** Cold-arm (no-learning) subset — the uncontaminated difficulty signal. */
+    coldAttempts: number;
+    coldSuccesses: number;
+    /** Distinct ingested runs that touched this family (advisory-gate input). */
+    runs: number;
+}
+export interface DifficultyRegistryFile {
+    version: 1;
+    updatedAt: string;
+    /** benchmark name → family (skillTag) → stats. */
+    benchmarks: Record<string, {
+        families: Record<string, FamilyDifficultyStats>;
+    }>;
+}
+/** One episode row extracted from a run artifact for ingestion. */
+export interface IngestRow {
+    family: string;
+    arm: string;
+    success: boolean;
+    status: TaskResult['status'];
+}
+export declare function emptyDifficultyRegistry(): DifficultyRegistryFile;
+/** Load a registry file, returning an empty registry when the file is absent. */
+export declare function loadDifficultyRegistry(path: string): DifficultyRegistryFile;
+/** Atomic write (temp + rename), matching the checkpoint writer's discipline. */
+export declare function saveDifficultyRegistry(path: string, registry: DifficultyRegistryFile): void;
+/**
+ * Fold one run's episode rows into the registry. `runs` is incremented once
+ * per family per call, so a call corresponds to one run artifact.
+ */
+export declare function ingestRunRows(registry: DifficultyRegistryFile, benchmark: string, rows: IngestRow[]): void;
+/**
+ * Extract ingestible rows from a run artifact: either a final result JSON
+ * (learning.perSeed + noLearning.perSeed) or a <out>.checkpoint.json partial.
+ * Returns the benchmark name found in the artifact alongside the rows.
+ */
+export declare function extractIngestRows(raw: Record<string, unknown>): {
+    benchmark: string | undefined;
+    rows: IngestRow[];
+};
+/**
+ * Posterior-mean success estimate for a family under a Beta(1,1) prior.
+ * Cold-arm counts drive the estimate once they are thick enough; otherwise
+ * pooled counts (learning contamination beats no data).
+ */
+export declare function estimateFamilySuccess(stats: FamilyDifficultyStats): number;
+/**
+ * Sampling weight for a family: 4·p̂(1−p̂), which is 1 at p̂=0.5 and falls
+ * toward 0 as the family saturates in either direction. Unknown or
+ * under-observed families (< minRuns ingested runs) get weight 1 — the
+ * registry is advisory until it has seen enough evidence.
+ */
+export declare function discriminativeWeight(stats: FamilyDifficultyStats | undefined, minRuns?: number): number;
+/**
+ * Weighted sampling without replacement (Efraimidis–Spirakis: key = u^(1/w),
+ * keep the `count` largest keys), deterministic per (registry, benchmark,
+ * seed). Selected tasks keep their original relative order — ordered
+ * benchmarks stay ordered, just with gaps — and are re-indexed 0..n-1 so
+ * positional aggregation (curves, checkpoint rows) stays consistent.
+ */
+export declare function selectDiscriminativeTasks(pool: EvalTask[], count: number, registry: DifficultyRegistryFile, benchmark: string, seed: number, minRuns?: number): EvalTask[];
+//# sourceMappingURL=difficulty-registry.d.ts.map

package/dist/eval/live/difficulty-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"difficulty-registry.d.ts","sourceRoot":"","sources":["../../../src/eval/live/difficulty-registry.ts"],"names":[],"mappings":"AAiCA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGxD,gFAAgF;AAChF,MAAM,WAAW,qBAAqB;IACpC,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,CAAC,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAA;KAAE,CAAC,CAAC;CACjF;AAED,mEAAmE;AACnE,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;CAC9B;AAUD,wBAAgB,uBAAuB,IAAI,sBAAsB,CAEhE;AAED,iFAAiF;AACjF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,sBAAsB,CAO3E;AAED,iFAAiF;AACjF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,sBAAsB,GAAG,IAAI,CAM3F;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,EAAE,GAChB,IAAI,CAuBN;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IAC/D,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,IAAI,EAAE,SAAS,EAAE,CAAC;CACnB,CA0BA;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,qBAAqB,GAAG,MAAM,CAK1E;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,qBAAqB,GAAG,SAAS,EACxC,OAAO,SAAmB,GACzB,MAAM,CAIR;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,QAAQ,EAAE,EAChB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,sBAAsB,EAChC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,OAAO,SAAmB,GACzB,QAAQ,EAAE,CAgBZ"}