@nexus-cortex/cli 4.26.0

package/dist/commands/autoresearch/bench.js

@@ -0,0 +1,123 @@
+/**
+ * `cortex autoresearch bench` — run a task set through the harness, GRADE each
+ * output with the task's verifier, and write REAL scored records to
+ * router-matrix.jsonl. This is the grader the decision layer was missing:
+ * `evaluate` (the gate) only produces a meaningful keep/discard once arms carry
+ * real qualitativeScores, which this command supplies.
+ *
+ * One invocation benches ONE harness build (records auto-stamped with its git
+ * SHA, or --harness-ref). The swarm orchestrator runs it in the base worktree
+ * and the candidate worktree, then calls `evaluate --base … --candidate …`.
+ */
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { ModelRouterMatrix, runBench, parseTaskSet, ResearchBacklog, } from '@nexus-cortex/core';
+import { ThemeManager } from '../../themes/ThemeManager.js';
+import { findProjectRoot } from '../config/utils.js';
+import { serverRunner } from './harnessProcess.js';
+import { commandRunner } from './commandRunner.js';
+/** Load tasks from a file (array or single object) or a directory of *.json. */
+function loadTasks(taskSetPath) {
+    const st = statSync(taskSetPath);
+    const files = st.isDirectory()
+        ? readdirSync(taskSetPath).filter(f => f.endsWith('.json')).map(f => join(taskSetPath, f))
+        : [taskSetPath];
+    const tasks = [];
+    for (const f of files) {
+        const raw = JSON.parse(readFileSync(f, 'utf8'));
+        tasks.push(...parseTaskSet(raw, f));
+    }
+    return tasks;
+}
+export async function autoResearchBench(options) {
+    const theme = ThemeManager.getTheme();
+    const projectRoot = findProjectRoot();
+    if (!options.taskSet) {
+        console.error(theme.colors.error('Error: --task-set is required'));
+        process.exit(1);
+    }
+    if (!options.experimentTag) {
+        console.error(theme.colors.error('Error: --experiment-tag is required'));
+        process.exit(1);
+    }
+    const split = options.split ?? 'train';
+    if (split !== 'train' && split !== 'holdout') {
+        console.error(theme.colors.error("Error: --split must be 'train' or 'holdout'"));
+        process.exit(1);
+    }
+    try {
+        const tasks = loadTasks(options.taskSet);
+        if (tasks.length === 0) {
+            console.error(theme.colors.error(`Error: no tasks found in ${options.taskSet}`));
+            process.exit(1);
+        }
+        const matrix = new ModelRouterMatrix(projectRoot);
+        const model = options.model ?? process.env.DEFAULT_MODEL_ID;
+        const log = (m) => { if (!options.json)
+            console.log(theme.colors.muted(` ${m}`)); };
+        let runner;
+        let source;
+        if (options.runCmd) {
+            // Non-cortex command target: optionally build once, then grade a shell command per task.
+            const cwd = options.cwd ? resolve(options.cwd) : projectRoot;
+            const acceptExit = (options.acceptExit ?? '0').split(',').map(s => Number(s.trim())).filter(n => Number.isFinite(n));
+            if (options.buildCmd) {
+                log(`Building target: ${options.buildCmd}  (cwd ${cwd})`);
+                const b = spawnSync('sh', ['-c', options.buildCmd], { cwd, stdio: options.json ? 'ignore' : 'inherit' });
+                if (b.status !== 0) {
+                    console.error(theme.colors.error(`Error: build command failed (exit ${b.status})`));
+                    process.exit(1);
+                }
+            }
+            runner = commandRunner({ cwd, template: options.runCmd, acceptExitCodes: acceptExit, log });
+            source = `cmd "${options.runCmd}" (cwd ${cwd})`;
+        }
+        else {
+            const serverUrl = options.serverUrl ?? process.env.CORTEX_SERVER_URL ?? 'http://localhost:4000';
+            runner = serverRunner(serverUrl, model);
+            source = serverUrl;
+        }
+        if (!options.json) {
+            console.log();
+            console.log(theme.colors.muted(` Benching ${tasks.length} task(s) × ${options.runs ?? 2} run(s) via ${source} [${split}]  tag=${options.experimentTag}`));
+        }
+        const summary = await runBench(tasks, runner, matrix, {
+            experimentTag: options.experimentTag,
+            runs: options.runs ? Number(options.runs) : undefined,
+            split,
+            modelId: model,
+            benchmarkSource: options.benchmarkSource,
+            harnessRef: options.harnessRef,
+            temperature: options.temperature !== undefined && Number.isFinite(Number(options.temperature)) ? Number(options.temperature) : undefined,
+            strategy: options.strategy,
+            backlog: options.seedBacklog === false ? undefined : new ResearchBacklog(projectRoot),
+            discoveredRound: options.experimentTag,
+            discoveredRef: options.harnessRef,
+            onRun: options.json ? undefined : (info) => {
+                const mark = info.pass ? theme.colors.success('[OK]') : theme.colors.error('[FAIL]');
+                console.log(theme.colors.muted(` ${mark} ${info.taskId} run ${info.run}: ${info.qualitativeScore}`));
+            },
+        });
+        if (options.json) {
+            console.log(JSON.stringify(summary, null, 2));
+            return;
+        }
+        console.log();
+        for (const t of summary.tasks) {
+            console.log(` ${theme.colors.highlight(t.taskId.padEnd(28))} ${theme.colors.secondary(t.taskType)}  mean ${t.meanScore}  pass ${Math.round(t.passRate * 100)}%`);
+        }
+        console.log();
+        console.log(theme.colors.muted(` ${summary.totalRuns} run(s) recorded → ${projectRoot}/.cortex/router-matrix.jsonl  (harnessRef ${summary.harnessRef ?? 'auto'})`));
+        if (summary.seededDeficiencies > 0) {
+            console.log(theme.colors.muted(` ${summary.seededDeficiencies} deficiency(ies) seeded → ${projectRoot}/.cortex/research-backlog.jsonl  (ResearchBacklog list / next)`));
+        }
+        console.log(theme.colors.muted(` Next: cortex autoresearch evaluate --experiment-tag ${options.experimentTag} --base <ref> --candidate <ref> --branch <wt>`));
+        console.log();
+    }
+    catch (error) {
+        console.error(theme.colors.error(`Error: ${error.message}`));
+        process.exit(1);
+    }
+}
+//# sourceMappingURL=bench.js.map

package/dist/commands/autoresearch/bench.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bench.js","sourceRoot":"","sources":["../../../src/commands/autoresearch/bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EACL,iBAAiB,EACjB,QAAQ,EACR,YAAY,EACZ,eAAe,GAGhB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAiCnD,gFAAgF;AAChF,SAAS,SAAS,CAAC,WAAmB;IACpC,MAAM,EAAE,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE;QAC5B,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC1F,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAClB,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAiC;IACvE,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,EAAE,CAAC;IACtC,MAAM,WAAW,GAAG,eAAe,EAAE,CAAC;IAEtC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAAC,CAAC;IAC9G,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAAC,CAAC;IAE1H,MAAM,KAAK,GAAI,OAAO,CAAC,KAAyC,IAAI,OAAO,CAAC;IAC5E,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC7C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;QACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,OAAQ,CAAC,CAAC;QAC1C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;QAE9H,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAClD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;QAC5D,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5F,IAAI,MAAqB,CAAC;QAC1B,IAAI,MAAc,CAAC;QACnB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,yFAAyF;YACzF,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;YAC7D,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACrH,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrB,GAAG,CAAC,oBAAoB,OAAO,CAAC,QAAQ,UAAU,GAAG,GAAG,CAAC,CAAC;gBAC1D,MAAM,CAAC,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;gBACzG,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,CAAC;YAC/H,CAAC;YACD,MAAM,GAAG,aAAa,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;YAC5F,MAAM,GAAG,QAAQ,OAAO,CAAC,MAAM,UAAU,GAAG,GAAG,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,uBAAuB,CAAC;YAChG,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,GAAG,SAAS,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,KAAK,CAAC,MAAM,cAAc,OAAO,CAAC,IAAI,IAAI,CAAC,eAAe,MAAM,KAAK,KAAK,UAAU,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;QAC5J,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;YACpD,aAAa,EAAE,OAAO,CAAC,aAAc;YACrC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YACrD,KAAK;YACL,OAAO,EAAE,KAAK;YACd,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YACxI,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,OAAO,EAAE,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,WAAW,CAAC;YACrF,eAAe,EAAE,OAAO,CAAC,aAAa;YACtC,aAAa,EAAE,OAAO,CAAC,UAAU;YACjC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE;gBACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,QAAQ,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;YACvG,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,SAAS,UAAU,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACnK,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,SAAS,sBAAsB,WAAW,6CAA6C,OAAO,CAAC,UAAU,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC;QACpK,IAAI,OAAO,CAAC,kBAAkB,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,kBAAkB,6BAA6B,WAAW,gEAAgE,CAAC,CAAC,CAAC;QAC1K,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,wDAAwD,OAAO,CAAC,aAAa,+CAA+C,CAAC,CAAC,CAAC;QAC9J,OAAO,CAAC,GAAG,EAAE,CAAC;IAEhB,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}

package/dist/commands/autoresearch/commandRunner.d.ts

@@ -0,0 +1,35 @@
+import type { HarnessRunner } from '@nexus-cortex/core';
+import type { ExperimentTarget, PreparedArm, PrepareArmOptions } from './harnessProcess.js';
+export interface CommandRunnerOptions {
+    /** Working directory the command runs in (e.g. the candidate worktree). */
+    cwd: string;
+    /** Command template, e.g. `./eval.sh {prompt}` or `python eval.py --case {prompt}`.
+     *  If it contains no `{prompt}`/`{case}` placeholder, the prompt is appended as a
+     *  single quoted argument. */
+    template: string;
+    /** Exit codes whose stdout is accepted for grading. Default `[0]`. */
+    acceptExitCodes?: number[];
+    /** Per-run hard timeout in ms. Default 120000. */
+    timeoutMs?: number;
+    /** Progress/diagnostic sink (stderr + nonzero-exit notices). */
+    log?: (message: string) => void;
+}
+export declare function commandRunner(opts: CommandRunnerOptions): HarnessRunner;
+/**
+ * CommandTarget — the non-cortex `ExperimentTarget`. An optional one-shot build command,
+ * then grade a shell command per task via `commandRunner`. Nothing to serve, nothing to
+ * tear down — so `cortex autoresearch experiment` can run base-vs-candidate on any project
+ * (a library, CLI, test suite, backtest) through the same statistical gate as the harness.
+ */
+export declare class CommandTarget implements ExperimentTarget {
+    private readonly cfg;
+    readonly kind = "command";
+    constructor(cfg: {
+        template: string;
+        buildCmd?: string;
+        acceptExitCodes?: number[];
+        timeoutMs?: number;
+    });
+    prepare(dir: string, opts: PrepareArmOptions): Promise<PreparedArm>;
+}
+//# sourceMappingURL=commandRunner.d.ts.map

package/dist/commands/autoresearch/commandRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"commandRunner.d.ts","sourceRoot":"","sources":["../../../src/commands/autoresearch/commandRunner.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,aAAa,EAAoB,MAAM,oBAAoB,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE5F,MAAM,WAAW,oBAAoB;IACnC,2EAA2E;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ;;kCAE8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACjC;AAOD,wBAAgB,aAAa,CAAC,IAAI,EAAE,oBAAoB,GAAG,aAAa,CAoCvE;AAED;;;;;GAKG;AACH,qBAAa,aAAc,YAAW,gBAAgB;IAGlD,OAAO,CAAC,QAAQ,CAAC,GAAG;IAFtB,QAAQ,CAAC,IAAI,aAAa;gBAEP,GAAG,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE;IAGzG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC;CAe1E"}

package/dist/commands/autoresearch/commandRunner.js

@@ -0,0 +1,91 @@
+/**
+ * CommandRunner — a `HarnessRunner` that grades a SHELL COMMAND instead of an LLM
+ * endpoint. This is the non-cortex experiment path: run `template` (with `{prompt}` /
+ * `{case}` substituted) per task in `cwd`, capture stdout, and hand it to the task's
+ * verifier (typically a `numeric` verifier that extracts a metric). It lets the
+ * auto-research loop measure a library / CLI / test suite / backtest — anything with a
+ * build + run + metric — through the same statistical gate as the cortex harness.
+ *
+ * Exit-code contract: stdout is graded only when the exit code is in `acceptExitCodes`
+ * (default `[0]`); otherwise `text=''`, so every verifier fails — a crashed run is not a
+ * valid measurement, so it fails the bench and seeds the backlog. stderr (and the exit
+ * code) are surfaced via `log`, never graded.
+ *
+ * The substituted `{prompt}` value is single-quote-escaped before it reaches the shell,
+ * so a task prompt cannot inject shell syntax. The `template` itself is operator-supplied
+ * (the experiment spec) and trusted.
+ */
+import { spawn, spawnSync } from 'node:child_process';
+/** POSIX single-quote escape: wrap in '…', and close/escape/reopen any embedded quote. */
+function shQuote(s) {
+    return `'${s.replace(/'/g, `'\\''`)}'`;
+}
+export function commandRunner(opts) {
+    const accept = opts.acceptExitCodes ?? [0];
+    const timeoutMs = opts.timeoutMs ?? 120_000;
+    return {
+        run(prompt) {
+            const cmd = /\{prompt\}|\{case\}/.test(opts.template)
+                ? opts.template.replace(/\{prompt\}|\{case\}/g, shQuote(prompt))
+                : `${opts.template} ${shQuote(prompt)}`;
+            const start = Date.now();
+            return new Promise((resolve) => {
+                const proc = spawn('sh', ['-c', cmd], { cwd: opts.cwd, stdio: ['ignore', 'pipe', 'pipe'] });
+                let out = '';
+                let err = '';
+                let timedOut = false;
+                const timer = setTimeout(() => { timedOut = true; try {
+                    proc.kill('SIGKILL');
+                }
+                catch { /* already gone */ } }, timeoutMs);
+                proc.stdout.on('data', (d) => { out += d.toString(); });
+                proc.stderr.on('data', (d) => { err += d.toString(); });
+                const done = (text, latencyMs) => ({ text, modelId: 'command', inputTokens: 0, outputTokens: 0, toolCallCount: 0, latencyMs });
+                proc.on('close', (code) => {
+                    clearTimeout(timer);
+                    const ok = !timedOut && code != null && accept.includes(code);
+                    if (!ok) {
+                        const reason = timedOut ? `timeout after ${timeoutMs}ms` : `exit ${code}`;
+                        opts.log?.(`[command ${reason}] ${err.trim().slice(-300)}`);
+                    }
+                    resolve(done(ok ? out : '', Date.now() - start));
+                });
+                proc.on('error', (e) => {
+                    clearTimeout(timer);
+                    opts.log?.(`[command error] ${e.message}`);
+                    resolve(done('', Date.now() - start));
+                });
+            });
+        },
+    };
+}
+/**
+ * CommandTarget — the non-cortex `ExperimentTarget`. An optional one-shot build command,
+ * then grade a shell command per task via `commandRunner`. Nothing to serve, nothing to
+ * tear down — so `cortex autoresearch experiment` can run base-vs-candidate on any project
+ * (a library, CLI, test suite, backtest) through the same statistical gate as the harness.
+ */
+export class CommandTarget {
+    cfg;
+    kind = 'command';
+    constructor(cfg) {
+        this.cfg = cfg;
+    }
+    async prepare(dir, opts) {
+        if (opts.build && this.cfg.buildCmd) {
+            opts.log(`build: ${this.cfg.buildCmd}  (cwd ${dir})`);
+            const b = spawnSync('sh', ['-c', this.cfg.buildCmd], { cwd: dir, stdio: ['ignore', 'ignore', 'inherit'] });
+            if (b.status !== 0)
+                throw new Error(`build command failed (exit ${b.status}) in ${dir}`);
+        }
+        const runner = commandRunner({
+            cwd: dir,
+            template: this.cfg.template,
+            acceptExitCodes: this.cfg.acceptExitCodes,
+            timeoutMs: this.cfg.timeoutMs,
+            log: opts.log,
+        });
+        return { runner, stop: () => { } };
+    }
+}
+//# sourceMappingURL=commandRunner.js.map

package/dist/commands/autoresearch/commandRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"commandRunner.js","sourceRoot":"","sources":["../../../src/commands/autoresearch/commandRunner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAmBtD,0FAA0F;AAC1F,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAA0B;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC;IAC5C,OAAO;QACL,GAAG,CAAC,MAAc;YAChB,MAAM,GAAG,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACnD,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,sBAAsB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;gBAChE,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,OAAO,IAAI,OAAO,CAAmB,CAAC,OAAO,EAAE,EAAE;gBAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC5F,IAAI,GAAG,GAAG,EAAE,CAAC;gBACb,IAAI,GAAG,GAAG,EAAE,CAAC;gBACb,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC;oBAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;gBAC3H,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxD,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,SAAiB,EAAoB,EAAE,CACjE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC/F,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,MAAM,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBAC9D,IAAI,CAAC,EAAE,EAAE,CAAC;wBACR,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,iBAAiB,SAAS,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;wBAC1E,IAAI,CAAC,GAAG,EAAE,CAAC,YAAY,MAAM,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC9D,CAAC;oBACD,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;gBACnD,CAAC,CAAC,CAAC;gBACH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;oBACrB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBAC3C,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;gBACxC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IAGL;IAFV,IAAI,GAAG,SAAS,CAAC;IAC1B,YACmB,GAA4F;QAA5F,QAAG,GAAH,GAAG,CAAyF;IAC5G,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,IAAuB;QAChD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,QAAQ,UAAU,GAAG,GAAG,CAAC,CAAC;YACtD,MAAM,CAAC,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;YAC3G,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAC;QAC3F,CAAC;QACD,MAAM,MAAM,GAAG,aAAa,CAAC;YAC3B,GAAG,EAAE,GAAG;YACR,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ;YAC3B,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe;YACzC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS;YAC7B,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAA8B,CAAC,EAAE,CAAC;IAChE,CAAC;CACF"}

package/dist/commands/autoresearch/evaluate.d.ts

@@ -0,0 +1,18 @@
+export interface AutoResearchEvaluateOptions {
+    experimentTag?: string;
+    base?: string;
+    candidate?: string;
+    branch?: string;
+    deficiencyId?: string;
+    benchmarkSource?: string;
+    modelId?: string;
+    nFamily?: string;
+    alpha?: string;
+    seed?: string;
+    epsilon?: string;
+    minRuns?: string;
+    verifyHoldout?: boolean;
+    json?: boolean;
+}
+export declare function autoResearchEvaluate(options: AutoResearchEvaluateOptions): Promise<void>;
+//# sourceMappingURL=evaluate.d.ts.map

package/dist/commands/autoresearch/evaluate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../../src/commands/autoresearch/evaluate.ts"],"names":[],"mappings":"AA0BA,MAAM,WAAW,2BAA2B;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAQD,wBAAsB,oBAAoB,CAAC,OAAO,EAAE,2BAA2B,GAAG,OAAO,CAAC,IAAI,CAAC,CAoG9F"}

package/dist/commands/autoresearch/evaluate.js

@@ -0,0 +1,117 @@
+/**
+ * `cortex autoresearch evaluate` — headless invocation of the keep/discard gate.
+ *
+ * This is the entry point a swarm member (or the cortex-bench flow) calls AFTER
+ * it has recorded base-version and candidate-version benchmark runs to the
+ * matrix (router-matrix.jsonl) under the same experimentTag. It runs the full
+ * decision pipeline (regressionScan → Monte-Carlo gate → ledger.decide) and
+ * writes the audited verdict to `.cortex/experiments.jsonl`.
+ *
+ * The output JSONL is THE integration boundary: the nexus Layer-3 STDB module
+ * ingests it (header → `experiment`, results[] → `experiment_task_result`).
+ * Nothing downstream re-derives the statistics — the decision is final here.
+ *
+ * Overfitting guard: keep/discard reads split='train' records; `--verify-holdout`
+ * additionally runs the held-out gate (a candidate is only merge-eligible when
+ * kept-on-train AND verified-on-holdout).
+ */
+import { ModelRouterMatrix, ExperimentLedger, evaluateAutoResearchExperiment, verifyOnHoldout, } from '@nexus-cortex/core';
+import { ThemeManager } from '../../themes/ThemeManager.js';
+import { findProjectRoot } from '../config/utils.js';
+function numOrUndef(v) {
+    if (v === undefined)
+        return undefined;
+    const n = Number(v);
+    return Number.isFinite(n) ? n : undefined;
+}
+export async function autoResearchEvaluate(options) {
+    const theme = ThemeManager.getTheme();
+    const projectRoot = findProjectRoot();
+    const required = [
+        ['experimentTag', '--experiment-tag'],
+        ['base', '--base'],
+        ['candidate', '--candidate'],
+        ['branch', '--branch'],
+    ];
+    const missing = required.filter(([k]) => !options[k]).map(([, flag]) => flag);
+    if (missing.length > 0) {
+        console.error(theme.colors.error(`Error: missing required option(s): ${missing.join(', ')}`));
+        process.exit(1);
+    }
+    try {
+        const matrix = new ModelRouterMatrix(projectRoot);
+        const ledger = new ExperimentLedger(projectRoot);
+        const gate = {
+            alpha: numOrUndef(options.alpha),
+            seed: numOrUndef(options.seed),
+            minRunsPerArm: numOrUndef(options.minRuns),
+        };
+        const result = evaluateAutoResearchExperiment(matrix, ledger, {
+            experimentTag: options.experimentTag,
+            baseRef: options.base,
+            candidateRef: options.candidate,
+            branch: options.branch,
+            deficiencyId: options.deficiencyId,
+            benchmarkSource: options.benchmarkSource,
+            modelId: options.modelId,
+            nFamilyExperiments: numOrUndef(options.nFamily) ?? 1,
+            gate,
+            epsilon: numOrUndef(options.epsilon),
+        });
+        let holdout = null;
+        if (options.verifyHoldout) {
+            holdout = verifyOnHoldout(matrix, {
+                baseRef: options.base,
+                candidateRef: options.candidate,
+                benchmarkSource: options.benchmarkSource,
+                modelId: options.modelId,
+                nFamilyExperiments: numOrUndef(options.nFamily) ?? 1,
+                gate,
+                epsilon: numOrUndef(options.epsilon),
+            });
+        }
+        // Merge-eligibility: kept on train AND (if checked) verified on holdout.
+        const mergeEligible = result.verdict.decision === 'keep' &&
+            result.verdict.fwerAdjusted === true &&
+            (!options.verifyHoldout || holdout?.decision === 'keep');
+        if (options.json) {
+            console.log(JSON.stringify({
+                record: result.record,
+                verdict: result.verdict,
+                regressedTasks: result.regressedTasks,
+                holdoutVerdict: holdout,
+                mergeEligible,
+            }, null, 2));
+            return;
+        }
+        const v = result.verdict;
+        const decColor = v.decision === 'keep' ? theme.colors.success
+            : v.decision === 'discard' ? theme.colors.error
+                : theme.colors.muted;
+        console.log();
+        console.log(` ${theme.colors.highlight('Experiment')}   ${options.experimentTag}  (${options.base} → ${options.candidate})`);
+        console.log(` ${theme.colors.highlight('Decision')}     ${decColor(v.decision.toUpperCase())}`);
+        console.log(` ${theme.colors.highlight('Effect')}       ${v.effect >= 0 ? '+' : ''}${v.effect}  (95% CI [${v.ciLow ?? '—'}, ${v.ciHigh ?? '—'}])`);
+        console.log(` ${theme.colors.highlight('p-value')}      ${v.pValue ?? '—'}  vs alpha_adj ${v.alphaAdjusted ?? '—'}  (N=${numOrUndef(options.nFamily) ?? 1}, FWER ${v.fwerAdjusted ? 'on' : 'off'})`);
+        console.log(` ${theme.colors.highlight('Runs/Tasks')}   ${v.nRuns} runs over ${v.nTasks} task(s)`);
+        if (result.regressedTasks.length > 0) {
+            console.log(` ${theme.colors.error('Regressions')}  ${result.regressedTasks.length} task(s): ${result.regressedTasks.join(', ')}`);
+        }
+        if (options.verifyHoldout) {
+            const hd = holdout
+                ? `${holdout.decision.toUpperCase()} (effect ${holdout.effect >= 0 ? '+' : ''}${holdout.effect}, CI [${holdout.ciLow ?? '—'}, ${holdout.ciHigh ?? '—'}])`
+                : theme.colors.muted('no held-out evidence — unverifiable');
+            console.log(` ${theme.colors.highlight('Holdout')}      ${hd}`);
+        }
+        console.log(` ${theme.colors.highlight('Mergeable')}    ${mergeEligible ? theme.colors.success('YES') : theme.colors.muted('no')}`);
+        console.log(theme.colors.muted(` Recorded → ${projectRoot}/.cortex/experiments.jsonl`));
+        console.log();
+        console.log(theme.colors.muted(` ${v.reason}`));
+        console.log();
+    }
+    catch (error) {
+        console.error(theme.colors.error(`Error: ${error.message}`));
+        process.exit(1);
+    }
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/commands/autoresearch/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../../src/commands/autoresearch/evaluate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,8BAA8B,EAC9B,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAmBrD,SAAS,UAAU,CAAC,CAAqB;IACvC,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACtC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACpB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC5C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,OAAoC;IAC7E,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,EAAE,CAAC;IACtC,MAAM,WAAW,GAAG,eAAe,EAAE,CAAC;IAEtC,MAAM,QAAQ,GAAuD;QACnE,CAAC,eAAe,EAAE,kBAAkB,CAAC;QACrC,CAAC,MAAM,EAAE,QAAQ,CAAC;QAClB,CAAC,WAAW,EAAE,aAAa,CAAC;QAC5B,CAAC,QAAQ,EAAE,UAAU,CAAC;KACvB,CAAC;IACF,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IAC9E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAEjD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;YAChC,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC;YAC9B,aAAa,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3C,CAAC;QAEF,MAAM,MAAM,GAAG,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE;YAC5D,aAAa,EAAE,OAAO,CAAC,aAAc;YACrC,OAAO,EAAE,OAAO,CAAC,IAAK;YACtB,YAAY,EAAE,OAAO,CAAC,SAAU;YAChC,MAAM,EAAE,OAAO,CAAC,MAAO;YACvB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,kBAAkB,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;YACpD,IAAI;YACJ,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC;SACrC,CAAC,CAAC;QAEH,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,OAAO,GAAG,eAAe,CAAC,MAAM,EAAE;gBAChC,OAAO,EAAE,OAAO,CAAC,IAAK;gBACtB,YAAY,EAAE,OAAO,CAAC,SAAU;gBAChC,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,kBAAkB,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;gBACpD,IAAI;gBACJ,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC;aACrC,CAAC,CAAC;QACL,CAAC;QAED,yEAAyE;QACzE,MAAM,aAAa,GACjB,MAAM,CAAC,OAAO,CAAC,QAAQ,KAAK,MAAM;YAClC,MAAM,CAAC,OAAO,CAAC,YAAY,KAAK,IAAI;YACpC,CAAC,CAAC,OAAO,CAAC,aAAa,IAAI,OAAO,EAAE,QAAQ,KAAK,MAAM,CAAC,CAAC;QAE3D,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,cAAc,EAAE,OAAO;gBACvB,aAAa;aACd,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACb,OAAO;QACT,CAAC;QAED,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;QACzB,MAAM,QAAQ,GACZ,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO;YAC5C,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK;gBAC/C,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;QAEvB,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,MAAM,OAAO,CAAC,aAAa,MAAM,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAC7H,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;QAChG,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,cAAc,CAAC,CAAC,KAAK,IAAI,GAAG,KAAK,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC;QACnJ,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,GAAG,kBAAkB,CAAC,CAAC,aAAa,IAAI,GAAG,QAAQ,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;QACrM,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,KAAK,cAAc,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC;QACnG,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,MAAM,CAAC,cAAc,CAAC,MAAM,aAAa,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrI,CAAC;QACD,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,OAAO;gBAChB,CAAC,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,YAAY,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,SAAS,OAAO,CAAC,KAAK,IAAI,GAAG,KAAK,OAAO,CAAC,MAAM,IAAI,GAAG,IAAI;gBACzJ,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YAC9D,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,WAAW,4BAA4B,CAAC,CAAC,CAAC;QACxF,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEhB,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}

package/dist/commands/autoresearch/experiment.d.ts

@@ -0,0 +1,38 @@
+export interface AutoResearchExperimentOptions {
+    experimentTag?: string;
+    candidateDir?: string;
+    baseDir?: string;
+    taskSet?: string;
+    holdoutSet?: string;
+    branch?: string;
+    nFamily?: string;
+    runs?: string;
+    model?: string;
+    deficiencyId?: string;
+    benchmarkSource?: string;
+    baseRef?: string;
+    candidateRef?: string;
+    buildBase?: boolean;
+    noBuild?: boolean;
+    basePort?: string;
+    candidatePort?: string;
+    cortexDir?: string;
+    seed?: string;
+    alpha?: string;
+    epsilon?: string;
+    minRuns?: string;
+    json?: boolean;
+    /** Non-cortex target: grade a shell command per task (with --build-cmd/--accept-exit)
+     *  instead of building+serving a cortex server. Both arms use the same command; the
+     *  base/candidate difference is the worktree the command runs in. */
+    runCmd?: string;
+    buildCmd?: string;
+    acceptExit?: string;
+    /** Effectiveness-arm labels recorded on both base + candidate records (shared dispatch
+     *  config; the experiment isolates the harness-version variable). Fall back to the
+     *  CORTEX_SUBAGENT_TEMPERATURE / CORTEX_ARM_STRATEGY env stamp when omitted. */
+    temperature?: string;
+    strategy?: string;
+}
+export declare function autoResearchExperiment(options: AutoResearchExperimentOptions): Promise<void>;
+//# sourceMappingURL=experiment.d.ts.map

package/dist/commands/autoresearch/experiment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../../src/commands/autoresearch/experiment.ts"],"names":[],"mappings":"AA+BA,MAAM,WAAW,6BAA6B;IAC5C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;yEAEqE;IACrE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;oFAEgF;IAChF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAkBD,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,6BAA6B,GAAG,OAAO,CAAC,IAAI,CAAC,CAwHlG"}

package/dist/commands/autoresearch/experiment.js

@@ -0,0 +1,168 @@
+/**
+ * `cortex autoresearch experiment` — the full single-experiment lifecycle.
+ *
+ * Builds the candidate (and optionally base) checkout, serves each on its own
+ * isolated port, benches both arms (train + optional holdout) into ONE shared
+ * `.cortex` store, runs the keep/discard gate + held-out verification, and emits
+ * the audited verdict + the JSONL artifact a downstream ingest consumes.
+ *
+ * This is the piece that owns the "two builds, not one relabel" correctness: each
+ * arm is served by a server BUILT FROM ITS OWN CODE, so the comparison is real.
+ * Servers run with MODEL_ROUTER_RECORD off; only the bench's graded records land
+ * in the shared store. Teardown is guaranteed in `finally`.
+ *
+ * Scope: this is a HARNESS-CODE experiment (base build vs candidate build,
+ * compared by git SHA). Model/config experiments (same code, different --model)
+ * use the lower-level `bench` + `evaluate` directly.
+ */
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, basename } from 'node:path';
+import { ModelRouterMatrix, ExperimentLedger, runExperiment, parseTaskSet, } from '@nexus-cortex/core';
+import { ThemeManager } from '../../themes/ThemeManager.js';
+import { findProjectRoot } from '../config/utils.js';
+import { freePort, gitShortSha, CortexTarget } from './harnessProcess.js';
+import { CommandTarget } from './commandRunner.js';
+function num(v) {
+    if (v === undefined)
+        return undefined;
+    const n = Number(v);
+    return Number.isFinite(n) ? n : undefined;
+}
+function loadTasks(p) {
+    const st = statSync(p);
+    const files = st.isDirectory()
+        ? readdirSync(p).filter(f => f.endsWith('.json')).map(f => join(p, f))
+        : [p];
+    const out = [];
+    for (const f of files)
+        out.push(...parseTaskSet(JSON.parse(readFileSync(f, 'utf8')), f));
+    return out;
+}
+export async function autoResearchExperiment(options) {
+    const theme = ThemeManager.getTheme();
+    const json = !!options.json;
+    const log = (m) => { if (!json)
+        console.log(theme.colors.muted(` ${m}`)); };
+    const projectRoot = findProjectRoot();
+    const candidateDir = options.candidateDir;
+    const baseDir = options.baseDir ?? projectRoot;
+    const cortexDir = options.cortexDir ?? projectRoot;
+    const missing = [];
+    if (!options.experimentTag)
+        missing.push('--experiment-tag');
+    if (!candidateDir)
+        missing.push('--candidate-dir');
+    if (!options.taskSet)
+        missing.push('--task-set');
+    if (missing.length) {
+        console.error(theme.colors.error(`Error: missing ${missing.join(', ')}`));
+        process.exit(1);
+    }
+    const arms = [];
+    try {
+        const trainTasks = loadTasks(options.taskSet);
+        const holdoutTasks = options.holdoutSet ? loadTasks(options.holdoutSet) : undefined;
+        if (trainTasks.length === 0) {
+            console.error(theme.colors.error('Error: empty --task-set'));
+            process.exit(1);
+        }
+        // Distinct arm labels: git SHA when the dir is a checkout, else its basename.
+        const refFor = (dir, override) => {
+            if (override)
+                return override;
+            const sha = gitShortSha(dir);
+            return sha !== 'unknown' ? sha : basename(dir);
+        };
+        const baseRef = refFor(baseDir, options.baseRef);
+        const candidateRef = refFor(candidateDir, options.candidateRef);
+        if (baseRef === candidateRef) {
+            console.error(theme.colors.error(`Error: base and candidate resolve to the same ref (${baseRef}) — an experiment needs two distinct arms. Pass --base-ref/--candidate-ref to label, or use a real candidate worktree.`));
+            process.exit(1);
+        }
+        const model = options.model ?? process.env.DEFAULT_MODEL_ID;
+        // Select the target: a shell-command target (any project) or the default cortex server.
+        const target = options.runCmd
+            ? new CommandTarget({
+                template: options.runCmd,
+                buildCmd: options.buildCmd,
+                acceptExitCodes: (options.acceptExit ?? '0').split(',').map(s => Number(s.trim())).filter(n => Number.isFinite(n)),
+            })
+            : new CortexTarget();
+        if (!json) {
+            console.log();
+            console.log(` ${theme.colors.highlight('Experiment')}  ${options.experimentTag}  ${baseRef} → ${candidateRef}  [${target.kind}]`);
+        }
+        // Prepare each arm (build if asked + start its runner). Candidate builds unless
+        // --no-build; base builds only with --build-base. Each arm gets its own reserved port
+        // (server targets bind it; command targets ignore it).
+        const basePort = num(options.basePort) ?? await freePort();
+        const candPort = num(options.candidatePort) ?? await freePort();
+        const baseArm = await target.prepare(baseDir, { port: basePort, model, build: !options.noBuild && !!options.buildBase, log });
+        arms.push(baseArm);
+        const candArm = await target.prepare(candidateDir, { port: candPort, model, build: !options.noBuild, log });
+        arms.push(candArm);
+        // Bench both arms + gate (shared store at cortexDir/.cortex).
+        const matrix = new ModelRouterMatrix(cortexDir);
+        const ledger = new ExperimentLedger(cortexDir);
+        const result = await runExperiment(matrix, ledger, {
+            baseRunner: baseArm.runner,
+            candidateRunner: candArm.runner,
+        }, {
+            experimentTag: options.experimentTag,
+            baseRef, candidateRef,
+            branch: options.branch ?? candidateRef,
+            trainTasks, holdoutTasks,
+            runs: num(options.runs),
+            nFamily: num(options.nFamily) ?? 1,
+            modelId: model,
+            temperature: num(options.temperature),
+            strategy: options.strategy,
+            deficiencyId: options.deficiencyId,
+            benchmarkSource: options.benchmarkSource,
+            gate: { alpha: num(options.alpha), seed: num(options.seed), minRunsPerArm: num(options.minRuns) },
+            epsilon: num(options.epsilon),
+            onProgress: log,
+        });
+        const out = {
+            experimentTag: options.experimentTag,
+            baseRef, candidateRef, branch: options.branch ?? candidateRef,
+            verdict: result.verdict,
+            holdoutVerdict: result.holdoutVerdict,
+            regressedTasks: result.regressedTasks,
+            mergeEligible: result.mergeEligible,
+            benchSummaries: result.benchSummaries,
+            cortexDir,
+            jsonlPaths: {
+                matrix: join(cortexDir, '.cortex', 'router-matrix.jsonl'),
+                experiments: join(cortexDir, '.cortex', 'experiments.jsonl'),
+                backlog: join(cortexDir, '.cortex', 'research-backlog.jsonl'),
+            },
+        };
+        if (json) {
+            console.log(JSON.stringify(out, null, 2));
+            return;
+        }
+        const v = result.verdict;
+        const dc = v.decision === 'keep' ? theme.colors.success : v.decision === 'discard' ? theme.colors.error : theme.colors.muted;
+        console.log();
+        console.log(` ${theme.colors.highlight('Decision')}    ${dc(v.decision.toUpperCase())}   effect ${v.effect >= 0 ? '+' : ''}${v.effect}  CI [${v.ciLow ?? '—'}, ${v.ciHigh ?? '—'}]  p=${v.pValue ?? '—'} vs ${v.alphaAdjusted ?? '—'} (N=${num(options.nFamily) ?? 1})`);
+        console.log(` ${theme.colors.highlight('Holdout')}     ${result.holdoutVerdict ? result.holdoutVerdict.decision.toUpperCase() + ` (effect ${result.holdoutVerdict.effect >= 0 ? '+' : ''}${result.holdoutVerdict.effect})` : theme.colors.muted('not provided → not verifiable')}`);
+        if (result.regressedTasks.length)
+            console.log(` ${theme.colors.error('Regressions')} ${result.regressedTasks.length}: ${result.regressedTasks.join(', ')}`);
+        console.log(` ${theme.colors.highlight('Mergeable')}   ${result.mergeEligible ? theme.colors.success('YES') : theme.colors.muted('no')}`);
+        console.log(theme.colors.muted(` artifact → ${cortexDir}/.cortex/{router-matrix,experiments,research-backlog}.jsonl`));
+        console.log();
+    }
+    catch (error) {
+        if (json)
+            console.log(JSON.stringify({ error: error.message }, null, 2));
+        else
+            console.error(theme.colors.error(`Error: ${error.message}`));
+        process.exitCode = 1;
+    }
+    finally {
+        for (const a of arms)
+            a.stop();
+    }
+}
+//# sourceMappingURL=experiment.js.map