jiva-core 0.3.47 → 0.3.48-dev.0fd3c65

package/dist/code/benchmark/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/code/benchmark/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9D,CAAC;AAED,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS;IAC/B,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,UAAU,CAAC,CAAa;IAC/B,IAAI,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC;IAClC,OAAO,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAkB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC;IAC1C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,gCAAgC,KAAK,CAAC,SAAS,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC;IACzF,IAAI,KAAK,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACnE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,SAAS,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IACrE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,SAAS;IACT,KAAK,CAAC,IAAI,CACR,KAAK,CAAC,IAAI,CACR,IAAI;QACF,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACd,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;QACf,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC;QACjB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACf,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACf,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC;QACjB,MAAM,CACT,CACF,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE9C,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAChE,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAClE,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAClF,KAAK,CAAC,IAAI,CACR,IAAI;YACF,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACtB,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YAChB,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,wCAAwC;YACtE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YACb,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YACb,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;YACf,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CACtB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE9C,IAAI,MAAM,EAAE,CAAC;QACX,6EAA6E;QAC7E,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,aAAa,WAAW,KAAK,CAAC,QAAQ,IAAI,CAAC;QAC5F,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QACpG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACN,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACpD,MAAM,OAAO,GACX,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAC3G,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;IAC3F,CAAC;IACD,KAAK,CAAC,IAAI,CACR,KAAK,CAAC,IAAI,CACR,iBAAiB,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE;QAC7C,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,sBAAsB,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACvE,CACF,CAAC;IAEF,sFAAsF;IACtF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACtD,KAAK,MAAM,IAAI,IAAI,MAAM;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,kFAAkF;IAClF,gFAAgF;IAChF,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IACjE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,2BAA2B,CAAC;YAClD,KAAK,CAAC,IAAI,CAAC,OAAO,aAAa,CAAC,MAAM,+DAA+D,CAAC,CAAC,CAAC;QAC1G,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,gBAAgB,uDAAuD,CAAC,CAAC,CAAC;QACtH,CAAC;IACH,CAAC;IAED,mFAAmF;IACnF,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACtD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,IAAI,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;YACpF,IAAI,CAAC,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkB;IAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC"}

package/dist/code/benchmark/runner.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Benchmark runner — drives CodeAgent through the suite in isolated workspaces
+ * and collects deterministic pass/fail plus diagnostic metrics.
+ *
+ * Each task (default mode):
+ *   1. mkdtemp an isolated workspace and write the scaffold (golden baseline +
+ *      cumulative tests + any planted bug).
+ *   2. Build a fresh minimal CodeAgent pointed at it.
+ *   3. Run agent.chat(prompt) under a wall-clock timeout.
+ *   4. Run the deterministic verifier (node --test + tamper check).
+ *   5. Record metrics and tear everything down.
+ *
+ * In `continuous` mode the agent's own workspace is carried forward between
+ * tiers: only the new test files (and planted source the agent hasn't written
+ * yet) are overlaid, so cascading failures surface the way a real session would.
+ */
+import type { ModelOrchestrator } from '../../models/orchestrator.js';
+import type { BenchmarkTask, RunnerOptions, ProgressEvents, SuiteResult } from './types.js';
+export declare class BenchmarkRunner {
+    private orchestrator;
+    constructor(orchestrator: ModelOrchestrator);
+    /** Run a selected set of tasks. `tasks` must already be filtered/ordered by tier. */
+    run(tasks: BenchmarkTask[], options?: RunnerOptions, progress?: ProgressEvents): Promise<SuiteResult>;
+    private runTask;
+    private aggregate;
+}
+/** Convenience entry point used by both the CLI command and the HTTP route. */
+export declare function runBenchmark(orchestrator: ModelOrchestrator, tasks: BenchmarkTask[], options?: RunnerOptions, progress?: ProgressEvents): Promise<SuiteResult>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/code/benchmark/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAItE,OAAO,KAAK,EACV,aAAa,EACb,aAAa,EACb,cAAc,EAEd,WAAW,EACZ,MAAM,YAAY,CAAC;AAmBpB,qBAAa,eAAe;IACd,OAAO,CAAC,YAAY;gBAAZ,YAAY,EAAE,iBAAiB;IAEnD,qFAAqF;IAC/E,GAAG,CACP,KAAK,EAAE,aAAa,EAAE,EACtB,OAAO,GAAE,aAAkB,EAC3B,QAAQ,GAAE,cAAmB,GAC5B,OAAO,CAAC,WAAW,CAAC;YA2DT,OAAO;IAmGrB,OAAO,CAAC,SAAS;CAkClB;AAED,+EAA+E;AAC/E,wBAAsB,YAAY,CAChC,YAAY,EAAE,iBAAiB,EAC/B,KAAK,EAAE,aAAa,EAAE,EACtB,OAAO,GAAE,aAAkB,EAC3B,QAAQ,GAAE,cAAmB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAKtB"}

package/dist/code/benchmark/runner.js

@@ -0,0 +1,219 @@
+/**
+ * Benchmark runner — drives CodeAgent through the suite in isolated workspaces
+ * and collects deterministic pass/fail plus diagnostic metrics.
+ *
+ * Each task (default mode):
+ *   1. mkdtemp an isolated workspace and write the scaffold (golden baseline +
+ *      cumulative tests + any planted bug).
+ *   2. Build a fresh minimal CodeAgent pointed at it.
+ *   3. Run agent.chat(prompt) under a wall-clock timeout.
+ *   4. Run the deterministic verifier (node --test + tamper check).
+ *   5. Record metrics and tear everything down.
+ *
+ * In `continuous` mode the agent's own workspace is carried forward between
+ * tiers: only the new test files (and planted source the agent hasn't written
+ * yet) are overlaid, so cascading failures surface the way a real session would.
+ */
+import { mkdtemp, mkdir, writeFile, rm, access } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join, dirname } from 'path';
+import { logger } from '../../utils/logger.js';
+import { buildBenchmarkAgent } from './agent-factory.js';
+import { runNodeTests } from './verify.js';
+async function exists(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function writeWorkspaceFile(workspaceDir, relPath, content) {
+    const abs = join(workspaceDir, relPath);
+    await mkdir(dirname(abs), { recursive: true });
+    await writeFile(abs, content, 'utf-8');
+}
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+export class BenchmarkRunner {
+    orchestrator;
+    constructor(orchestrator) {
+        this.orchestrator = orchestrator;
+    }
+    /** Run a selected set of tasks. `tasks` must already be filtered/ordered by tier. */
+    async run(tasks, options = {}, progress = {}) {
+        const startedAt = new Date().toISOString();
+        const results = [];
+        let carriedWorkspace;
+        for (let i = 0; i < tasks.length; i++) {
+            const task = tasks[i];
+            progress.onTaskStart?.(task, i, tasks.length);
+            const maxIterations = options.maxIterations ?? task.maxIterations;
+            const timeoutMs = options.timeoutMs ?? task.timeoutMs;
+            // Resolve the workspace and write the scaffold.
+            let workspaceDir;
+            let ephemeral = false;
+            if (options.continuous && carriedWorkspace) {
+                workspaceDir = carriedWorkspace;
+                // Overlay new tests always; overlay planted source only where absent.
+                for (const [rel, content] of Object.entries(task.scaffold)) {
+                    const isTest = rel.startsWith('test/');
+                    const isPkg = rel === 'package.json';
+                    if (isTest || isPkg || !(await exists(join(workspaceDir, rel)))) {
+                        await writeWorkspaceFile(workspaceDir, rel, content);
+                    }
+                }
+            }
+            else {
+                workspaceDir = await mkdtemp(join(tmpdir(), `jiva-bench-${task.id}-`));
+                ephemeral = !options.continuous;
+                for (const [rel, content] of Object.entries(task.scaffold)) {
+                    await writeWorkspaceFile(workspaceDir, rel, content);
+                }
+                if (options.continuous)
+                    carriedWorkspace = workspaceDir;
+            }
+            // Snapshot the protected test files (from the task's own scaffold) for the tamper check.
+            const protectedFiles = {};
+            for (const p of task.protectedPaths) {
+                protectedFiles[p] = task.scaffold[p] ?? '';
+            }
+            const result = await this.runTask(task, workspaceDir, maxIterations, timeoutMs, protectedFiles, !!options.lspEnabled);
+            results.push(result);
+            progress.onTaskDone?.(result, i, tasks.length);
+            // Clean up ephemeral (non-continuous) workspaces unless asked to keep them.
+            if (ephemeral && !options.keepWorkspaces) {
+                await rm(workspaceDir, { recursive: true, force: true }).catch(() => { });
+            }
+        }
+        // Clean up the carried-forward workspace at the very end.
+        if (options.continuous && carriedWorkspace && !options.keepWorkspaces) {
+            await rm(carriedWorkspace, { recursive: true, force: true }).catch(() => { });
+        }
+        const finishedAt = new Date().toISOString();
+        return this.aggregate(tasks, results, startedAt, finishedAt, options);
+    }
+    async runTask(task, workspaceDir, maxIterations, timeoutMs, protectedFiles, lspEnabled) {
+        const agent = await buildBenchmarkAgent({
+            orchestrator: this.orchestrator,
+            workspaceDir,
+            maxIterations,
+            lspEnabled,
+        });
+        const base = {
+            id: task.id,
+            tier: task.tier,
+            title: task.title,
+            capability: task.capability,
+            kind: task.kind,
+        };
+        const start = Date.now();
+        let iterations = 0;
+        let toolsUsed = [];
+        let timedOut = false;
+        let agentError;
+        let truncationEvents = 0;
+        try {
+            const chatPromise = agent.chat(task.prompt);
+            const raced = await Promise.race([
+                chatPromise.then((r) => ({ kind: 'ok', r })),
+                delay(timeoutMs).then(() => ({ kind: 'timeout' })),
+            ]);
+            if (raced.kind === 'ok') {
+                iterations = raced.r.iterations;
+                toolsUsed = raced.r.toolsUsed;
+                truncationEvents = raced.r.truncationEvents ?? 0;
+            }
+            else {
+                timedOut = true;
+                agent.stop();
+                chatPromise.catch(() => { }); // swallow the late rejection/resolution
+                iterations = maxIterations;
+            }
+        }
+        catch (err) {
+            agentError = err instanceof Error ? err.message : String(err);
+        }
+        const wallTimeMs = Date.now() - start;
+        let tokenUsage;
+        try {
+            tokenUsage = agent.getTokenUsage();
+        }
+        catch {
+            tokenUsage = undefined;
+        }
+        // Verify regardless — the agent may have completed the files even on timeout.
+        const verify = await runNodeTests(workspaceDir, protectedFiles, 60_000);
+        await agent.cleanup().catch(() => { });
+        const passed = verify.passed && !agentError;
+        let reason;
+        if (!passed) {
+            if (agentError)
+                reason = 'agent-error';
+            else if (timedOut && !verify.passed)
+                reason = 'timeout';
+            else
+                reason = verify.reason;
+        }
+        const outputLimited = !passed && truncationEvents > 0;
+        const notes = [
+            agentError ? `agent error: ${agentError}` : '',
+            timedOut ? `timed out after ${timeoutMs}ms` : '',
+            outputLimited ? `hit output-token limit ${truncationEvents}×` : '',
+            verify.output ? `verifier: ${verify.output.split('\n').slice(-6).join(' ').slice(0, 300)}` : '',
+        ]
+            .filter(Boolean)
+            .join(' | ');
+        return {
+            ...base,
+            passed,
+            reason,
+            iterations,
+            hitMaxIterations: iterations >= maxIterations,
+            toolsUsed,
+            tokenUsage,
+            wallTimeMs,
+            testsPassed: verify.testsPassed,
+            testsFailed: verify.testsFailed,
+            failingTests: verify.failingTests,
+            truncationEvents,
+            outputLimited,
+            notes: notes || undefined,
+        };
+    }
+    aggregate(tasks, results, startedAt, finishedAt, options) {
+        const passed = results.filter((r) => r.passed).length;
+        const highestTierPassed = results.filter((r) => r.passed).reduce((m, r) => Math.max(m, r.tier), 0);
+        const totalWallTimeMs = results.reduce((s, r) => s + r.wallTimeMs, 0);
+        const totalTokens = results.reduce((s, r) => s + (r.tokenUsage?.totalTokens ?? 0), 0);
+        const totalTestsPassed = results.reduce((s, r) => s + r.testsPassed, 0);
+        const totalTestsRun = results.reduce((s, r) => s + r.testsPassed + r.testsFailed, 0);
+        const scorePct = totalTestsRun > 0 ? Math.round((totalTestsPassed / totalTestsRun) * 100) : 0;
+        return {
+            startedAt,
+            finishedAt,
+            suiteId: options.suiteId,
+            suiteName: options.suiteName,
+            scoring: options.scoring,
+            model: options.model,
+            totalTasks: tasks.length,
+            passed,
+            failed: results.length - passed,
+            highestTierPassed,
+            totalTestsPassed,
+            totalTestsRun,
+            scorePct,
+            totalWallTimeMs,
+            totalTokens: totalTokens || undefined,
+            tasks: results,
+        };
+    }
+}
+/** Convenience entry point used by both the CLI command and the HTTP route. */
+export async function runBenchmark(orchestrator, tasks, options = {}, progress = {}) {
+    if (tasks.length === 0) {
+        logger.warn('[benchmark] no tasks selected');
+    }
+    return new BenchmarkRunner(orchestrator).run(tasks, options, progress);
+}
+//# sourceMappingURL=runner.js.map

package/dist/code/benchmark/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/code/benchmark/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAErC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAS3C,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,YAAoB,EAAE,OAAe,EAAE,OAAe;IACtF,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACxC,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,MAAM,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEtF,MAAM,OAAO,eAAe;IACN;IAApB,YAAoB,YAA+B;QAA/B,iBAAY,GAAZ,YAAY,CAAmB;IAAG,CAAC;IAEvD,qFAAqF;IACrF,KAAK,CAAC,GAAG,CACP,KAAsB,EACtB,UAAyB,EAAE,EAC3B,WAA2B,EAAE;QAE7B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,IAAI,gBAAoC,CAAC;QAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAE9C,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC;YAClE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;YAEtD,gDAAgD;YAChD,IAAI,YAAoB,CAAC;YACzB,IAAI,SAAS,GAAG,KAAK,CAAC;YACtB,IAAI,OAAO,CAAC,UAAU,IAAI,gBAAgB,EAAE,CAAC;gBAC3C,YAAY,GAAG,gBAAgB,CAAC;gBAChC,sEAAsE;gBACtE,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC3D,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;oBACvC,MAAM,KAAK,GAAG,GAAG,KAAK,cAAc,CAAC;oBACrC,IAAI,MAAM,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;wBAChE,MAAM,kBAAkB,CAAC,YAAY,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;oBACvD,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBACvE,SAAS,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC;gBAChC,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC3D,MAAM,kBAAkB,CAAC,YAAY,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;gBACvD,CAAC;gBACD,IAAI,OAAO,CAAC,UAAU;oBAAE,gBAAgB,GAAG,YAAY,CAAC;YAC1D,CAAC;YAED,yFAAyF;YACzF,MAAM,cAAc,GAA2B,EAAE,CAAC;YAClD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpC,cAAc,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACtH,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,QAAQ,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAE/C,4EAA4E;YAC5E,IAAI,SAAS,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBACzC,MAAM,EAAE,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,0DAA0D;QAC1D,IAAI,OAAO,CAAC,UAAU,IAAI,gBAAgB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACtE,MAAM,EAAE,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACxE,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,IAAmB,EACnB,YAAoB,EACpB,aAAqB,EACrB,SAAiB,EACjB,cAAsC,EACtC,UAAmB;QAEnB,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC;YACtC,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,YAAY;YACZ,aAAa;YACb,UAAU;SACX,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;QAEF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,SAAS,GAAa,EAAE,CAAC;QAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,UAA8B,CAAC;QACnC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAC/B,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAa,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrD,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAkB,EAAE,CAAC,CAAC;aAC5D,CAAC,CAAC;YAEH,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBACxB,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;gBAChC,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC9B,gBAAgB,GAAG,KAAK,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACN,QAAQ,GAAG,IAAI,CAAC;gBAChB,KAAK,CAAC,IAAI,EAAE,CAAC;gBACb,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,wCAAwC;gBACrE,UAAU,GAAG,aAAa,CAAC;YAC7B,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,UAAU,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,UAAU,CAAC;QACf,IAAI,CAAC;YACH,UAAU,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,UAAU,GAAG,SAAS,CAAC;QACzB,CAAC;QAED,8EAA8E;QAC9E,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAEtC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC;QAC5C,IAAI,MAA4B,CAAC;QACjC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,UAAU;gBAAE,MAAM,GAAG,aAAa,CAAC;iBAClC,IAAI,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM;gBAAE,MAAM,GAAG,SAAS,CAAC;;gBACnD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9B,CAAC;QAED,MAAM,aAAa,GAAG,CAAC,MAAM,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG;YACZ,UAAU,CAAC,CAAC,CAAC,gBAAgB,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE;YAC9C,QAAQ,CAAC,CAAC,CAAC,mBAAmB,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE;YAChD,aAAa,CAAC,CAAC,CAAC,0BAA0B,gBAAgB,GAAG,CAAC,CAAC,CAAC,EAAE;YAClE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;SAChG;aACE,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,KAAK,CAAC,CAAC;QAEf,OAAO;YACL,GAAG,IAAI;YACP,MAAM;YACN,MAAM;YACN,UAAU;YACV,gBAAgB,EAAE,UAAU,IAAI,aAAa;YAC7C,SAAS;YACT,UAAU;YACV,UAAU;YACV,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,gBAAgB;YAChB,aAAa;YACb,KAAK,EAAE,KAAK,IAAI,SAAS;SAC1B,CAAC;IACJ,CAAC;IAEO,SAAS,CACf,KAAsB,EACtB,OAAqB,EACrB,SAAiB,EACjB,UAAkB,EAClB,OAAsB;QAEtB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACnG,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtF,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACrF,MAAM,QAAQ,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,gBAAgB,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9F,OAAO;YACL,SAAS;YACT,UAAU;YACV,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;YAC/B,iBAAiB;YACjB,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,eAAe;YACf,WAAW,EAAE,WAAW,IAAI,SAAS;YACrC,KAAK,EAAE,OAAO;SACf,CAAC;IACJ,CAAC;CACF;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,YAA+B,EAC/B,KAAsB,EACtB,UAAyB,EAAE,EAC3B,WAA2B,EAAE;IAE7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,IAAI,eAAe,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AACzE,CAAC"}

package/dist/code/benchmark/suites.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Benchmark suite registry.
+ *
+ * Three tiers of suites with distinct purposes (see docs/guides/benchmark-suite.md):
+ *  - taskstore (baseline)  — gating: does code mode work at all? Most usable configs ~100%.
+ *  - microcrm  (capability)— scored: build a Node+SQLite REST API to spec; differentiates
+ *                            models and configurations by pass-rate.
+ *  - frontier  (future)    — scored: ~50 tests, even an optimal config only ~30-40%; full
+ *                            pass needs a frontier model.
+ */
+import type { BenchmarkSuite } from './types.js';
+export declare const BENCHMARK_SUITES: BenchmarkSuite[];
+/** Default suite when none is specified (preserves prior `jiva benchmark` behaviour). */
+export declare const DEFAULT_SUITE_ID = "taskstore";
+export declare function getSuite(id: string): BenchmarkSuite | undefined;
+export declare function listSuiteMetadata(): Array<Pick<BenchmarkSuite, 'id' | 'name' | 'description' | 'level' | 'scoring'> & {
+    taskCount: number;
+}>;
+//# sourceMappingURL=suites.d.ts.map

package/dist/code/benchmark/suites.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"suites.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/suites.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIjD,eAAO,MAAM,gBAAgB,EAAE,cAAc,EAqB5C,CAAC;AAEF,yFAAyF;AACzF,eAAO,MAAM,gBAAgB,cAAc,CAAC;AAE5C,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAE/D;AAED,wBAAgB,iBAAiB,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,MAAM,GAAG,aAAa,GAAG,OAAO,GAAG,SAAS,CAAC,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAS5I"}

package/dist/code/benchmark/suites.js

@@ -0,0 +1,48 @@
+/**
+ * Benchmark suite registry.
+ *
+ * Three tiers of suites with distinct purposes (see docs/guides/benchmark-suite.md):
+ *  - taskstore (baseline)  — gating: does code mode work at all? Most usable configs ~100%.
+ *  - microcrm  (capability)— scored: build a Node+SQLite REST API to spec; differentiates
+ *                            models and configurations by pass-rate.
+ *  - frontier  (future)    — scored: ~50 tests, even an optimal config only ~30-40%; full
+ *                            pass needs a frontier model.
+ */
+import { BENCHMARK_TASKS } from './tasks.js';
+import { MICROCRM_TASKS } from './microcrm/index.js';
+export const BENCHMARK_SUITES = [
+    {
+        id: 'taskstore',
+        name: 'Taskstore (baseline)',
+        description: 'TDD smoke/regression suite over an evolving in-memory library. Binary pass/fail, tasks ' +
+            'build on one another. Confirms a model + config can do code mode at all.',
+        level: 'baseline',
+        scoring: 'gating',
+        tasks: BENCHMARK_TASKS,
+    },
+    {
+        id: 'microcrm',
+        name: 'micro-CRM API (capability)',
+        description: 'Build a Node + SQLite REST API to a fixed spec, graded by the fraction of spec tests ' +
+            'passed. Differentiates models and configurations; use it to find the optimal setup.',
+        level: 'capability',
+        scoring: 'scored',
+        tasks: MICROCRM_TASKS,
+    },
+];
+/** Default suite when none is specified (preserves prior `jiva benchmark` behaviour). */
+export const DEFAULT_SUITE_ID = 'taskstore';
+export function getSuite(id) {
+    return BENCHMARK_SUITES.find((s) => s.id === id);
+}
+export function listSuiteMetadata() {
+    return BENCHMARK_SUITES.map(({ id, name, description, level, scoring, tasks }) => ({
+        id,
+        name,
+        description,
+        level,
+        scoring,
+        taskCount: tasks.length,
+    }));
+}
+//# sourceMappingURL=suites.js.map

package/dist/code/benchmark/suites.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suites.js","sourceRoot":"","sources":["../../../src/code/benchmark/suites.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,MAAM,CAAC,MAAM,gBAAgB,GAAqB;IAChD;QACE,EAAE,EAAE,WAAW;QACf,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EACT,yFAAyF;YACzF,0EAA0E;QAC5E,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE,eAAe;KACvB;IACD;QACE,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,4BAA4B;QAClC,WAAW,EACT,uFAAuF;YACvF,qFAAqF;QACvF,KAAK,EAAE,YAAY;QACnB,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE,cAAc;KACtB;CACF,CAAC;AAEF,yFAAyF;AACzF,MAAM,CAAC,MAAM,gBAAgB,GAAG,WAAW,CAAC;AAE5C,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QACjF,EAAE;QACF,IAAI;QACJ,WAAW;QACX,KAAK;QACL,OAAO;QACP,SAAS,EAAE,KAAK,CAAC,MAAM;KACxB,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/code/benchmark/tasks.d.ts

@@ -0,0 +1,16 @@
+/**
+ * The benchmark suite — 8 TDD-style coding tasks of increasing complexity over a
+ * single evolving `taskstore` library. Tier 1 is a from-scratch one-liner; tier 8
+ * is a long-horizon cross-module debug that stresses iteration count and output
+ * length — the regime where rate-limited or short-output models fall down.
+ */
+import type { BenchmarkTask } from './types.js';
+export declare const BENCHMARK_TASKS: BenchmarkTask[];
+/** Lightweight metadata for listing endpoints (no scaffold/test payloads). */
+export declare function listTaskMetadata(tasks: BenchmarkTask[]): Array<Pick<BenchmarkTask, 'id' | 'tier' | 'title' | 'capability' | 'kind'>>;
+/** Resolve which tasks to run from a suite's task list, given runner filters. */
+export declare function selectTasks(tasks: BenchmarkTask[], opts: {
+    maxTier?: number;
+    taskIds?: string[];
+}): BenchmarkTask[];
+//# sourceMappingURL=tasks.d.ts.map

package/dist/code/benchmark/tasks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tasks.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/tasks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,YAAY,CAAC;AA+I1D,eAAO,MAAM,eAAe,EAAE,aAAa,EAWxC,CAAC;AAEJ,8EAA8E;AAC9E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,aAAa,EAAE,GACrB,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC,CAAC,CAE7E;AAED,iFAAiF;AACjF,wBAAgB,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAAG,aAAa,EAAE,CASnH"}

package/dist/code/benchmark/tasks.js

@@ -0,0 +1,153 @@
+/**
+ * The benchmark suite — 8 TDD-style coding tasks of increasing complexity over a
+ * single evolving `taskstore` library. Tier 1 is a from-scratch one-liner; tier 8
+ * is a long-horizon cross-module debug that stresses iteration count and output
+ * length — the regime where rate-limited or short-output models fall down.
+ */
+import { PACKAGE_JSON, scaffoldSrc } from './fixtures.js';
+import { cumulativeTests, cumulativeTestPaths } from './tests.js';
+const COMMON_RULES = `
+Rules:
+- The workspace is a Node ES-module project. Run the test suite with: node --test
+- Make ALL tests pass. Do not stop until "node --test" reports zero failures.
+- You MUST NOT modify, delete or move anything under the test/ directory.
+- Keep existing passing tests green — only change source files under src/.`;
+function buildScaffold(tier) {
+    return {
+        'package.json': PACKAGE_JSON,
+        ...scaffoldSrc(tier),
+        ...cumulativeTests(tier),
+    };
+}
+const SPECS = [
+    {
+        id: 't01-create',
+        tier: 1,
+        title: 'Create createTask from scratch',
+        capability: 'Write a new file',
+        kind: 'scratch',
+        instruction: 'Create src/index.js exporting a function `createTask(title)` that returns an object ' +
+            '`{ title, done: false }` and throws an Error when the title is empty or only whitespace.',
+        maxIterations: 15,
+        timeoutMs: 120_000,
+    },
+    {
+        id: 't02-extend-crud',
+        tier: 2,
+        title: 'Add addTask / removeTask / listTasks',
+        capability: 'Read & extend an existing file',
+        kind: 'extend',
+        instruction: 'src/index.js already has createTask. Add and export `addTask(list, title)` (returns a NEW ' +
+            'list with an appended task that has a unique incrementing numeric `id` starting at 1, without ' +
+            'mutating the input), `removeTask(list, id)` (returns a new list without that task), and ' +
+            '`listTasks(list)` (returns all tasks).',
+        maxIterations: 20,
+        timeoutMs: 180_000,
+    },
+    {
+        id: 't03-bugfix-toggle',
+        tier: 3,
+        title: 'Fix the toggleTask bug',
+        capability: 'Diagnose & fix a targeted bug',
+        kind: 'bugfix',
+        instruction: 'src/index.js contains a `toggleTask(list, id)` function that is supposed to flip the `done` ' +
+            'flag of the task whose `id` matches, but the tests are failing. Find and fix the bug. ' +
+            'Do not rewrite unrelated code.',
+        maxIterations: 20,
+        timeoutMs: 180_000,
+    },
+    {
+        id: 't04-feature-priority',
+        tier: 4,
+        title: 'Add task priorities',
+        capability: 'Multi-function feature',
+        kind: 'extend',
+        instruction: 'Add priority support to src/index.js. New tasks must default to priority "normal". Export ' +
+            '`setPriority(list, id, priority)` that updates the matching task (allowed values: "low", ' +
+            '"normal", "high"; throw on anything else) and `tasksByPriority(list, priority)` that filters ' +
+            'tasks by priority.',
+        maxIterations: 25,
+        timeoutMs: 240_000,
+    },
+    {
+        id: 't05-refactor-split',
+        tier: 5,
+        title: 'Split into model / query modules',
+        capability: 'Multi-file refactor without regressions',
+        kind: 'refactor',
+        instruction: 'Refactor the single src/index.js into three files WITHOUT changing behaviour: ' +
+            '`src/model.js` exporting the mutation functions (createTask, addTask, removeTask, toggleTask, ' +
+            'setPriority), `src/query.js` exporting the read functions (listTasks, tasksByPriority), and ' +
+            '`src/index.js` re-exporting everything from both so the existing public API is unchanged. ' +
+            'All previously passing tests must remain green.',
+        maxIterations: 30,
+        timeoutMs: 300_000,
+    },
+    {
+        id: 't06-algorithm-sort',
+        tier: 6,
+        title: 'Implement sortTasks with edge cases',
+        capability: 'Algorithmic reasoning & edge cases',
+        kind: 'extend',
+        instruction: 'Add and export `sortTasks(list, { by })` (currently only `by: "due"` is supported — throw on ' +
+            'anything else). It returns a NEW array sorted by the `due` field (an ISO date string) ascending. ' +
+            'Tasks whose `due` is null or undefined must sort to the end, and the sort must be stable for ' +
+            'tasks with equal due dates. Put it in the appropriate module and re-export it from src/index.js.',
+        maxIterations: 30,
+        timeoutMs: 300_000,
+    },
+    {
+        id: 't07-storage-roundtrip',
+        tier: 7,
+        title: 'Add JSON serialize / deserialize',
+        capability: 'New module + serialization',
+        kind: 'extend',
+        instruction: 'Create `src/storage.js` exporting `serialize(list)` (returns a JSON string) and ' +
+            '`deserialize(str)` (parses it back into a task list, throwing if the parsed value is not an ' +
+            'array). A serialize→deserialize round-trip must preserve every field of every task. Re-export ' +
+            'both from src/index.js.',
+        maxIterations: 30,
+        timeoutMs: 300_000,
+    },
+    {
+        id: 't08-debug-id-collision',
+        tier: 8,
+        title: 'Debug the id-collision integration bug',
+        capability: 'Long-horizon cross-module debugging',
+        kind: 'bugfix',
+        instruction: 'The integration tests in test/t08.test.mjs are failing: after a task is removed (or a list is ' +
+            'restored from storage) and a new task is added, the new task can collide with an existing id. ' +
+            'Run the tests, read the failure, find the root cause across the source modules, and fix it so ' +
+            'ids are always unique. Do not weaken or change any test.',
+        maxIterations: 40,
+        timeoutMs: 420_000,
+    },
+];
+export const BENCHMARK_TASKS = SPECS.map((s) => ({
+    id: s.id,
+    tier: s.tier,
+    title: s.title,
+    capability: s.capability,
+    kind: s.kind,
+    prompt: `${s.instruction}\n${COMMON_RULES}`,
+    scaffold: buildScaffold(s.tier),
+    protectedPaths: cumulativeTestPaths(s.tier),
+    maxIterations: s.maxIterations,
+    timeoutMs: s.timeoutMs,
+}));
+/** Lightweight metadata for listing endpoints (no scaffold/test payloads). */
+export function listTaskMetadata(tasks) {
+    return tasks.map(({ id, tier, title, capability, kind }) => ({ id, tier, title, capability, kind }));
+}
+/** Resolve which tasks to run from a suite's task list, given runner filters. */
+export function selectTasks(tasks, opts) {
+    if (opts.taskIds && opts.taskIds.length > 0) {
+        const set = new Set(opts.taskIds);
+        return tasks.filter((t) => set.has(t.id));
+    }
+    if (typeof opts.maxTier === 'number') {
+        return tasks.filter((t) => t.tier <= opts.maxTier);
+    }
+    return [...tasks];
+}
+//# sourceMappingURL=tasks.js.map

package/dist/code/benchmark/tasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tasks.js","sourceRoot":"","sources":["../../../src/code/benchmark/tasks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAElE,MAAM,YAAY,GAAG;;;;;2EAKsD,CAAC;AAE5E,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO;QACL,cAAc,EAAE,YAAY;QAC5B,GAAG,WAAW,CAAC,IAAI,CAAC;QACpB,GAAG,eAAe,CAAC,IAAI,CAAC;KACzB,CAAC;AACJ,CAAC;AAaD,MAAM,KAAK,GAAe;IACxB;QACE,EAAE,EAAE,YAAY;QAChB,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,gCAAgC;QACvC,UAAU,EAAE,kBAAkB;QAC9B,IAAI,EAAE,SAAS;QACf,WAAW,EACT,sFAAsF;YACtF,0FAA0F;QAC5F,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,sCAAsC;QAC7C,UAAU,EAAE,gCAAgC;QAC5C,IAAI,EAAE,QAAQ;QACd,WAAW,EACT,4FAA4F;YAC5F,gGAAgG;YAChG,0FAA0F;YAC1F,wCAAwC;QAC1C,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,wBAAwB;QAC/B,UAAU,EAAE,+BAA+B;QAC3C,IAAI,EAAE,QAAQ;QACd,WAAW,EACT,8FAA8F;YAC9F,wFAAwF;YACxF,gCAAgC;QAClC,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,qBAAqB;QAC5B,UAAU,EAAE,wBAAwB;QACpC,IAAI,EAAE,QAAQ;QACd,WAAW,EACT,4FAA4F;YAC5F,2FAA2F;YAC3F,+FAA+F;YAC/F,oBAAoB;QACtB,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,kCAAkC;QACzC,UAAU,EAAE,yCAAyC;QACrD,IAAI,EAAE,UAAU;QAChB,WAAW,EACT,gFAAgF;YAChF,gGAAgG;YAChG,8FAA8F;YAC9F,4FAA4F;YAC5F,iDAAiD;QACnD,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,qCAAqC;QAC5C,UAAU,EAAE,oCAAoC;QAChD,IAAI,EAAE,QAAQ;QACd,WAAW,EACT,+FAA+F;YAC/F,mGAAmG;YACnG,+FAA+F;YAC/F,kGAAkG;QACpG,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB;IACD;QACE,EAAE,EAAE,uBAAuB;QAC3B,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,kCAAkC;QACzC,UAAU,EAAE,4BAA4B;QACxC,IAAI,EAAE,QAAQ;QACd,WAAW,EACT,kFAAkF;YAClF,8FAA8F;YAC9F,gGAAgG;YAChG,yBAAyB;QAC3B,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,wCAAwC;QAC/C,UAAU,EAAE,qCAAqC;QACjD,IAAI,EAAE,QAAQ;QACd,WAAW,EACT,gGAAgG;YAChG,gGAAgG;YAChG,gGAAgG;YAChG,0DAA0D;QAC5D,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAoB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChE,EAAE,EAAE,CAAC,CAAC,EAAE;IACR,IAAI,EAAE,CAAC,CAAC,IAAI;IACZ,KAAK,EAAE,CAAC,CAAC,KAAK;IACd,UAAU,EAAE,CAAC,CAAC,UAAU;IACxB,IAAI,EAAE,CAAC,CAAC,IAAI;IACZ,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,KAAK,YAAY,EAAE;IAC3C,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/B,cAAc,EAAE,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3C,aAAa,EAAE,CAAC,CAAC,aAAa;IAC9B,SAAS,EAAE,CAAC,CAAC,SAAS;CACvB,CAAC,CAAC,CAAC;AAEJ,8EAA8E;AAC9E,MAAM,UAAU,gBAAgB,CAC9B,KAAsB;IAEtB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACvG,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,WAAW,CAAC,KAAsB,EAAE,IAA8C;IAChG,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACrC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,OAAQ,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACpB,CAAC"}

package/dist/code/benchmark/tests.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Cumulative node:test files for the benchmark suite.
+ *
+ * Tier N's workspace contains every test file test1..testN, so the verifier
+ * catches regressions in earlier tiers (critical for the tier-5 refactor and
+ * the tier-8 debug task). All tests import the library through `../src/index.js`
+ * except the tier-5 structural test, which deliberately imports the split
+ * modules to force the refactor.
+ *
+ * The agent must NOT modify any of these files (see BenchmarkTask.protectedPaths).
+ */
+import type { TaskFile } from './types.js';
+/** All test files for tiers 1..tier (cumulative). */
+export declare function cumulativeTests(tier: number): TaskFile;
+/** Paths of all test files for tiers 1..tier (the protected set). */
+export declare function cumulativeTestPaths(tier: number): string[];
+//# sourceMappingURL=tests.d.ts.map

package/dist/code/benchmark/tests.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tests.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/tests.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAmN3C,qDAAqD;AACrD,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAOtD;AAED,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAE1D"}