jiva-core 0.3.47 → 0.3.48

package/dist/code/benchmark/tests.js

@@ -0,0 +1,234 @@
+/**
+ * Cumulative node:test files for the benchmark suite.
+ *
+ * Tier N's workspace contains every test file test1..testN, so the verifier
+ * catches regressions in earlier tiers (critical for the tier-5 refactor and
+ * the tier-8 debug task). All tests import the library through `../src/index.js`
+ * except the tier-5 structural test, which deliberately imports the split
+ * modules to force the refactor.
+ *
+ * The agent must NOT modify any of these files (see BenchmarkTask.protectedPaths).
+ */
+const TEST_FILES = {
+    1: {
+        path: 'test/t01.test.mjs',
+        content: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createTask } from '../src/index.js';
+
+test('createTask sets the title and defaults done to false', () => {
+  const t = createTask('Buy milk');
+  assert.equal(t.title, 'Buy milk');
+  assert.equal(t.done, false);
+});
+
+test('createTask rejects empty or blank titles', () => {
+  assert.throws(() => createTask(''));
+  assert.throws(() => createTask('   '));
+});
+`,
+    },
+    2: {
+        path: 'test/t02.test.mjs',
+        content: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { addTask, removeTask, listTasks } from '../src/index.js';
+
+test('addTask appends a task with an incrementing id', () => {
+  let list = [];
+  list = addTask(list, 'a');
+  assert.equal(list.length, 1);
+  assert.equal(list[0].id, 1);
+  list = addTask(list, 'b');
+  assert.equal(list.length, 2);
+  assert.equal(list[1].id, 2);
+});
+
+test('addTask does not mutate the input list', () => {
+  const original = [];
+  const next = addTask(original, 'a');
+  assert.equal(original.length, 0);
+  assert.equal(next.length, 1);
+});
+
+test('removeTask removes the task with the given id', () => {
+  let list = addTask(addTask(addTask([], 'a'), 'b'), 'c'); // ids 1,2,3
+  list = removeTask(list, 2);
+  assert.deepEqual(list.map((t) => t.id), [1, 3]);
+});
+
+test('listTasks returns all tasks', () => {
+  const list = addTask(addTask([], 'a'), 'b');
+  assert.equal(listTasks(list).length, 2);
+});
+`,
+    },
+    3: {
+        path: 'test/t03.test.mjs',
+        content: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { addTask, toggleTask } from '../src/index.js';
+
+test('toggleTask flips done for the task with the matching id', () => {
+  let list = addTask(addTask([], 'a'), 'b'); // ids 1,2
+  list = toggleTask(list, 2);
+  assert.equal(list.find((t) => t.id === 1).done, false);
+  assert.equal(list.find((t) => t.id === 2).done, true);
+});
+
+test('toggleTask toggles back to false when called twice', () => {
+  let list = addTask([], 'a'); // id 1
+  list = toggleTask(list, 1);
+  list = toggleTask(list, 1);
+  assert.equal(list.find((t) => t.id === 1).done, false);
+});
+`,
+    },
+    4: {
+        path: 'test/t04.test.mjs',
+        content: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { addTask, setPriority, tasksByPriority } from '../src/index.js';
+
+test('new tasks default to normal priority', () => {
+  const list = addTask([], 'a');
+  assert.equal(list[0].priority, 'normal');
+});
+
+test('setPriority updates the priority of the matching task', () => {
+  let list = addTask(addTask([], 'a'), 'b'); // ids 1,2
+  list = setPriority(list, 2, 'high');
+  assert.equal(list.find((t) => t.id === 2).priority, 'high');
+  assert.equal(list.find((t) => t.id === 1).priority, 'normal');
+});
+
+test('setPriority rejects an invalid priority', () => {
+  const list = addTask([], 'a');
+  assert.throws(() => setPriority(list, 1, 'urgent'));
+});
+
+test('tasksByPriority filters by priority', () => {
+  let list = addTask(addTask([], 'a'), 'b');
+  list = setPriority(list, 1, 'high');
+  assert.equal(tasksByPriority(list, 'high').length, 1);
+  assert.equal(tasksByPriority(list, 'normal').length, 1);
+});
+`,
+    },
+    5: {
+        path: 'test/t05.test.mjs',
+        content: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import * as model from '../src/model.js';
+import * as query from '../src/query.js';
+import * as index from '../src/index.js';
+
+test('model module exposes the mutation functions', () => {
+  for (const fn of ['createTask', 'addTask', 'removeTask', 'toggleTask', 'setPriority']) {
+    assert.equal(typeof model[fn], 'function', 'model.' + fn);
+  }
+});
+
+test('query module exposes the read functions', () => {
+  for (const fn of ['listTasks', 'tasksByPriority']) {
+    assert.equal(typeof query[fn], 'function', 'query.' + fn);
+  }
+});
+
+test('index re-exports the full public API', () => {
+  for (const fn of ['createTask', 'addTask', 'removeTask', 'toggleTask', 'setPriority', 'listTasks', 'tasksByPriority']) {
+    assert.equal(typeof index[fn], 'function', 'index.' + fn);
+  }
+});
+`,
+    },
+    6: {
+        path: 'test/t06.test.mjs',
+        content: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { sortTasks } from '../src/index.js';
+
+const mk = (id, due) => ({ id, title: 't' + id, done: false, priority: 'normal', due });
+
+test('sorts by due date ascending', () => {
+  const list = [mk(1, '2026-03-01'), mk(2, '2026-01-01'), mk(3, '2026-02-01')];
+  assert.deepEqual(sortTasks(list, { by: 'due' }).map((t) => t.id), [2, 3, 1]);
+});
+
+test('places tasks without a due date last', () => {
+  const list = [mk(1, null), mk(2, '2026-01-01'), mk(3, undefined)];
+  const ids = sortTasks(list, { by: 'due' }).map((t) => t.id);
+  assert.equal(ids[0], 2);
+  assert.deepEqual(ids.slice(1).sort(), [1, 3]);
+});
+
+test('is stable for equal due dates', () => {
+  const list = [mk(1, '2026-01-01'), mk(2, '2026-01-01'), mk(3, '2026-01-01')];
+  assert.deepEqual(sortTasks(list, { by: 'due' }).map((t) => t.id), [1, 2, 3]);
+});
+
+test('does not mutate the input list', () => {
+  const list = [mk(2, '2026-02-01'), mk(1, '2026-01-01')];
+  sortTasks(list, { by: 'due' });
+  assert.deepEqual(list.map((t) => t.id), [2, 1]);
+});
+`,
+    },
+    7: {
+        path: 'test/t07.test.mjs',
+        content: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { addTask, setPriority, serialize, deserialize } from '../src/index.js';
+
+test('serialize/deserialize round-trips a list preserving all fields', () => {
+  let list = addTask(addTask([], 'a'), 'b');
+  list = setPriority(list, 1, 'high');
+  const restored = deserialize(serialize(list));
+  assert.deepEqual(restored, list);
+});
+
+test('deserialize rejects JSON that is not an array', () => {
+  assert.throws(() => deserialize('{}'));
+});
+`,
+    },
+    8: {
+        path: 'test/t08.test.mjs',
+        content: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { addTask, removeTask, serialize, deserialize } from '../src/index.js';
+
+test('ids stay unique after a removal and re-add', () => {
+  let list = addTask(addTask(addTask([], 'a'), 'b'), 'c'); // ids 1,2,3
+  list = removeTask(list, 2); // ids 1,3
+  list = addTask(list, 'd');
+  const ids = list.map((t) => t.id);
+  assert.equal(new Set(ids).size, ids.length, 'duplicate ids: ' + ids.join(','));
+});
+
+test('ids stay unique after a serialize round-trip and add', () => {
+  let list = addTask(addTask(addTask([], 'a'), 'b'), 'c');
+  list = removeTask(list, 2);
+  list = deserialize(serialize(list));
+  list = addTask(list, 'e');
+  const ids = list.map((t) => t.id);
+  assert.equal(new Set(ids).size, ids.length, 'duplicate ids: ' + ids.join(','));
+});
+`,
+    },
+};
+/** All test files for tiers 1..tier (cumulative). */
+export function cumulativeTests(tier) {
+    const files = {};
+    for (let t = 1; t <= tier; t++) {
+        const entry = TEST_FILES[t];
+        if (entry)
+            files[entry.path] = entry.content;
+    }
+    return files;
+}
+/** Paths of all test files for tiers 1..tier (the protected set). */
+export function cumulativeTestPaths(tier) {
+    return Object.keys(cumulativeTests(tier));
+}
+//# sourceMappingURL=tests.js.map

package/dist/code/benchmark/tests.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tests.js","sourceRoot":"","sources":["../../../src/code/benchmark/tests.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,MAAM,UAAU,GAAsD;IACpE,CAAC,EAAE;QACD,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;;;;;;;;;;;;;;CAcZ;KACE;IACD,CAAC,EAAE;QACD,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+BZ;KACE;IACD,CAAC,EAAE;QACD,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;;;;;;;;;;;;;;;;;CAiBZ;KACE;IACD,CAAC,EAAE;QACD,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2BZ;KACE;IACD,CAAC,EAAE;QACD,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;CAuBZ;KACE;IACD,CAAC,EAAE;QACD,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4BZ;KACE;IACD,CAAC,EAAE;QACD,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;;;;;;;;;;;;;;CAcZ;KACE;IACD,CAAC,EAAE;QACD,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;CAoBZ;KACE;CACF,CAAC;AAEF,qDAAqD;AACrD,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,KAAK;YAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,OAAO,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5C,CAAC"}

package/dist/code/benchmark/types.d.ts

@@ -0,0 +1,161 @@
+/**
+ * Benchmark types — shared by the runner, the CLI command and the HTTP routes.
+ *
+ * The benchmark measures how well a given model + config performs in code mode
+ * across a TDD-style suite of increasing complexity. Each task scaffolds an
+ * isolated workspace containing failing tests; the agent must make them pass
+ * without editing the protected test files. Scoring is deterministic — it is the
+ * exit status of Node's built-in test runner, not an LLM judgement.
+ */
+import type { TokenUsageSnapshot } from '../../models/token-tracker.js';
+/** A map of workspace-relative path → file content, written before the agent runs. */
+export type TaskFile = Record<string, string>;
+/**
+ * A single benchmark task. Tasks are ordered by `tier` (1 = easiest) and build
+ * on one another: the scaffold for tier N is the canonical solution of tiers
+ * 1..N-1 plus the new failing test(s) for tier N.
+ */
+export interface BenchmarkTask {
+    /** Stable identifier, e.g. "t03-bugfix-toggle". */
+    id: string;
+    /** Complexity tier, 1 (trivial) .. 8 (long-horizon debug). */
+    tier: number;
+    /** Short human title. */
+    title: string;
+    /** The capability this tier primarily exercises (shown in reports). */
+    capability: string;
+    /** Whether the task builds from scratch or improves/fixes existing code. */
+    kind: 'scratch' | 'extend' | 'bugfix' | 'refactor';
+    /** The instruction handed to the agent. */
+    prompt: string;
+    /** Files written into the workspace before the agent runs (the scaffold). */
+    scaffold: TaskFile;
+    /**
+     * Workspace-relative paths the agent must NOT modify (the tests). If any of
+     * these differ after the run, the task fails with reason `tests-modified`.
+     */
+    protectedPaths: string[];
+    /** Per-task iteration cap (overridable globally). */
+    maxIterations: number;
+    /** Per-task wall-clock timeout in ms (overridable globally). */
+    timeoutMs: number;
+}
+/**
+ * Scoring mode for a suite:
+ *  - 'gating' : binary pass/fail per task; tasks build on one another (the baseline
+ *    taskstore suite). Measures "does code mode work at all".
+ *  - 'scored' : graded by the FRACTION of spec tests passed (the capability/frontier
+ *    suites). The headline metric is the pass-rate, not all-or-nothing.
+ */
+export type ScoringMode = 'gating' | 'scored';
+/** A named collection of tasks with a shared purpose and scoring philosophy. */
+export interface BenchmarkSuite {
+    id: string;
+    name: string;
+    description: string;
+    /** Difficulty/intent tier of the whole suite. */
+    level: 'baseline' | 'capability' | 'frontier';
+    scoring: ScoringMode;
+    tasks: BenchmarkTask[];
+}
+/** Outcome of running the deterministic verifier against a workspace. */
+export interface VerifyResult {
+    passed: boolean;
+    /** Number of passing / failing test cases parsed from the runner output. */
+    testsPassed: number;
+    testsFailed: number;
+    /** Names of failing test cases (parsed from `not ok` lines) — drives scored reports. */
+    failingTests: string[];
+    /** Categorised failure reason when !passed. */
+    reason?: 'tests-failed' | 'tests-modified' | 'runner-error' | 'timeout';
+    /** Trimmed runner output (stdout+stderr), useful for diagnosis. */
+    output: string;
+}
+/** Per-task result with diagnostic metrics. */
+export interface TaskResult {
+    id: string;
+    tier: number;
+    title: string;
+    capability: string;
+    kind: BenchmarkTask['kind'];
+    passed: boolean;
+    /** Categorised reason when !passed (mirrors VerifyResult.reason or 'agent-error'). */
+    reason?: VerifyResult['reason'] | 'agent-error';
+    /** Agent loop iterations consumed. */
+    iterations: number;
+    /** True when the agent hit its iteration cap (a strong "struggling" signal). */
+    hitMaxIterations: boolean;
+    /** Tool names the agent invoked. */
+    toolsUsed: string[];
+    /** Token usage accumulated by the agent during this task. */
+    tokenUsage?: TokenUsageSnapshot;
+    /** Wall-clock duration of the agent turn (not counting verification). */
+    wallTimeMs: number;
+    testsPassed: number;
+    testsFailed: number;
+    /** Names of failing test cases (for scored suites, this is the per-capability gap list). */
+    failingTests?: string[];
+    /** Times the model hit its output-token limit mid tool-call (write/edit cut off). */
+    truncationEvents?: number;
+    /** True when this task failed AND hit output-token limits — an output-length limitation. */
+    outputLimited?: boolean;
+    /** Free-form diagnostic notes (errors, truncation hints, verifier output head). */
+    notes?: string;
+}
+/** Aggregate result for a full suite run. */
+export interface SuiteResult {
+    startedAt: string;
+    finishedAt: string;
+    /** Suite identity and scoring mode (drives report rendering). */
+    suiteId?: string;
+    suiteName?: string;
+    scoring?: ScoringMode;
+    /** Model label, when resolvable from the orchestrator config. */
+    model?: string;
+    totalTasks: number;
+    passed: number;
+    failed: number;
+    /** Highest tier reached with a pass (a single capability ceiling number). */
+    highestTierPassed: number;
+    /** Aggregate spec-test counts across the suite (the headline for scored suites). */
+    totalTestsPassed: number;
+    totalTestsRun: number;
+    /** Percentage of spec tests passed (0–100). */
+    scorePct: number;
+    totalWallTimeMs: number;
+    totalTokens?: number;
+    tasks: TaskResult[];
+}
+/** Options controlling a suite run. */
+export interface RunnerOptions {
+    /** Run only tiers 1..maxTier. */
+    maxTier?: number;
+    /** Run only these task ids (overrides maxTier). */
+    taskIds?: string[];
+    /** Global override for each task's iteration cap. */
+    maxIterations?: number;
+    /** Global override for each task's timeout. */
+    timeoutMs?: number;
+    /** Enable LSP during the run (default: off, to reduce variance). */
+    lspEnabled?: boolean;
+    /**
+     * Carry the agent's own resulting workspace forward to the next tier instead
+     * of scaffolding from the canonical golden baseline. Surfaces cascading
+     * failures the way a real session would.
+     */
+    continuous?: boolean;
+    /** Keep temp workspaces on disk for debugging. */
+    keepWorkspaces?: boolean;
+    /** Model label for reporting. */
+    model?: string;
+    /** Suite identity + scoring, embedded into the result for reporting. */
+    suiteId?: string;
+    suiteName?: string;
+    scoring?: ScoringMode;
+}
+/** Progress callback fired around each task (enables CLI lines and SSE). */
+export interface ProgressEvents {
+    onTaskStart?: (task: BenchmarkTask, index: number, total: number) => void;
+    onTaskDone?: (result: TaskResult, index: number, total: number) => void;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/code/benchmark/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAExE,sFAAsF;AACtF,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9C;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,mDAAmD;IACnD,EAAE,EAAE,MAAM,CAAC;IACX,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB,4EAA4E;IAC5E,IAAI,EAAE,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,CAAC;IACnD,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,QAAQ,EAAE,QAAQ,CAAC;IACnB;;;OAGG;IACH,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,qDAAqD;IACrD,aAAa,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE9C,gFAAgF;AAChF,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,KAAK,EAAE,UAAU,GAAG,YAAY,GAAG,UAAU,CAAC;IAC9C,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE,aAAa,EAAE,CAAC;CACxB;AAED,yEAAyE;AACzE,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,OAAO,CAAC;IAChB,4EAA4E;IAC5E,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,wFAAwF;IACxF,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,cAAc,GAAG,gBAAgB,GAAG,cAAc,GAAG,SAAS,CAAC;IACxE,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,+CAA+C;AAC/C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,sFAAsF;IACtF,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC;IAChD,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,gFAAgF;IAChF,gBAAgB,EAAE,OAAO,CAAC;IAC1B,oCAAoC;IACpC,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,4FAA4F;IAC5F,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qFAAqF;IACrF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4FAA4F;IAC5F,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,6CAA6C;AAC7C,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oFAAoF;IACpF,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,uCAAuC;AACvC,MAAM,WAAW,aAAa;IAC5B,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,4EAA4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC1E,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzE"}

package/dist/code/benchmark/types.js

@@ -0,0 +1,11 @@
+/**
+ * Benchmark types — shared by the runner, the CLI command and the HTTP routes.
+ *
+ * The benchmark measures how well a given model + config performs in code mode
+ * across a TDD-style suite of increasing complexity. Each task scaffolds an
+ * isolated workspace containing failing tests; the agent must make them pass
+ * without editing the protected test files. Scoring is deterministic — it is the
+ * exit status of Node's built-in test runner, not an LLM judgement.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/code/benchmark/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/code/benchmark/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}

package/dist/code/benchmark/verify.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Deterministic verifier — runs Node's built-in test runner against a workspace
+ * and reports pass/fail. No external test framework, no network.
+ *
+ * Two checks gate a pass:
+ *   1. Tamper check — the protected test files must be byte-identical to what was
+ *      scaffolded. If the agent edited a test to make it pass, the task fails
+ *      with reason `tests-modified`.
+ *   2. Test run — `node --test` must exit 0 with at least one passing test and
+ *      zero failures.
+ */
+import type { VerifyResult } from './types.js';
+/**
+ * Run the verifier.
+ *
+ * @param workspaceDir absolute path of the task workspace
+ * @param protectedFiles map of relative path → original content (the scaffolded tests)
+ * @param timeoutMs hard cap for the test process
+ */
+export declare function runNodeTests(workspaceDir: string, protectedFiles: Record<string, string>, timeoutMs?: number): Promise<VerifyResult>;
+//# sourceMappingURL=verify.d.ts.map

package/dist/code/benchmark/verify.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/verify.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AA+B/C;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,YAAY,EAAE,MAAM,EACpB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACtC,SAAS,SAAS,GACjB,OAAO,CAAC,YAAY,CAAC,CAwDvB"}

package/dist/code/benchmark/verify.js

@@ -0,0 +1,108 @@
+/**
+ * Deterministic verifier — runs Node's built-in test runner against a workspace
+ * and reports pass/fail. No external test framework, no network.
+ *
+ * Two checks gate a pass:
+ *   1. Tamper check — the protected test files must be byte-identical to what was
+ *      scaffolded. If the agent edited a test to make it pass, the task fails
+ *      with reason `tests-modified`.
+ *   2. Test run — `node --test` must exit 0 with at least one passing test and
+ *      zero failures.
+ */
+import { spawn } from 'child_process';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+/** Read a workspace file, returning null if it is missing. */
+async function tryRead(workspaceDir, relPath) {
+    try {
+        return await readFile(join(workspaceDir, relPath), 'utf-8');
+    }
+    catch {
+        return null;
+    }
+}
+/** Parse `# pass N` / `# fail N` from Node's TAP-ish test output. */
+function parseCounts(output) {
+    const pass = output.match(/^# pass (\d+)/m);
+    const fail = output.match(/^# fail (\d+)/m);
+    return {
+        passed: pass ? parseInt(pass[1], 10) : 0,
+        failed: fail ? parseInt(fail[1], 10) : 0,
+    };
+}
+/** Parse failing top-level test names from the TAP `not ok N - <name>` lines. */
+function parseFailingTests(output) {
+    const names = [];
+    for (const line of output.split('\n')) {
+        const m = line.match(/^not ok \d+ - (.+?)(?: # .*)?$/);
+        if (m)
+            names.push(m[1].trim());
+    }
+    return names;
+}
+/**
+ * Run the verifier.
+ *
+ * @param workspaceDir absolute path of the task workspace
+ * @param protectedFiles map of relative path → original content (the scaffolded tests)
+ * @param timeoutMs hard cap for the test process
+ */
+export async function runNodeTests(workspaceDir, protectedFiles, timeoutMs = 60_000) {
+    // 1. Tamper check.
+    for (const [relPath, original] of Object.entries(protectedFiles)) {
+        const current = await tryRead(workspaceDir, relPath);
+        if (current === null) {
+            return { passed: false, testsPassed: 0, testsFailed: 0, failingTests: [], reason: 'tests-modified', output: `Protected test file was removed: ${relPath}` };
+        }
+        if (current !== original) {
+            return { passed: false, testsPassed: 0, testsFailed: 0, failingTests: [], reason: 'tests-modified', output: `Protected test file was modified: ${relPath}` };
+        }
+    }
+    // 2. Run `node --test`, scoped to ONLY the canonical protected test files.
+    // Bare `node --test` discovers any test file in the workspace, so an agent's own
+    // scratch tests would pollute the counts and could cause false failures. Passing the
+    // protected paths explicitly makes scoring depend solely on the canonical suite.
+    const testPaths = Object.keys(protectedFiles);
+    const args = testPaths.length > 0 ? ['--test', ...testPaths] : ['--test'];
+    return new Promise((resolve) => {
+        const child = spawn(process.execPath, args, {
+            cwd: workspaceDir,
+            env: { ...process.env, NODE_OPTIONS: '' },
+        });
+        let out = '';
+        const append = (buf) => {
+            out += buf.toString();
+            if (out.length > 200_000)
+                out = out.slice(-200_000); // cap memory
+        };
+        child.stdout.on('data', append);
+        child.stderr.on('data', append);
+        const timer = setTimeout(() => {
+            child.kill('SIGKILL');
+            resolve({ passed: false, testsPassed: 0, testsFailed: 0, failingTests: parseFailingTests(out), reason: 'timeout', output: trimOutput(out) });
+        }, timeoutMs);
+        child.on('error', (err) => {
+            clearTimeout(timer);
+            resolve({ passed: false, testsPassed: 0, testsFailed: 0, failingTests: [], reason: 'runner-error', output: `${err.message}\n${trimOutput(out)}` });
+        });
+        child.on('close', (code) => {
+            clearTimeout(timer);
+            const { passed, failed } = parseCounts(out);
+            const ok = code === 0 && failed === 0 && passed > 0;
+            resolve({
+                passed: ok,
+                testsPassed: passed,
+                testsFailed: failed,
+                failingTests: parseFailingTests(out),
+                reason: ok ? undefined : 'tests-failed',
+                output: trimOutput(out),
+            });
+        });
+    });
+}
+/** Keep the most relevant tail of the runner output for diagnostics. */
+function trimOutput(out) {
+    const max = 4_000;
+    return out.length > max ? out.slice(-max) : out;
+}
+//# sourceMappingURL=verify.js.map

package/dist/code/benchmark/verify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify.js","sourceRoot":"","sources":["../../../src/code/benchmark/verify.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAG5B,8DAA8D;AAC9D,KAAK,UAAU,OAAO,CAAC,YAAoB,EAAE,OAAe;IAC1D,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,qEAAqE;AACrE,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAC5C,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,iFAAiF;AACjF,SAAS,iBAAiB,CAAC,MAAc;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACvD,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,YAAoB,EACpB,cAAsC,EACtC,SAAS,GAAG,MAAM;IAElB,mBAAmB;IACnB,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QACjE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACrD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,oCAAoC,OAAO,EAAE,EAAE,CAAC;QAC9J,CAAC;QACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,qCAAqC,OAAO,EAAE,EAAE,CAAC;QAC/J,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,iFAAiF;IACjF,qFAAqF;IACrF,iFAAiF;IACjF,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1E,OAAO,IAAI,OAAO,CAAe,CAAC,OAAO,EAAE,EAAE;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE;YAC1C,GAAG,EAAE,YAAY;YACjB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE;SAC1C,CAAC,CAAC;QAEH,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,EAAE;YAC7B,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,GAAG,CAAC,MAAM,GAAG,OAAO;gBAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa;QACpE,CAAC,CAAC;QACF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEhC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,iBAAiB,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/I,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACxB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,KAAK,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACrJ,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,EAAE,GAAG,IAAI,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;YACpD,OAAO,CAAC;gBACN,MAAM,EAAE,EAAE;gBACV,WAAW,EAAE,MAAM;gBACnB,WAAW,EAAE,MAAM;gBACnB,YAAY,EAAE,iBAAiB,CAAC,GAAG,CAAC;gBACpC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc;gBACvC,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC;aACxB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AACxE,SAAS,UAAU,CAAC,GAAW;IAC7B,MAAM,GAAG,GAAG,KAAK,CAAC;IAClB,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAClD,CAAC"}

package/dist/code/tools/edit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"edit.d.ts","sourceRoot":"","sources":["../../../src/code/tools/edit.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,YAAY,CAAC;AAI7D,eAAO,MAAM,YAAY,EAAE,SAwI1B,CAAC;AA2OF,qDAAqD;AACrD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,UAAQ,GAAG,MAAM,CAiCzG"}
+{"version":3,"file":"edit.d.ts","sourceRoot":"","sources":["../../../src/code/tools/edit.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,YAAY,CAAC;AAI7D,eAAO,MAAM,YAAY,EAAE,SAiJ1B,CAAC;AA2OF,qDAAqD;AACrD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,UAAQ,GAAG,MAAM,CAiCzG"}

package/dist/code/tools/edit.js

@@ -38,7 +38,11 @@ Returns a diff of the changes made, plus any LSP errors detected.`,
         properties: {
             file_path: {
                 type: 'string',
-                description: 'Absolute path to the file to edit',
+                description: 'Absolute path to the file to edit (preferred). The alias "path" is also accepted.',
+            },
+            path: {
+                type: 'string',
+                description: 'Alias for file_path. Prefer file_path; either is accepted.',
             },
             old_string: {
                 type: 'string',
@@ -53,10 +57,15 @@ Returns a diff of the changes made, plus any LSP errors detected.`,
                 description: 'Replace all occurrences of old_string (default: false)',
             },
         },
-        required: ['file_path', 'old_string', 'new_string'],
+        // file_path is intentionally not in `required` so the `path` alias is accepted by
+        // provider-side validation; it is validated in execute() below.
+        required: ['old_string', 'new_string'],
     },
     async execute(args, ctx) {
-        const rawPath = args.file_path;
+        const rawPath = (args.file_path ?? args.path);
+        if (typeof rawPath !== 'string' || rawPath.trim() === '') {
+            return 'Error: file_path is required — provide the absolute path to the file to edit.';
+        }
         const filePath = path.isAbsolute(rawPath) ? rawPath : path.resolve(ctx.workspaceDir, rawPath);
         const oldString = args.old_string;
         const newString = args.new_string;