jiva-core 0.3.47 → 0.3.48-dev.3254e6a

package/dist/code/benchmark/microcrm/assets/query.test.mjs

@@ -0,0 +1,96 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createApp } from '../src/app.js';
+
+async function withServer(run) {
+  const server = createApp(':memory:');
+  await new Promise((res) => server.listen(0, res));
+  const { port } = server.address();
+  try { await run(`http://127.0.0.1:${port}`); } finally { await new Promise((res) => server.close(res)); }
+}
+async function req(base, method, path, body) {
+  const init = { method, headers: {} };
+  if (body !== undefined) { init.headers['content-type'] = 'application/json'; init.body = JSON.stringify(body); }
+  const res = await fetch(base + path, init);
+  const text = await res.text();
+  let json = null; try { json = text ? JSON.parse(text) : null; } catch { json = null; }
+  return { status: res.status, body: json };
+}
+
+// Seed one contact with deals of known amounts/stages.
+async function seed(base) {
+  const c = await req(base, 'POST', '/contacts', { name: 'A', email: `a${Math.random().toString(36).slice(2)}@x.com` });
+  const id = c.body.id;
+  const deals = [
+    { title: 'd1', amount: 100, stage: 'lead' },
+    { title: 'd2', amount: 200, stage: 'qualified' },
+    { title: 'd3', amount: 300, stage: 'won' },
+    { title: 'd4', amount: 400, stage: 'won' },
+  ];
+  for (const d of deals) await req(base, 'POST', `/contacts/${id}/deals`, d);
+  return id;
+}
+
+test('minAmount filters deals', () =>
+  withServer(async (base) => {
+    await seed(base);
+    const r = await req(base, 'GET', '/deals?minAmount=300');
+    assert.equal(r.status, 200);
+    assert.equal(r.body.total, 2);
+    assert.ok(r.body.data.every((d) => d.amount >= 300));
+  }));
+
+test('maxAmount filters deals', () =>
+  withServer(async (base) => {
+    await seed(base);
+    const r = await req(base, 'GET', '/deals?maxAmount=200');
+    assert.equal(r.body.total, 2);
+    assert.ok(r.body.data.every((d) => d.amount <= 200));
+  }));
+
+test('minAmount + maxAmount form a range', () =>
+  withServer(async (base) => {
+    await seed(base);
+    const r = await req(base, 'GET', '/deals?minAmount=200&maxAmount=300');
+    assert.deepEqual(r.body.data.map((d) => d.amount).sort((a, b) => a - b), [200, 300]);
+  }));
+
+test('sort=amount:desc orders descending', () =>
+  withServer(async (base) => {
+    await seed(base);
+    const r = await req(base, 'GET', '/deals?sort=amount:desc');
+    assert.deepEqual(r.body.data.map((d) => d.amount), [400, 300, 200, 100]);
+  }));
+
+test('sort=amount:asc orders ascending', () =>
+  withServer(async (base) => {
+    await seed(base);
+    const r = await req(base, 'GET', '/deals?sort=amount:asc');
+    assert.deepEqual(r.body.data.map((d) => d.amount), [100, 200, 300, 400]);
+  }));
+
+test('invalid sort field → 400', () =>
+  withServer(async (base) => {
+    await seed(base);
+    const r = await req(base, 'GET', '/deals?sort=banana:asc');
+    assert.equal(r.status, 400);
+  }));
+
+test('filters combine with stage', () =>
+  withServer(async (base) => {
+    await seed(base);
+    const r = await req(base, 'GET', '/deals?stage=won&minAmount=350');
+    assert.equal(r.body.total, 1);
+    assert.equal(r.body.data[0].amount, 400);
+  }));
+
+test('pagination returns hasMore and a full filtered total', () =>
+  withServer(async (base) => {
+    await seed(base);
+    const page1 = await req(base, 'GET', '/deals?limit=2&offset=0&sort=amount:asc');
+    assert.equal(page1.body.total, 4);
+    assert.equal(page1.body.data.length, 2);
+    assert.equal(page1.body.hasMore, true);
+    const page2 = await req(base, 'GET', '/deals?limit=2&offset=2&sort=amount:asc');
+    assert.equal(page2.body.hasMore, false);
+  }));

package/dist/code/benchmark/microcrm/assets/stats.test.mjs

@@ -0,0 +1,67 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createApp } from '../src/app.js';
+
+async function withServer(run) {
+  const server = createApp(':memory:');
+  await new Promise((res) => server.listen(0, res));
+  const { port } = server.address();
+  try { await run(`http://127.0.0.1:${port}`); } finally { await new Promise((res) => server.close(res)); }
+}
+async function req(base, method, path, body) {
+  const init = { method, headers: {} };
+  if (body !== undefined) { init.headers['content-type'] = 'application/json'; init.body = JSON.stringify(body); }
+  const res = await fetch(base + path, init);
+  const text = await res.text();
+  let json = null; try { json = text ? JSON.parse(text) : null; } catch { json = null; }
+  return { status: res.status, body: json };
+}
+const mkContact = () => ({ name: 'A', email: `a${Math.random().toString(36).slice(2)}@x.com` });
+
+test('stats for unknown contact → 404', () =>
+  withServer(async (base) => {
+    const r = await req(base, 'GET', '/contacts/99999/stats');
+    assert.equal(r.status, 404);
+  }));
+
+test('stats for a contact with no deals are all zero', () =>
+  withServer(async (base) => {
+    const c = await req(base, 'POST', '/contacts', mkContact());
+    const r = await req(base, 'GET', `/contacts/${c.body.id}/stats`);
+    assert.equal(r.status, 200);
+    assert.deepEqual(r.body, { totalDeals: 0, totalValue: 0, wonValue: 0, winRate: 0, weightedPipeline: 0 });
+  }));
+
+test('totalValue and wonValue sum the right deals', () =>
+  withServer(async (base) => {
+    const c = await req(base, 'POST', '/contacts', mkContact());
+    const id = c.body.id;
+    await req(base, 'POST', `/contacts/${id}/deals`, { title: 'a', amount: 100, stage: 'won' });
+    await req(base, 'POST', `/contacts/${id}/deals`, { title: 'b', amount: 50, stage: 'lead' });
+    const r = await req(base, 'GET', `/contacts/${id}/stats`);
+    assert.equal(r.body.totalDeals, 2);
+    assert.equal(r.body.totalValue, 150);
+    assert.equal(r.body.wonValue, 100);
+  }));
+
+test('winRate is won deals over total deals', () =>
+  withServer(async (base) => {
+    const c = await req(base, 'POST', '/contacts', mkContact());
+    const id = c.body.id;
+    await req(base, 'POST', `/contacts/${id}/deals`, { title: 'a', amount: 10, stage: 'won' });
+    for (const s of ['lead', 'qualified', 'lost']) await req(base, 'POST', `/contacts/${id}/deals`, { title: s, amount: 10, stage: s });
+    const r = await req(base, 'GET', `/contacts/${id}/stats`);
+    assert.equal(r.body.winRate, 0.25); // 1 of 4
+  }));
+
+test('weightedPipeline applies per-stage probabilities (lead .1, qualified .4, won 1, lost 0)', () =>
+  withServer(async (base) => {
+    const c = await req(base, 'POST', '/contacts', mkContact());
+    const id = c.body.id;
+    await req(base, 'POST', `/contacts/${id}/deals`, { title: 'l', amount: 100, stage: 'lead' });       // 10
+    await req(base, 'POST', `/contacts/${id}/deals`, { title: 'q', amount: 100, stage: 'qualified' });  // 40
+    await req(base, 'POST', `/contacts/${id}/deals`, { title: 'w', amount: 100, stage: 'won' });        // 100
+    await req(base, 'POST', `/contacts/${id}/deals`, { title: 'x', amount: 100, stage: 'lost' });       // 0
+    const r = await req(base, 'GET', `/contacts/${id}/stats`);
+    assert.equal(r.body.weightedPipeline, 150);
+  }));

package/dist/code/benchmark/microcrm/index.d.ts

@@ -0,0 +1,23 @@
+/**
+ * The micro-CRM benchmark suite — a building, scored suite.
+ *
+ * Tier 1 (`crm-build`) builds the base Node + SQLite REST API from scratch (large single
+ * output — a genuine test of whether the model can emit a sizeable file). Tiers 2-5 each
+ * scaffold the WORKING base implementation and ask the agent to add one harder feature
+ * (atomic bulk insert, advanced querying, weighted analytics, idempotency). Each task is
+ * graded by its own hidden test file; the suite score is the fraction of all spec tests
+ * passed.
+ *
+ * Test files and reference implementations live as plain `.mjs` assets (read verbatim at
+ * runtime; copied into dist by scripts/copy-benchmark-assets.mjs).
+ */
+import type { BenchmarkTask, TaskFile } from '../types.js';
+export declare const MICROCRM_PACKAGE_JSON: string;
+/** Base API the agent builds in tier 1 and that scaffolds the extend tasks. */
+export declare const MICROCRM_BASE_APP: string;
+/** Reference with every extension implemented — used only by the harness self-test. */
+export declare const MICROCRM_FULL_APP: string;
+export declare const MICROCRM_TASKS: BenchmarkTask[];
+/** Self-test helper: the known-good files that should pass a given task's tests. */
+export declare function microcrmReferenceFor(task: BenchmarkTask): TaskFile;
+//# sourceMappingURL=index.d.ts.map

package/dist/code/benchmark/microcrm/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/code/benchmark/microcrm/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAY3D,eAAO,MAAM,qBAAqB,QAAsF,CAAC;AACzH,+EAA+E;AAC/E,eAAO,MAAM,iBAAiB,QAAiC,CAAC;AAChE,uFAAuF;AACvF,eAAO,MAAM,iBAAiB,QAAsC,CAAC;AA6CrE,eAAO,MAAM,cAAc,EAAE,aAAa,EAMzC,CAAC;AAEF,oFAAoF;AACpF,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,aAAa,GAAG,QAAQ,CAIlE"}

package/dist/code/benchmark/microcrm/index.js

@@ -0,0 +1,72 @@
+/**
+ * The micro-CRM benchmark suite — a building, scored suite.
+ *
+ * Tier 1 (`crm-build`) builds the base Node + SQLite REST API from scratch (large single
+ * output — a genuine test of whether the model can emit a sizeable file). Tiers 2-5 each
+ * scaffold the WORKING base implementation and ask the agent to add one harder feature
+ * (atomic bulk insert, advanced querying, weighted analytics, idempotency). Each task is
+ * graded by its own hidden test file; the suite score is the fraction of all spec tests
+ * passed.
+ *
+ * Test files and reference implementations live as plain `.mjs` assets (read verbatim at
+ * runtime; copied into dist by scripts/copy-benchmark-assets.mjs).
+ */
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+import { MICROCRM_SPEC, MICROCRM_BULK_SPEC, MICROCRM_QUERY_SPEC, MICROCRM_STATS_SPEC, MICROCRM_IDEMPOTENCY_SPEC, } from './spec.js';
+const assetsDir = join(dirname(fileURLToPath(import.meta.url)), 'assets');
+const readAsset = (name) => readFileSync(join(assetsDir, name), 'utf-8');
+export const MICROCRM_PACKAGE_JSON = JSON.stringify({ name: 'microcrm', type: 'module', private: true }, null, 2) + '\n';
+/** Base API the agent builds in tier 1 and that scaffolds the extend tasks. */
+export const MICROCRM_BASE_APP = readAsset('app.reference.mjs');
+/** Reference with every extension implemented — used only by the harness self-test. */
+export const MICROCRM_FULL_APP = readAsset('app.full.reference.mjs');
+/** A base-build task (from scratch) — no src provided. */
+function buildTask() {
+    return {
+        id: 'crm-build',
+        tier: 1,
+        title: 'Build the micro-CRM REST API',
+        capability: 'Build a spec-defined Node + SQLite REST API (large output)',
+        kind: 'scratch',
+        prompt: MICROCRM_SPEC,
+        scaffold: { 'package.json': MICROCRM_PACKAGE_JSON, 'test/api.test.mjs': readAsset('api.test.mjs') },
+        protectedPaths: ['test/api.test.mjs'],
+        maxIterations: 60,
+        timeoutMs: 600_000,
+    };
+}
+/** An extend task — scaffolds the working base app and adds one feature. */
+function extendTask(id, tier, title, capability, prompt, testFile) {
+    return {
+        id,
+        tier,
+        title,
+        capability,
+        kind: 'extend',
+        prompt,
+        scaffold: {
+            'package.json': MICROCRM_PACKAGE_JSON,
+            'src/app.js': MICROCRM_BASE_APP,
+            [`test/${testFile}`]: readAsset(testFile),
+        },
+        protectedPaths: [`test/${testFile}`],
+        maxIterations: 40,
+        timeoutMs: 360_000,
+    };
+}
+export const MICROCRM_TASKS = [
+    buildTask(),
+    extendTask('crm-bulk', 2, 'Atomic bulk deal insert', 'Transactions / atomicity', MICROCRM_BULK_SPEC, 'bulk.test.mjs'),
+    extendTask('crm-query', 3, 'Advanced deal querying', 'Filtering, sorting & pagination', MICROCRM_QUERY_SPEC, 'query.test.mjs'),
+    extendTask('crm-stats', 4, 'Pipeline analytics', 'Aggregation & weighted arithmetic', MICROCRM_STATS_SPEC, 'stats.test.mjs'),
+    extendTask('crm-idempotency', 5, 'Idempotency-Key on create', 'Idempotency / dedup reasoning', MICROCRM_IDEMPOTENCY_SPEC, 'idempotency.test.mjs'),
+];
+/** Self-test helper: the known-good files that should pass a given task's tests. */
+export function microcrmReferenceFor(task) {
+    const src = task.id === 'crm-build' ? MICROCRM_BASE_APP : MICROCRM_FULL_APP;
+    // Keep the task's test files, but override src/app.js with the reference implementation.
+    return { ...task.scaffold, 'package.json': MICROCRM_PACKAGE_JSON, 'src/app.js': src };
+}
+//# sourceMappingURL=index.js.map

package/dist/code/benchmark/microcrm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/code/benchmark/microcrm/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAErC,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,yBAAyB,GAC1B,MAAM,WAAW,CAAC;AAEnB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;AAC1E,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;AAEzF,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AACzH,+EAA+E;AAC/E,MAAM,CAAC,MAAM,iBAAiB,GAAG,SAAS,CAAC,mBAAmB,CAAC,CAAC;AAChE,uFAAuF;AACvF,MAAM,CAAC,MAAM,iBAAiB,GAAG,SAAS,CAAC,wBAAwB,CAAC,CAAC;AAErE,0DAA0D;AAC1D,SAAS,SAAS;IAChB,OAAO;QACL,EAAE,EAAE,WAAW;QACf,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,8BAA8B;QACrC,UAAU,EAAE,4DAA4D;QACxE,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,aAAa;QACrB,QAAQ,EAAE,EAAE,cAAc,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,SAAS,CAAC,cAAc,CAAC,EAAE;QACnG,cAAc,EAAE,CAAC,mBAAmB,CAAC;QACrC,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,SAAS,UAAU,CACjB,EAAU,EACV,IAAY,EACZ,KAAa,EACb,UAAkB,EAClB,MAAc,EACd,QAAgB;IAEhB,OAAO;QACL,EAAE;QACF,IAAI;QACJ,KAAK;QACL,UAAU;QACV,IAAI,EAAE,QAAQ;QACd,MAAM;QACN,QAAQ,EAAE;YACR,cAAc,EAAE,qBAAqB;YACrC,YAAY,EAAE,iBAAiB;YAC/B,CAAC,QAAQ,QAAQ,EAAE,CAAC,EAAE,SAAS,CAAC,QAAQ,CAAC;SAC1C;QACD,cAAc,EAAE,CAAC,QAAQ,QAAQ,EAAE,CAAC;QACpC,aAAa,EAAE,EAAE;QACjB,SAAS,EAAE,OAAO;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAoB;IAC7C,SAAS,EAAE;IACX,UAAU,CAAC,UAAU,EAAE,CAAC,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,kBAAkB,EAAE,eAAe,CAAC;IACrH,UAAU,CAAC,WAAW,EAAE,CAAC,EAAE,wBAAwB,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,gBAAgB,CAAC;IAC9H,UAAU,CAAC,WAAW,EAAE,CAAC,EAAE,oBAAoB,EAAE,mCAAmC,EAAE,mBAAmB,EAAE,gBAAgB,CAAC;IAC5H,UAAU,CAAC,iBAAiB,EAAE,CAAC,EAAE,2BAA2B,EAAE,+BAA+B,EAAE,yBAAyB,EAAE,sBAAsB,CAAC;CAClJ,CAAC;AAEF,oFAAoF;AACpF,MAAM,UAAU,oBAAoB,CAAC,IAAmB;IACtD,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,KAAK,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAC5E,yFAAyF;IACzF,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC;AACxF,CAAC"}

package/dist/code/benchmark/microcrm/spec.d.ts

@@ -0,0 +1,13 @@
+/**
+ * The micro-CRM specification handed to the agent. It pins the exact module
+ * contract the hidden test suite depends on (entry file, exported factory, the
+ * built-in `node:http` + `node:sqlite` stack) and then enumerates every endpoint
+ * with its status codes and response shapes. Ambiguity here would unfairly fail a
+ * competent model, so the spec mirrors the test expectations precisely.
+ */
+export declare const MICROCRM_SPEC = "Build a small REST API for a \"micro CRM\" using ONLY Node.js built-in modules \u2014 no npm install, no external packages.\n\nSTACK (mandatory):\n- HTTP server: the built-in `node:http` module.\n- Database: the built-in `node:sqlite` module (import { DatabaseSync } from 'node:sqlite').\n- The project is an ES module (package.json has \"type\": \"module\").\n\nMODULE CONTRACT (the test suite imports exactly this):\n- Create the file src/app.js.\n- Export a function: `export function createApp(dbPath = ':memory:')`.\n- createApp must create the database (initialise the schema if needed) and return a\n  Node http.Server created with http.createServer(handler). DO NOT call .listen() \u2014\n  the caller starts it. Each call to createApp uses its own database connection.\n- All request and response bodies are JSON. Always set Content-Type: application/json\n  (except for 204 responses, which have no body).\n\nDATA MODEL:\n- contact: { id (auto int), name (string, required, non-empty), email (string, required,\n  unique, must look like an email), company (string or null), created_at (ISO string) }\n- deal: { id (auto int), contact_id (int), title (string, required, non-empty),\n  amount (number >= 0, default 0), stage (one of \"lead\" | \"qualified\" | \"won\" | \"lost\",\n  default \"lead\"), created_at (ISO string) }\n\nENDPOINTS:\n\nContacts:\n- POST /contacts \u2192 create. 201 with the created contact. 400 if name or email missing/invalid.\n  409 if the email already exists. 400 if the request body is not valid JSON.\n- GET /contacts \u2192 200 with { data: [...contacts], total }. Supports:\n    ?q=<text>   case-insensitive substring match on name OR email\n    ?limit=&?offset=   pagination; \"total\" is the FULL count ignoring limit/offset.\n- GET /contacts/:id \u2192 200 with the contact, or 404.\n- PATCH /contacts/:id \u2192 partial update (any subset of name/email/company). 200 with the\n  updated contact. 404 if not found. 400 if a provided field is invalid. 409 if the new\n  email belongs to another contact.\n- DELETE /contacts/:id \u2192 204 (no body), or 404. Deleting a contact also deletes its deals.\n\nDeals:\n- POST /contacts/:id/deals \u2192 create a deal for that contact. 201 with the created deal\n  (stage defaults to \"lead\", amount defaults to 0). 404 if the contact does not exist.\n  400 if title is missing, amount is negative, or stage is not one of the four allowed values.\n- GET /contacts/:id/deals \u2192 200 with { data: [...deals], total } for that contact only.\n- GET /deals \u2192 200 with { data: [...deals], total }. Supports ?stage=<stage> to filter.\n- GET /deals/summary \u2192 200 with { summary: { lead: { count, totalAmount }, qualified: {...},\n  won: {...}, lost: {...} } } \u2014 every stage present, zeros when there are no deals.\n- PATCH /deals/:id \u2192 partial update (title/amount/stage). 200 with the updated deal, or 404.\n  400 on invalid amount/stage.\n\nAnything else (unknown path) \u2192 404 with a JSON body.\n\nGOAL: make the hidden test suite in test/api.test.mjs pass. Run it with: node --test\nDo NOT modify anything under test/. Implement everything in src/.";
+export declare const MICROCRM_BULK_SPEC = "src/app.js already contains a working micro-CRM REST API (contacts + deals) built on node:http + node:sqlite, exporting createApp(dbPath). Extend it \u2014 do not rewrite it, and keep all existing behaviour working. Run the tests with: node --test. Do NOT modify test/.\n\nAdd an ATOMIC bulk deal endpoint:\n- POST /contacts/:id/deals/bulk with body { deals: [ { title, amount?, stage? }, ... ] }.\n- 404 if the contact does not exist.\n- 400 if \"deals\" is missing, not an array, or empty.\n- Validate every deal with the same rules as a single deal (title required & non-empty,\n  amount a number >= 0, stage one of lead/qualified/won/lost). If ANY item is invalid,\n  respond 400 and insert NOTHING (the operation must be all-or-nothing / atomic).\n- On success \u2192 201 with { data: [...createdDeals], total }. Each created deal applies the\n  same defaults as the single-deal endpoint (stage \"lead\", amount 0).\nMake test/bulk.test.mjs pass.";
+export declare const MICROCRM_QUERY_SPEC = "src/app.js already contains a working micro-CRM REST API (contacts + deals) built on node:http + node:sqlite, exporting createApp(dbPath). Extend it \u2014 do not rewrite it, and keep all existing behaviour working. Run the tests with: node --test. Do NOT modify test/.\n\nUpgrade GET /deals with filtering, sorting and pagination metadata:\n- ?minAmount= and ?maxAmount= filter by amount (inclusive). A non-numeric value \u2192 400.\n- ?stage= filters by stage (already supported); all filters combine with AND.\n- ?sort=<field>:<dir> where field is one of amount | created_at | id and dir is asc | desc.\n  Any other field or direction \u2192 400. Default order is by id ascending.\n- ?limit= and ?offset= paginate. The response is { data, total, hasMore } where \"total\" is\n  the FULL count of rows matching the filters (ignoring limit/offset) and \"hasMore\" is true\n  when there are more rows after the current page (offset + data.length < total).\nMake test/query.test.mjs pass.";
+export declare const MICROCRM_STATS_SPEC = "src/app.js already contains a working micro-CRM REST API (contacts + deals) built on node:http + node:sqlite, exporting createApp(dbPath). Extend it \u2014 do not rewrite it, and keep all existing behaviour working. Run the tests with: node --test. Do NOT modify test/.\n\nAdd a per-contact analytics endpoint:\n- GET /contacts/:id/stats \u2192 404 if the contact does not exist, else 200 with:\n  { totalDeals, totalValue, wonValue, winRate, weightedPipeline }\n  where:\n    - totalDeals  = number of deals for the contact\n    - totalValue  = sum of all deal amounts\n    - wonValue    = sum of amounts of deals whose stage is \"won\"\n    - winRate     = wonDeals / totalDeals (0 when there are no deals), rounded to 4 decimals\n    - weightedPipeline = sum over deals of amount * P(stage), rounded to 2 decimals, where\n      P = { lead: 0.1, qualified: 0.4, won: 1.0, lost: 0.0 }\n  A contact with no deals returns all zeros.\nMake test/stats.test.mjs pass.";
+export declare const MICROCRM_IDEMPOTENCY_SPEC = "src/app.js already contains a working micro-CRM REST API (contacts + deals) built on node:http + node:sqlite, exporting createApp(dbPath). Extend it \u2014 do not rewrite it, and keep all existing behaviour working. Run the tests with: node --test. Do NOT modify test/.\n\nAdd idempotency to contact creation via the \"Idempotency-Key\" request header:\n- POST /contacts with an Idempotency-Key header: the FIRST request with a given key creates\n  the contact as normal (201) and remembers the key.\n- A REPEAT request with the SAME key must NOT create a duplicate \u2014 return the originally\n  created contact with status 200 (even if the new request body differs; do not validate or\n  conflict-check it).\n- Different keys create different contacts. Requests with no Idempotency-Key behave exactly\n  as before (each creates a contact). The key store may be in-memory (per createApp instance).\nMake test/idempotency.test.mjs pass.";
+//# sourceMappingURL=spec.d.ts.map

package/dist/code/benchmark/microcrm/spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spec.d.ts","sourceRoot":"","sources":["../../../../src/code/benchmark/microcrm/spec.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,eAAO,MAAM,aAAa,4lGAmDwC,CAAC;AAWnE,eAAO,MAAM,kBAAkB,47BAWD,CAAC;AAE/B,eAAO,MAAM,mBAAmB,g+BAUD,CAAC;AAEhC,eAAO,MAAM,mBAAmB,w8BAaD,CAAC;AAEhC,eAAO,MAAM,yBAAyB,86BAUD,CAAC"}

package/dist/code/benchmark/microcrm/spec.js

@@ -0,0 +1,115 @@
+/**
+ * The micro-CRM specification handed to the agent. It pins the exact module
+ * contract the hidden test suite depends on (entry file, exported factory, the
+ * built-in `node:http` + `node:sqlite` stack) and then enumerates every endpoint
+ * with its status codes and response shapes. Ambiguity here would unfairly fail a
+ * competent model, so the spec mirrors the test expectations precisely.
+ */
+export const MICROCRM_SPEC = `Build a small REST API for a "micro CRM" using ONLY Node.js built-in modules — no npm install, no external packages.
+
+STACK (mandatory):
+- HTTP server: the built-in \`node:http\` module.
+- Database: the built-in \`node:sqlite\` module (import { DatabaseSync } from 'node:sqlite').
+- The project is an ES module (package.json has "type": "module").
+
+MODULE CONTRACT (the test suite imports exactly this):
+- Create the file src/app.js.
+- Export a function: \`export function createApp(dbPath = ':memory:')\`.
+- createApp must create the database (initialise the schema if needed) and return a
+  Node http.Server created with http.createServer(handler). DO NOT call .listen() —
+  the caller starts it. Each call to createApp uses its own database connection.
+- All request and response bodies are JSON. Always set Content-Type: application/json
+  (except for 204 responses, which have no body).
+
+DATA MODEL:
+- contact: { id (auto int), name (string, required, non-empty), email (string, required,
+  unique, must look like an email), company (string or null), created_at (ISO string) }
+- deal: { id (auto int), contact_id (int), title (string, required, non-empty),
+  amount (number >= 0, default 0), stage (one of "lead" | "qualified" | "won" | "lost",
+  default "lead"), created_at (ISO string) }
+
+ENDPOINTS:
+
+Contacts:
+- POST /contacts → create. 201 with the created contact. 400 if name or email missing/invalid.
+  409 if the email already exists. 400 if the request body is not valid JSON.
+- GET /contacts → 200 with { data: [...contacts], total }. Supports:
+    ?q=<text>   case-insensitive substring match on name OR email
+    ?limit=&?offset=   pagination; "total" is the FULL count ignoring limit/offset.
+- GET /contacts/:id → 200 with the contact, or 404.
+- PATCH /contacts/:id → partial update (any subset of name/email/company). 200 with the
+  updated contact. 404 if not found. 400 if a provided field is invalid. 409 if the new
+  email belongs to another contact.
+- DELETE /contacts/:id → 204 (no body), or 404. Deleting a contact also deletes its deals.
+
+Deals:
+- POST /contacts/:id/deals → create a deal for that contact. 201 with the created deal
+  (stage defaults to "lead", amount defaults to 0). 404 if the contact does not exist.
+  400 if title is missing, amount is negative, or stage is not one of the four allowed values.
+- GET /contacts/:id/deals → 200 with { data: [...deals], total } for that contact only.
+- GET /deals → 200 with { data: [...deals], total }. Supports ?stage=<stage> to filter.
+- GET /deals/summary → 200 with { summary: { lead: { count, totalAmount }, qualified: {...},
+  won: {...}, lost: {...} } } — every stage present, zeros when there are no deals.
+- PATCH /deals/:id → partial update (title/amount/stage). 200 with the updated deal, or 404.
+  400 on invalid amount/stage.
+
+Anything else (unknown path) → 404 with a JSON body.
+
+GOAL: make the hidden test suite in test/api.test.mjs pass. Run it with: node --test
+Do NOT modify anything under test/. Implement everything in src/.`;
+/**
+ * The extend tasks scaffold a WORKING base API (src/app.js) and ask the agent to add
+ * one harder feature. Each prompt pins the precise contract its hidden test depends on.
+ */
+const EXTEND_PREAMBLE = 'src/app.js already contains a working micro-CRM REST API (contacts + deals) built on ' +
+    'node:http + node:sqlite, exporting createApp(dbPath). Extend it — do not rewrite it, and ' +
+    'keep all existing behaviour working. Run the tests with: node --test. Do NOT modify test/.';
+export const MICROCRM_BULK_SPEC = `${EXTEND_PREAMBLE}
+
+Add an ATOMIC bulk deal endpoint:
+- POST /contacts/:id/deals/bulk with body { deals: [ { title, amount?, stage? }, ... ] }.
+- 404 if the contact does not exist.
+- 400 if "deals" is missing, not an array, or empty.
+- Validate every deal with the same rules as a single deal (title required & non-empty,
+  amount a number >= 0, stage one of lead/qualified/won/lost). If ANY item is invalid,
+  respond 400 and insert NOTHING (the operation must be all-or-nothing / atomic).
+- On success → 201 with { data: [...createdDeals], total }. Each created deal applies the
+  same defaults as the single-deal endpoint (stage "lead", amount 0).
+Make test/bulk.test.mjs pass.`;
+export const MICROCRM_QUERY_SPEC = `${EXTEND_PREAMBLE}
+
+Upgrade GET /deals with filtering, sorting and pagination metadata:
+- ?minAmount= and ?maxAmount= filter by amount (inclusive). A non-numeric value → 400.
+- ?stage= filters by stage (already supported); all filters combine with AND.
+- ?sort=<field>:<dir> where field is one of amount | created_at | id and dir is asc | desc.
+  Any other field or direction → 400. Default order is by id ascending.
+- ?limit= and ?offset= paginate. The response is { data, total, hasMore } where "total" is
+  the FULL count of rows matching the filters (ignoring limit/offset) and "hasMore" is true
+  when there are more rows after the current page (offset + data.length < total).
+Make test/query.test.mjs pass.`;
+export const MICROCRM_STATS_SPEC = `${EXTEND_PREAMBLE}
+
+Add a per-contact analytics endpoint:
+- GET /contacts/:id/stats → 404 if the contact does not exist, else 200 with:
+  { totalDeals, totalValue, wonValue, winRate, weightedPipeline }
+  where:
+    - totalDeals  = number of deals for the contact
+    - totalValue  = sum of all deal amounts
+    - wonValue    = sum of amounts of deals whose stage is "won"
+    - winRate     = wonDeals / totalDeals (0 when there are no deals), rounded to 4 decimals
+    - weightedPipeline = sum over deals of amount * P(stage), rounded to 2 decimals, where
+      P = { lead: 0.1, qualified: 0.4, won: 1.0, lost: 0.0 }
+  A contact with no deals returns all zeros.
+Make test/stats.test.mjs pass.`;
+export const MICROCRM_IDEMPOTENCY_SPEC = `${EXTEND_PREAMBLE}
+
+Add idempotency to contact creation via the "Idempotency-Key" request header:
+- POST /contacts with an Idempotency-Key header: the FIRST request with a given key creates
+  the contact as normal (201) and remembers the key.
+- A REPEAT request with the SAME key must NOT create a duplicate — return the originally
+  created contact with status 200 (even if the new request body differs; do not validate or
+  conflict-check it).
+- Different keys create different contacts. Requests with no Idempotency-Key behave exactly
+  as before (each creates a contact). The key store may be in-memory (per createApp instance).
+Make test/idempotency.test.mjs pass.`;
+//# sourceMappingURL=spec.js.map

package/dist/code/benchmark/microcrm/spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"spec.js","sourceRoot":"","sources":["../../../../src/code/benchmark/microcrm/spec.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kEAmDqC,CAAC;AAEnE;;;GAGG;AACH,MAAM,eAAe,GACnB,uFAAuF;IACvF,2FAA2F;IAC3F,4FAA4F,CAAC;AAE/F,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,eAAe;;;;;;;;;;;8BAWtB,CAAC;AAE/B,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,eAAe;;;;;;;;;;+BAUtB,CAAC;AAEhC,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,eAAe;;;;;;;;;;;;;+BAatB,CAAC;AAEhC,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,eAAe;;;;;;;;;;qCAUtB,CAAC"}

package/dist/code/benchmark/orchestrator-factory.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Shared ModelOrchestrator builder for the benchmark, used by both the CLI
+ * command and the HTTP route. Mirrors the model-construction logic in the CLI
+ * `run` command so benchmarks exercise exactly the same model stack a real
+ * code-mode session would.
+ */
+import { ModelOrchestrator } from '../../models/orchestrator.js';
+/** Subset of the stored ModelConfig shape we need to build a client. */
+export interface ModelConfigInput {
+    endpoint: string;
+    apiKey?: string;
+    /** Preferred model id; `defaultModel` is the config-store alias. */
+    model?: string;
+    defaultModel?: string;
+    useHarmonyFormat?: boolean;
+    defaultReasoningEffort?: 'low' | 'medium' | 'high';
+    reasoningEffortStrategy?: 'api_param' | 'system_prompt' | 'both';
+    defaultMaxTokens?: number;
+    useGoogleADC?: boolean;
+}
+export interface OrchestratorInput {
+    reasoning: ModelConfigInput;
+    multimodal?: ModelConfigInput;
+    toolCalling?: ModelConfigInput;
+    /** When true, ping each model first; throws if the reasoning model is unreachable. */
+    testConnectivity?: boolean;
+    /** Optional progress logger. */
+    log?: (msg: string) => void;
+}
+export interface OrchestratorBundle {
+    orchestrator: ModelOrchestrator;
+    /** A short label for reports, e.g. the reasoning model id. */
+    modelLabel: string;
+}
+export declare function createOrchestrator(input: OrchestratorInput): Promise<OrchestratorBundle>;
+//# sourceMappingURL=orchestrator-factory.d.ts.map

package/dist/code/benchmark/orchestrator-factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator-factory.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/orchestrator-factory.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAEjE,wEAAwE;AACxE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oEAAoE;IACpE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,sBAAsB,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACnD,uBAAuB,CAAC,EAAE,WAAW,GAAG,eAAe,GAAG,MAAM,CAAC;IACjE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,sFAAsF;IACtF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gCAAgC;IAChC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,iBAAiB,CAAC;IAChC,8DAA8D;IAC9D,UAAU,EAAE,MAAM,CAAC;CACpB;AAoBD,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAoC9F"}

package/dist/code/benchmark/orchestrator-factory.js

@@ -0,0 +1,57 @@
+/**
+ * Shared ModelOrchestrator builder for the benchmark, used by both the CLI
+ * command and the HTTP route. Mirrors the model-construction logic in the CLI
+ * `run` command so benchmarks exercise exactly the same model stack a real
+ * code-mode session would.
+ */
+import { createModelClient } from '../../models/model-client.js';
+import { ModelOrchestrator } from '../../models/orchestrator.js';
+function resolveModelId(cfg) {
+    return cfg.model || cfg.defaultModel || '';
+}
+function buildClient(cfg, type) {
+    return createModelClient({
+        endpoint: cfg.endpoint,
+        apiKey: cfg.apiKey ?? '',
+        model: resolveModelId(cfg),
+        type,
+        useHarmonyFormat: cfg.useHarmonyFormat,
+        defaultReasoningEffort: cfg.defaultReasoningEffort,
+        reasoningEffortStrategy: cfg.reasoningEffortStrategy,
+        defaultMaxTokens: cfg.defaultMaxTokens,
+        useGoogleADC: cfg.useGoogleADC,
+    });
+}
+export async function createOrchestrator(input) {
+    const log = input.log ?? (() => { });
+    if (!input.reasoning?.endpoint) {
+        throw new Error('A reasoning model configuration is required for the benchmark');
+    }
+    const reasoningModel = buildClient(input.reasoning, 'reasoning');
+    if (input.testConnectivity) {
+        const t = await reasoningModel.testConnectivity();
+        if (!t.success) {
+            throw new Error(`Reasoning model connection failed: ${t.error}`);
+        }
+        log(`Reasoning model connected (${t.latency}ms)`);
+    }
+    let multimodalModel = input.multimodal ? buildClient(input.multimodal, 'multimodal') : undefined;
+    if (multimodalModel && input.testConnectivity) {
+        const t = await multimodalModel.testConnectivity();
+        if (!t.success) {
+            log(`Multimodal model unreachable, continuing without it: ${t.error}`);
+            multimodalModel = undefined;
+        }
+    }
+    let toolCallingModel = input.toolCalling ? buildClient(input.toolCalling, 'tool-calling') : undefined;
+    if (toolCallingModel && input.testConnectivity) {
+        const t = await toolCallingModel.testConnectivity();
+        if (!t.success) {
+            log(`Tool-calling model unreachable, continuing without it: ${t.error}`);
+            toolCallingModel = undefined;
+        }
+    }
+    const orchestrator = new ModelOrchestrator({ reasoningModel, multimodalModel, toolCallingModel });
+    return { orchestrator, modelLabel: resolveModelId(input.reasoning) || 'unknown' };
+}
+//# sourceMappingURL=orchestrator-factory.js.map

package/dist/code/benchmark/orchestrator-factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator-factory.js","sourceRoot":"","sources":["../../../src/code/benchmark/orchestrator-factory.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAgCjE,SAAS,cAAc,CAAC,GAAqB;IAC3C,OAAO,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;AAC7C,CAAC;AAED,SAAS,WAAW,CAAC,GAAqB,EAAE,IAAiD;IAC3F,OAAO,iBAAiB,CAAC;QACvB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QACxB,KAAK,EAAE,cAAc,CAAC,GAAG,CAAC;QAC1B,IAAI;QACJ,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,sBAAsB,EAAE,GAAG,CAAC,sBAAsB;QAClD,uBAAuB,EAAE,GAAG,CAAC,uBAAuB;QACpD,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,YAAY,EAAE,GAAG,CAAC,YAAY;KAC/B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAwB;IAC/D,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAEpC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACjE,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAClD,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,GAAG,CAAC,8BAA8B,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACjG,IAAI,eAAe,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;QACnD,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,GAAG,CAAC,wDAAwD,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YACvE,eAAe,GAAG,SAAS,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtG,IAAI,gBAAgB,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,EAAE,CAAC;QACpD,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,GAAG,CAAC,0DAA0D,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YACzE,gBAAgB,GAAG,SAAS,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC,EAAE,cAAc,EAAE,eAAe,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAClG,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,SAAS,EAAE,CAAC;AACpF,CAAC"}

package/dist/code/benchmark/report.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Report formatting for benchmark results — a chalk table for the CLI and a
+ * plain JSON serializer for files / HTTP.
+ */
+import type { SuiteResult } from './types.js';
+export declare function formatReportForCLI(suite: SuiteResult): string;
+export declare function toReportJSON(suite: SuiteResult): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/code/benchmark/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/code/benchmark/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAc,MAAM,YAAY,CAAC;AAgB1D,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAgG7D;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAEvD"}

package/dist/code/benchmark/report.js

@@ -0,0 +1,103 @@
+/**
+ * Report formatting for benchmark results — a chalk table for the CLI and a
+ * plain JSON serializer for files / HTTP.
+ */
+import chalk from 'chalk';
+function fmtMs(ms) {
+    return ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(1)}s`;
+}
+function pad(s, n) {
+    return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
+}
+function statusCell(r) {
+    if (r.passed)
+        return chalk.green('PASS');
+    const reason = r.reason ?? 'fail';
+    return chalk.red('FAIL') + chalk.gray(` ${reason}`);
+}
+export function formatReportForCLI(suite) {
+    const lines = [];
+    lines.push('');
+    const scored = suite.scoring === 'scored';
+    lines.push(chalk.bold(`  Jiva Code-Mode Benchmark — ${suite.suiteName ?? 'taskstore'}`));
+    if (suite.model)
+        lines.push(chalk.gray(`  Model: ${suite.model}`));
+    lines.push(chalk.gray(`  ${suite.startedAt} → ${suite.finishedAt}`));
+    lines.push('');
+    // Header
+    lines.push(chalk.gray('  ' +
+        pad('Tier', 5) +
+        pad('Task', 26) +
+        pad('Status', 18) +
+        pad('Iters', 7) +
+        pad('Tests', 9) +
+        pad('Tokens', 10) +
+        'Time'));
+    lines.push(chalk.gray('  ' + '─'.repeat(86)));
+    for (const r of suite.tasks) {
+        const iters = `${r.iterations}${r.hitMaxIterations ? '!' : ''}`;
+        const tests = `${r.testsPassed}/${r.testsPassed + r.testsFailed}`;
+        const tokens = r.tokenUsage?.totalTokens ? String(r.tokenUsage.totalTokens) : '-';
+        lines.push('  ' +
+            pad(String(r.tier), 5) +
+            pad(r.title, 26) +
+            pad(statusCell(r), 18 + 10) + // +10 to absorb chalk color codes width
+            pad(iters, 7) +
+            pad(tests, 9) +
+            pad(tokens, 10) +
+            fmtMs(r.wallTimeMs));
+    }
+    lines.push(chalk.gray('  ' + '─'.repeat(86)));
+    if (scored) {
+        // Headline for scored suites is the spec-test pass-rate, not all-or-nothing.
+        const score = `${suite.totalTestsPassed}/${suite.totalTestsRun} tests (${suite.scorePct}%)`;
+        const colour = suite.scorePct >= 90 ? chalk.green : suite.scorePct >= 50 ? chalk.yellow : chalk.red;
+        lines.push(colour(`  Score: ${score}`));
+    }
+    else {
+        const ratio = `${suite.passed}/${suite.totalTasks}`;
+        const summary = suite.passed === suite.totalTasks ? chalk.green(`  ${ratio} passed`) : chalk.yellow(`  ${ratio} passed`);
+        lines.push(summary + chalk.gray(`  ·  highest tier passed: ${suite.highestTierPassed}`));
+    }
+    lines.push(chalk.gray(`  total time: ${fmtMs(suite.totalWallTimeMs)}` +
+        (suite.totalTokens ? `  ·  total tokens: ${suite.totalTokens}` : '')));
+    // Scored suites: list the specific spec tests the model missed (the capability gaps).
+    if (scored) {
+        const missed = suite.tasks.flatMap((t) => t.failingTests ?? []);
+        if (missed.length > 0) {
+            lines.push('');
+            lines.push(chalk.bold(`  Missed (${missed.length})`));
+            for (const name of missed)
+                lines.push(chalk.red('  ✗ ') + chalk.gray(name));
+        }
+    }
+    // Output-length limitation flag: failures driven by output-token truncation are a
+    // model capacity limit (e.g. a hard 4096-token cap), distinct from a logic gap.
+    const outputLimited = suite.tasks.filter((t) => t.outputLimited);
+    if (outputLimited.length > 0) {
+        lines.push('');
+        lines.push(chalk.yellow('  ⚠ Output-length limited') +
+            chalk.gray(`  — ${outputLimited.length} task(s) failed after hitting the model's output-token limit:`));
+        for (const t of outputLimited) {
+            lines.push(chalk.gray(`      ${t.id} (${t.truncationEvents}× truncation) — couldn't emit a large enough response`));
+        }
+    }
+    // Surface failures with their diagnostic notes (infra/agent errors, not per-test).
+    const failures = suite.tasks.filter((t) => !t.passed);
+    if (failures.length > 0) {
+        lines.push('');
+        lines.push(chalk.bold('  Failures'));
+        for (const f of failures) {
+            const tag = f.outputLimited ? chalk.yellow(' [output-limited]') : '';
+            lines.push(chalk.red(`  • ${f.id}`) + chalk.gray(` (${f.reason ?? 'fail'})`) + tag);
+            if (f.notes)
+                lines.push(chalk.gray(`    ${f.notes}`));
+        }
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+export function toReportJSON(suite) {
+    return JSON.stringify(suite, null, 2);
+}
+//# sourceMappingURL=report.js.map