@castlemilk/omega 0.6.10 → 0.6.12

package/dist/cli.js

@@ -66,20 +66,20 @@ function useTasks(pollMs = 1e3) {
         const data = await res.json();
         setConnected(true);
         const current = {};
-        for (const task of data) {
-          current[task.id] = task;
-          const prev = previousTasks.current[task.id];
+        for (const task2 of data) {
+          current[task2.id] = task2;
+          const prev = previousTasks.current[task2.id];
           if (!prev) {
-            addLog(`New task queued: "${task.title}"`, "info");
-          } else if (prev.status !== task.status) {
-            if (task.status === "in_progress") {
-              const provider = task.provider ?? "unknown";
-              const model = task.model ?? "unknown";
-              addLog(`LLM picked up "${task.title}" \u2192 ${provider}/${model}`, "info");
-            } else if (task.status === "done") {
-              addLog(`Completed: "${task.title}"`, "success");
-            } else if (task.status === "failed") {
-              addLog(`Failed: "${task.title}"${task.error ? ` \u2014 ${task.error}` : ""}`, "error");
+            addLog(`New task queued: "${task2.title}"`, "info");
+          } else if (prev.status !== task2.status) {
+            if (task2.status === "in_progress") {
+              const provider = task2.provider ?? "unknown";
+              const model = task2.model ?? "unknown";
+              addLog(`LLM picked up "${task2.title}" \u2192 ${provider}/${model}`, "info");
+            } else if (task2.status === "done") {
+              addLog(`Completed: "${task2.title}"`, "success");
+            } else if (task2.status === "failed") {
+              addLog(`Failed: "${task2.title}"${task2.error ? ` \u2014 ${task2.error}` : ""}`, "error");
             }
           }
         }
@@ -231,16 +231,16 @@ function TaskList({
       children: [
         header,
         visible.length === 0 && /* @__PURE__ */ jsx(Text, { dimColor: true, children: "No tasks yet. Create one in the web UI." }),
-        visible.map((task) => {
-          const providerText = task.provider ? `${task.provider}/${task.model ?? ""}` : "-";
+        visible.map((task2) => {
+          const providerText = task2.provider ? `${task2.provider}/${task2.model ?? ""}` : "-";
           const titleMax = Math.max(10, columns - 62);
           return /* @__PURE__ */ jsxs(Box, { flexDirection: "row", gap: 1, children: [
-            /* @__PURE__ */ jsx(Text, { color: statusColor(task.status), children: statusSymbol(task.status) }),
-            /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { dimColor: true, children: formatTime(new Date(task.updatedAt)) }) }),
-            /* @__PURE__ */ jsx(Box, { width: 12, children: /* @__PURE__ */ jsx(Text, { color: statusColor(task.status), bold: true, children: task.status.toUpperCase() }) }),
+            /* @__PURE__ */ jsx(Text, { color: statusColor(task2.status), children: statusSymbol(task2.status) }),
+            /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { dimColor: true, children: formatTime(new Date(task2.updatedAt)) }) }),
+            /* @__PURE__ */ jsx(Box, { width: 12, children: /* @__PURE__ */ jsx(Text, { color: statusColor(task2.status), bold: true, children: task2.status.toUpperCase() }) }),
             /* @__PURE__ */ jsx(Box, { width: 24, children: /* @__PURE__ */ jsx(Text, { dimColor: true, children: truncate(providerText, 23) }) }),
-            /* @__PURE__ */ jsx(Box, { flexGrow: 1, children: /* @__PURE__ */ jsx(Text, { children: truncate(task.title, titleMax) }) })
-          ] }, task.id);
+            /* @__PURE__ */ jsx(Box, { flexGrow: 1, children: /* @__PURE__ */ jsx(Text, { children: truncate(task2.title, titleMax) }) })
+          ] }, task2.id);
         })
       ]
     }
@@ -318,6 +318,76 @@ var init_tui = __esm({
   }
 });
 
+// ../agent/dist/prompt-versioning.js
+var prompt_versioning_exports = {};
+__export(prompt_versioning_exports, {
+  envForPrompts: () => envForPrompts,
+  hashPrompts: () => hashPrompts,
+  loadCurrentPrompts: () => loadCurrentPrompts,
+  readPromptsSource: () => readPromptsSource,
+  saveCurrentPrompts: () => saveCurrentPrompts
+});
+import fs7 from "node:fs/promises";
+import path10 from "node:path";
+import crypto from "node:crypto";
+import { fileURLToPath as fileURLToPath4 } from "node:url";
+function hashString(value) {
+  return crypto.createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+function hashPrompts(input) {
+  return hashString([input.systemPrompt, input.textToolsPrompt, input.planningPrompt ?? ""].join("\n---\n"));
+}
+async function readPromptsSource() {
+  const source = await fs7.readFile(PROMPTS_PATH, "utf-8");
+  const systemMatch = /export const AGENT_SYSTEM_PROMPT =\s*(?:loadPromptFromEnv\('OMEGA_SYSTEM_PROMPT'\) \?\?\s*)?`([\s\S]*?)`;/m.exec(source);
+  const textToolsMatch = /export const TEXT_TOOLS_SYSTEM_PROMPT =\s*(?:loadPromptFromEnv\('OMEGA_TEXT_TOOLS_PROMPT'\) \?\?\s*)?`([\s\S]*?)`;/m.exec(source);
+  return {
+    systemPrompt: systemMatch?.[1] ?? "",
+    textToolsPrompt: textToolsMatch?.[1] ?? ""
+  };
+}
+async function loadCurrentPrompts() {
+  const { systemPrompt, textToolsPrompt } = await readPromptsSource();
+  const plannerSource = await fs7.readFile(PLANNER_PATH, "utf-8");
+  const planningMatch = /const PLAN_PROMPT = `([\s\S]*?)`;/m.exec(plannerSource);
+  const planningPrompt = planningMatch?.[1];
+  return {
+    name: `auto-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`,
+    sourcePath: PROMPTS_PATH,
+    systemPrompt,
+    textToolsPrompt,
+    planningPrompt,
+    hash: hashPrompts({ systemPrompt, textToolsPrompt, planningPrompt })
+  };
+}
+async function saveCurrentPrompts(input) {
+  const source = await fs7.readFile(PROMPTS_PATH, "utf-8");
+  const replaceConst = (name, value) => {
+    const pattern = new RegExp(`export const ${name} =\\s*(?:process\\.env\\.OMEGA_[A-Z_]+_PROMPT \\?\\?\\s*)?\`[\\s\\S]*?\`;`, "m");
+    return source.replace(pattern, `export const ${name} =
+  process.env.OMEGA_${name.replace(/([A-Z])/g, "_$1").toUpperCase()}_PROMPT ??
+  \`${value}\`;`);
+  };
+  let updated = replaceConst("AGENT_SYSTEM_PROMPT", input.systemPrompt);
+  updated = replaceConst("TEXT_TOOLS_SYSTEM_PROMPT", input.textToolsPrompt);
+  await fs7.writeFile(PROMPTS_PATH, updated, "utf-8");
+}
+function envForPrompts(input) {
+  return {
+    OMEGA_SYSTEM_PROMPT: input.systemPrompt,
+    OMEGA_TEXT_TOOLS_PROMPT: input.textToolsPrompt
+  };
+}
+var __dirname3, PROMPTS_PATH, PLANNER_PATH;
+var init_prompt_versioning = __esm({
+  "../agent/dist/prompt-versioning.js"() {
+    "use strict";
+    __dirname3 = path10.dirname(fileURLToPath4(import.meta.url));
+    PROMPTS_PATH = path10.resolve(__dirname3, "..", "src", "prompts.ts");
+    PLANNER_PATH = path10.resolve(__dirname3, "..", "src", "planner.ts");
+  }
+});
+
 // ../../apps/cli/src/index.ts
 import { program as program2 } from "commander";
 
@@ -328,10 +398,10 @@ import { Command } from "commander";
 import { program } from "commander";
 function getApiUrl() {
   const opts = program.opts();
-  return opts.api ?? "http://localhost:4000";
+  return opts.api ?? process.env.HARNESS_API_URL ?? "http://localhost:4000";
 }
-async function apiFetch(path4, init) {
-  const url = `${getApiUrl()}${path4}`;
+async function apiFetch(path12, init) {
+  const url = `${getApiUrl()}${path12}`;
   const res = await fetch(url, init);
   const data = await res.json().catch(() => ({}));
   if (!res.ok) {
@@ -446,7 +516,7 @@ var taskFeedCmd = new Command2("feed").description("Feed a task into the harness
 // ../../apps/cli/src/commands/task.ts
 var taskCmd = new Command3("task").description("Manage tasks");
 taskCmd.command("create").description("Create a task").requiredOption("--project <id>", "project id").requiredOption("--title <title>", "task title").option("--description <text>", "task description").option("--complexity <level>", "simple | medium | complex", "simple").option("--tags <tags>", "comma-separated tags").action(async (opts) => {
-  const task = await apiFetch("/tasks", {
+  const task2 = await apiFetch("/tasks", {
     method: "POST",
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify({
@@ -457,7 +527,7 @@ taskCmd.command("create").description("Create a task").requiredOption("--project
       tags: opts.tags ? opts.tags.split(",").map((t) => t.trim()) : []
     })
   });
-  console.log(JSON.stringify(task, null, 2));
+  console.log(JSON.stringify(task2, null, 2));
 });
 taskCmd.command("list").description("List tasks").option("--project <id>", "filter by project id").action(async (opts) => {
   const query = opts.project ? `?projectId=${opts.project}` : "";
@@ -472,14 +542,17 @@ taskCmd.addCommand(taskFeedCmd);
 
 // ../../apps/cli/src/commands/ui.ts
 import { Command as Command4 } from "commander";
+import path3 from "path";
+import open from "open";
+import { render } from "ink";
+import React2 from "react";
+
+// ../../apps/cli/src/lib/server.ts
 import { spawn } from "child_process";
 import { existsSync as existsSync2 } from "fs";
 import net from "net";
 import path2 from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
-import open from "open";
-import { render } from "ink";
-import React2 from "react";
 var __filename2 = fileURLToPath2(import.meta.url);
 var __dirname2 = path2.dirname(__filename2);
 var DEFAULT_PORT = 4e3;
@@ -489,7 +562,7 @@ var MAX_GRPC_PORT = 50061;
 function findAvailablePort(preferred = DEFAULT_PORT, max = MAX_PORT) {
   return new Promise((resolve, reject) => {
     if (preferred > max) {
-      reject(new Error(`No available ports between ${DEFAULT_PORT.toString()} and ${max.toString()}`));
+      reject(new Error(`No available ports between ${preferred.toString()} and ${max.toString()}`));
       return;
     }
     const tester = net.createServer();
@@ -511,11 +584,11 @@ function findAvailablePort(preferred = DEFAULT_PORT, max = MAX_PORT) {
   });
 }
 function startBundledServer(env) {
-  const realServerPath = path2.resolve(__dirname2, "server/dist/index.js");
+  const realServerPath = path2.resolve(__dirname2, "../../server/dist/index.js");
   if (existsSync2(realServerPath)) {
     return spawn(process.execPath, [realServerPath], { stdio: "inherit", env });
   }
-  const legacyServerPath = path2.resolve(__dirname2, "server.js");
+  const legacyServerPath = path2.resolve(__dirname2, "../../server.js");
   if (existsSync2(legacyServerPath)) {
     return spawn(process.execPath, [legacyServerPath], { stdio: "inherit", env });
   }
@@ -537,9 +610,29 @@ async function waitForApi(apiUrl, maxMs = 3e4) {
   }
   throw new Error("Server did not become ready in time");
 }
+async function startHarnessServer(env, options = {}) {
+  const port = await findAvailablePort(options.preferredPort ?? DEFAULT_PORT, MAX_PORT);
+  const grpcPort = await findAvailablePort(options.preferredGrpcPort ?? DEFAULT_GRPC_PORT, MAX_GRPC_PORT);
+  const apiUrl = `http://localhost:${port.toString()}`;
+  const serverEnv = {
+    ...env,
+    PORT: port.toString(),
+    GRPC_PORT: grpcPort.toString(),
+    HARNESS_API_URL: apiUrl
+  };
+  const server = startBundledServer(serverEnv) ?? spawn("pnpm", ["--filter", "@omega/server", "dev"], {
+    stdio: "inherit",
+    shell: true,
+    env: serverEnv
+  });
+  await waitForApi(apiUrl);
+  return { process: server, apiUrl, port, grpcPort };
+}
+
+// ../../apps/cli/src/commands/ui.ts
 async function ensureProject(apiUrl) {
   const cwd = process.cwd();
-  const name = path2.basename(cwd);
+  const name = path3.basename(cwd);
   try {
     const listRes = await fetch(`${apiUrl}/projects`);
     const projects = await listRes.json();
@@ -565,25 +658,16 @@ async function ensureProject(apiUrl) {
 var uiCmd = new Command4("ui").description("Open the harness web UI").option("--no-tui", "Do not open the terminal console alongside the web UI").action(async (options) => {
   const defaultApiUrl = process.env.HARNESS_API_URL ?? `http://localhost:${DEFAULT_PORT.toString()}`;
   let server;
+  let serverStarted = false;
   const alreadyRunning = await isApiReady(defaultApiUrl);
   const useTui = options.tui;
   let apiUrl = defaultApiUrl;
   if (!alreadyRunning) {
-    const port = await findAvailablePort();
-    const grpcPort = await findAvailablePort(DEFAULT_GRPC_PORT, MAX_GRPC_PORT);
-    apiUrl = `http://localhost:${port.toString()}`;
+    const started = await startHarnessServer(process.env);
+    server = started.process;
+    apiUrl = started.apiUrl;
+    serverStarted = true;
     process.env.HARNESS_API_URL = apiUrl;
-    const env = {
-      ...process.env,
-      PORT: port.toString(),
-      GRPC_PORT: grpcPort.toString(),
-      HARNESS_API_URL: apiUrl
-    };
-    server = startBundledServer(env) ?? spawn("pnpm", ["--filter", "@omega/server", "dev"], {
-      stdio: "inherit",
-      shell: true,
-      env
-    });
     server.on("exit", (code) => {
       process.exit(code ?? 0);
     });
@@ -607,7 +691,7 @@ var uiCmd = new Command4("ui").description("Open the harness web UI").option("--
     const { TuiApp: TuiApp2 } = await Promise.resolve().then(() => (init_tui(), tui_exports));
     const { waitUntilExit } = render(React2.createElement(TuiApp2));
     await waitUntilExit();
-    if (server && !alreadyRunning) {
+    if (serverStarted && server) {
       server.kill();
     }
   }
@@ -628,7 +712,7 @@ var consoleCmd = new Command5("console").description("Open the harness TUI conso
 
 // ../../apps/cli/src/commands/skill.ts
 import { Command as Command6 } from "commander";
-import path3 from "path";
+import path4 from "path";
 import { fileURLToPath as fileURLToPath3 } from "url";
 
 // ../skills/src/parse.ts
@@ -695,7 +779,8 @@ export class ${className} {
 
   async execute(args: Record<string, unknown>): Promise<string> {
     const prompt = ${instructionsLiteral} + '\\n\\nArgs:\\n' + JSON.stringify(args, null, 2);
-    return this.provider.send(prompt, { system: ${descriptionLiteral} });
+    const response = await this.provider.send(prompt, { system: ${descriptionLiteral} });
+    return response.content;
   }
 }
 `;
@@ -730,22 +815,22 @@ async function registerSkill(sourcePath, outputDir) {
 // ../../apps/cli/src/commands/skill.ts
 function getOutputDir() {
   if (process.env.OMEGA_HARNESS_ROOT) {
-    return path3.join(path3.resolve(process.env.OMEGA_HARNESS_ROOT), "packages/skills/src");
+    return path4.join(path4.resolve(process.env.OMEGA_HARNESS_ROOT), "packages/skills/src");
   }
   try {
     const __filename3 = fileURLToPath3(import.meta.url);
-    const __dirname3 = path3.dirname(__filename3);
-    return path3.join(path3.resolve(__dirname3, "../../../.."), "packages/skills/src");
+    const __dirname4 = path4.dirname(__filename3);
+    return path4.join(path4.resolve(__dirname4, "../../../.."), "packages/skills/src");
   } catch {
-    return path3.join(process.cwd(), "harness-skills");
+    return path4.join(process.cwd(), "harness-skills");
   }
 }
 var skillCmd = new Command6("skill").description("Manage skill artifacts");
 skillCmd.command("generate").description("Generate a harness adapter from a SKILL.md file").argument("<source>", "path to SKILL.md").action(async (source) => {
   const outputDir = getOutputDir();
-  const manifest = await registerSkill(path3.resolve(source), outputDir);
+  const manifest = await registerSkill(path4.resolve(source), outputDir);
   console.log(`Generated adapter for skill: ${manifest.name}`);
-  console.log(`Output directory: ${path3.join(outputDir, "generated")}`);
+  console.log(`Output directory: ${path4.join(outputDir, "generated")}`);
 });
 
 // ../../apps/cli/src/commands/agent.ts
@@ -755,7 +840,7 @@ agentCmd.command("run").description("Run a single agent task").requiredOption("-
   async (opts) => {
     const tags = ["agent"];
     if (opts.autoPublish) tags.push("publish");
-    const task = await apiFetch("/tasks", {
+    const task2 = await apiFetch("/tasks", {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
@@ -766,8 +851,8 @@ agentCmd.command("run").description("Run a single agent task").requiredOption("-
         tags
       })
     });
-    console.log(JSON.stringify(task, null, 2));
-    const result = await apiFetch(`/tasks/${task.id}/run`, { method: "POST" });
+    console.log(JSON.stringify(task2, null, 2));
+    const result = await apiFetch(`/tasks/${task2.id}/run`, { method: "POST" });
     console.log(JSON.stringify(result, null, 2));
   }
 );
@@ -779,7 +864,7 @@ agentCmd.command("loop").description("Continuously run agent tasks").requiredOpt
       const title = opts.prompt ?? "Improve the Omega harness: fix a TODO, add a test, refactor a function, or improve documentation";
       const tags = ["agent", "self-improve"];
       if (opts.autoPublish) tags.push("publish");
-      const task = await apiFetch("/tasks", {
+      const task2 = await apiFetch("/tasks", {
         method: "POST",
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({
@@ -790,8 +875,8 @@ agentCmd.command("loop").description("Continuously run agent tasks").requiredOpt
           tags
         })
       });
-      console.log(`Created task ${task.id}`);
-      const result = await apiFetch(`/tasks/${task.id}/run`, { method: "POST" });
+      console.log(`Created task ${task2.id}`);
+      const result = await apiFetch(`/tasks/${task2.id}/run`, { method: "POST" });
       console.log(JSON.stringify(result, null, 2));
       return result;
     };
@@ -817,6 +902,1607 @@ agentCmd.command("diffs").description("Show diffs for a task").argument("<id>",
   console.log(JSON.stringify(diffs, null, 2));
 });
 
+// ../../apps/cli/src/commands/bench.ts
+import { Command as Command8 } from "commander";
+import fs8 from "node:fs/promises";
+import path11 from "node:path";
+import os from "node:os";
+
+// ../bench/dist/api-client.js
+var ApiError = class extends Error {
+  status;
+  constructor(status, message) {
+    super(message);
+    this.status = status;
+  }
+};
+async function apiFetch2(url, init) {
+  const res = await fetch(url, init);
+  if (!res.ok) {
+    throw new ApiError(res.status, `${init?.method ?? "GET"} ${url} -> ${String(res.status)}`);
+  }
+  return res.json();
+}
+async function ensureProject2(apiUrl, name, path12) {
+  const res = await fetch(`${apiUrl}/projects`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ name, path: path12 })
+  });
+  if (res.status === 409) {
+    const projects = await fetch(`${apiUrl}/projects?path=${encodeURIComponent(path12)}`).then((r) => r.json());
+    const existing = projects.find((p) => p.path === path12);
+    if (existing)
+      return existing;
+  }
+  if (!res.ok)
+    throw new ApiError(res.status, `create project failed: ${String(res.status)}`);
+  return res.json();
+}
+async function createTask(apiUrl, projectId, title, options = {}) {
+  return apiFetch2(`${apiUrl}/tasks`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      projectId,
+      title,
+      description: options.description,
+      complexity: options.complexity ?? "simple",
+      tags: options.tags ?? ["benchmark"]
+    })
+  });
+}
+async function runTask(apiUrl, taskId) {
+  await apiFetch2(`${apiUrl}/tasks/${taskId}/run`, { method: "POST" });
+}
+async function getTask(apiUrl, taskId) {
+  return apiFetch2(`${apiUrl}/tasks/${taskId}`);
+}
+async function waitForTask(apiUrl, taskId, timeoutMs = 12e4) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const task2 = await getTask(apiUrl, taskId);
+    if (task2.status === "done" || task2.status === "failed")
+      return task2;
+    await new Promise((r) => setTimeout(r, 300));
+  }
+  return { status: "timeout", error: "Task did not finish in time" };
+}
+async function getAgentRun(apiUrl, taskId) {
+  try {
+    return await apiFetch2(`${apiUrl}/tasks/${taskId}/agent-run`);
+  } catch {
+    return void 0;
+  }
+}
+async function getDiffs(apiUrl, taskId) {
+  try {
+    return await apiFetch2(`${apiUrl}/tasks/${taskId}/diffs`);
+  } catch {
+    return [];
+  }
+}
+async function getTraceFlow(apiUrl, taskId) {
+  try {
+    return await apiFetch2(`${apiUrl}/tasks/${taskId}/trace-flow`);
+  } catch {
+    return void 0;
+  }
+}
+function countSpans(node) {
+  let count = 1;
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) {
+      count += countSpans(child);
+    }
+  }
+  return count;
+}
+
+// ../bench/dist/runner.js
+import fs from "node:fs/promises";
+import path5 from "node:path";
+import { execSync } from "node:child_process";
+
+// ../bench/dist/analyse.js
+function walkSpans(traceFlow, callback) {
+  if (!traceFlow)
+    return;
+  function visit(span) {
+    callback?.(span);
+    for (const child of span.children) {
+      visit(child);
+    }
+  }
+  for (const span of traceFlow.spans) {
+    visit(span);
+  }
+}
+function findSpans(traceFlow, predicate) {
+  const matches = [];
+  walkSpans(traceFlow, (span) => {
+    if (predicate(span))
+      matches.push(span);
+  });
+  return matches;
+}
+function spanError(span) {
+  if (span.status !== "error")
+    return void 0;
+  const err = span.attributes?.error;
+  return typeof err === "string" ? err : void 0;
+}
+function toolSpans(traceFlow) {
+  return findSpans(traceFlow, (s) => s.name.startsWith("agent.tool."));
+}
+function providerSpans(traceFlow) {
+  return findSpans(traceFlow, (s) => s.name === "provider.send");
+}
+function planSpan(traceFlow) {
+  return findSpans(traceFlow, (s) => s.name === "agent.plan").shift();
+}
+function validationSummary(result) {
+  if (!result.agentRun?.validationSummary)
+    return void 0;
+  try {
+    return JSON.parse(result.agentRun.validationSummary);
+  } catch {
+    return void 0;
+  }
+}
+function classifyFailure(result, traceFlow) {
+  const evidence = [];
+  if (result.status === "timeout") {
+    return {
+      category: "timeout",
+      rootCause: "Task did not finish within the timeout window.",
+      evidence: [`durationMs: ${String(result.durationMs)}`]
+    };
+  }
+  const validation = validationSummary(result);
+  if (validation?.allPassed === false) {
+    const failedSteps = [];
+    for (const key of ["lint", "test", "build"]) {
+      const step = validation[key];
+      if (step && !step.success) {
+        failedSteps.push(key);
+      }
+    }
+    return {
+      category: "validation_failure",
+      rootCause: `Validation failed: ${failedSteps.length > 0 ? failedSteps.join(", ") : "unspecified step"}.`,
+      evidence: failedSteps.map((s) => `${s} did not pass`)
+    };
+  }
+  const tools = toolSpans(traceFlow);
+  const failedTools = tools.filter((s) => s.status === "error");
+  if (failedTools.length > 0) {
+    const first = failedTools[0];
+    const toolName = first.name.replace("agent.tool.", "");
+    const err = spanError(first) ?? "unknown error";
+    evidence.push(`${toolName} failed: ${err}`);
+    if (toolName === "edit_file" || err.includes("old_string")) {
+      return {
+        category: "tool_misuse",
+        rootCause: "edit_file failed because the old_string did not match. The agent may not have read the file first or copied the exact text.",
+        evidence
+      };
+    }
+    return {
+      category: "tool_misuse",
+      rootCause: `Tool ${toolName} was invoked but failed.`,
+      evidence
+    };
+  }
+  const providers = providerSpans(traceFlow);
+  const failedProviders = providers.filter((s) => s.status === "error");
+  if (failedProviders.length > 0) {
+    const err = spanError(failedProviders[0]) ?? "provider error";
+    if (err.includes("JSON") || err.includes("parse") || err.includes("Unexpected token")) {
+      return {
+        category: "parse_error",
+        rootCause: "Provider response could not be parsed as the expected format.",
+        evidence: [err]
+      };
+    }
+  }
+  const plan = planSpan(traceFlow);
+  if (plan) {
+    const planSteps = plan.attributes?.planSteps;
+    const planStepsCount = typeof planSteps === "number" ? planSteps : 0;
+    if (planStepsCount === 0) {
+      return {
+        category: "plan_error",
+        rootCause: "The planner produced an empty plan with no actionable steps.",
+        evidence: ["planSteps: 0"]
+      };
+    }
+    const providerTurns = providers.length;
+    if (providerTurns > 10 && !result.evaluation.passed) {
+      return {
+        category: "plan_error",
+        rootCause: "The agent took many turns without finishing; the plan may be stuck or too vague.",
+        evidence: [`provider turns: ${String(providerTurns)}`, `plan steps: ${String(planStepsCount)}`]
+      };
+    }
+  }
+  const message = result.evaluation.message ?? result.agentRun?.resultStatus ?? "unknown";
+  return {
+    category: "unknown",
+    rootCause: `Failure cause could not be classified: ${message}.`,
+    evidence: [message]
+  };
+}
+function pickFocusResult(results, traceFlows) {
+  const failed = results.filter((r) => !r.evaluation.passed);
+  if (failed.length === 0)
+    return void 0;
+  const seenCategories = /* @__PURE__ */ new Set();
+  for (const result of failed) {
+    const analysis = result.failureAnalysis ?? classifyFailure(result, traceFlows.get(result.harnessTaskId));
+    if (!seenCategories.has(analysis.category)) {
+      seenCategories.add(analysis.category);
+      return result;
+    }
+  }
+  return failed[0];
+}
+function summariseFailures(results, traceFlows) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    if (result.evaluation.passed)
+      continue;
+    const analysis = result.failureAnalysis ?? classifyFailure(result, traceFlows.get(result.harnessTaskId));
+    const entry = counts.get(analysis.category) ?? { count: 0, examples: [] };
+    entry.count++;
+    if (entry.examples.length < 3) {
+      entry.examples.push(`${result.task.name}: ${analysis.rootCause}`);
+    }
+    counts.set(analysis.category, entry);
+  }
+  return Array.from(counts.entries()).map(([category, data]) => ({ category, ...data })).sort((a, b) => b.count - a.count);
+}
+
+// ../bench/dist/runner.js
+function countAllSpans(traceFlow) {
+  if (!traceFlow)
+    return 0;
+  return traceFlow.spans.reduce((acc, span) => acc + countSpans(span), 0);
+}
+function ensureGitRepo(repoPath) {
+  try {
+    execSync("git rev-parse --git-dir", { cwd: repoPath, stdio: "ignore" });
+    return;
+  } catch {
+  }
+  execSync("git init", { cwd: repoPath, stdio: "ignore" });
+  execSync('git config user.email "bench@omega.local"', { cwd: repoPath, stdio: "ignore" });
+  execSync('git config user.name "Omega Bench"', { cwd: repoPath, stdio: "ignore" });
+  execSync("git add .", { cwd: repoPath, stdio: "ignore" });
+  execSync('git commit -m "bench init"', { cwd: repoPath, stdio: "ignore" });
+}
+function sumUsage(a, b) {
+  return {
+    promptTokens: (a.promptTokens ?? 0) + (b?.promptTokens ?? 0),
+    completionTokens: (a.completionTokens ?? 0) + (b?.completionTokens ?? 0),
+    totalTokens: (a.totalTokens ?? 0) + (b?.totalTokens ?? 0)
+  };
+}
+async function runBenchmark(tasks, options) {
+  const { apiUrl, suiteName, timeoutMs = 12e4, projectPrefix = "bench" } = options;
+  const report = {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    suite: suiteName,
+    total: tasks.length,
+    passed: 0,
+    failed: 0,
+    timeouts: 0,
+    totalDurationMs: 0,
+    totalUsage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
+    results: []
+  };
+  const baseDir = path5.join("/tmp", `omega-bench-${String(Date.now())}`);
+  await fs.mkdir(baseDir, { recursive: true });
+  for (const task2 of tasks) {
+    const start = Date.now();
+    let harnessTaskId = "";
+    let status = "failed";
+    let agentRun;
+    let diffs = [];
+    let traceFlow;
+    let evaluation = { passed: false, message: "Task did not complete" };
+    let projectId = "";
+    let projectPath = "";
+    try {
+      projectPath = path5.join(baseDir, task2.id);
+      await fs.mkdir(projectPath, { recursive: true });
+      if (task2.setup) {
+        await task2.setup(projectPath);
+      }
+      ensureGitRepo(projectPath);
+      const project = await ensureProject2(apiUrl, `${projectPrefix}-${task2.id}`, projectPath);
+      projectId = project.id;
+      const harnessTask = await createTask(apiUrl, project.id, task2.title, {
+        description: task2.description,
+        complexity: task2.complexity ?? "simple",
+        tags: ["benchmark", "agent"]
+      });
+      harnessTaskId = harnessTask.id;
+      await runTask(apiUrl, harnessTask.id);
+      const finished = await waitForTask(apiUrl, harnessTask.id, timeoutMs);
+      status = finished.status === "timeout" ? "timeout" : finished.status;
+      [agentRun, diffs, traceFlow] = await Promise.all([
+        getAgentRun(apiUrl, harnessTask.id),
+        getDiffs(apiUrl, harnessTask.id),
+        getTraceFlow(apiUrl, harnessTask.id)
+      ]);
+      evaluation = await task2.evaluate({
+        apiUrl,
+        taskId: harnessTask.id,
+        projectPath,
+        projectId,
+        agentRun,
+        diffs,
+        traceFlow
+      });
+    } catch (err) {
+      evaluation = {
+        passed: false,
+        message: err instanceof Error ? err.message : String(err)
+      };
+    }
+    const durationMs = Date.now() - start;
+    const usage = agentRun ? {
+      promptTokens: agentRun.promptTokens,
+      completionTokens: agentRun.completionTokens,
+      totalTokens: agentRun.totalTokens
+    } : void 0;
+    const result = {
+      task: task2,
+      harnessTaskId,
+      durationMs,
+      status,
+      evaluation,
+      agentRun,
+      spanCount: countAllSpans(traceFlow),
+      usage,
+      failureAnalysis: evaluation.passed ? void 0 : classifyFailure({ task: task2, harnessTaskId, durationMs, status, evaluation, agentRun, spanCount: countAllSpans(traceFlow), usage }, traceFlow)
+    };
+    if (status === "timeout")
+      report.timeouts++;
+    else if (evaluation.passed)
+      report.passed++;
+    else
+      report.failed++;
+    report.totalDurationMs += durationMs;
+    report.totalUsage = sumUsage(report.totalUsage, usage);
+    report.results.push(result);
+    options.onProgress?.(result);
+  }
+  return report;
+}
+
+// ../bench/dist/report.js
+import fs2 from "node:fs/promises";
+import path6 from "node:path";
+function nowIso() {
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+}
+function formatDuration(ms) {
+  if (ms < 1e3)
+    return `${String(Math.round(ms))}ms`;
+  return `${(ms / 1e3).toFixed(1)}s`;
+}
+function resultLine(r, idx) {
+  const symbol = r.evaluation.passed ? "\u2713" : "\u2717";
+  const status = r.status === "timeout" ? "timeout" : r.status;
+  const score = r.evaluation.score !== void 0 ? ` (score ${String(r.evaluation.score)})` : "";
+  const tokens = r.usage?.totalTokens !== void 0 ? ` (${String(r.usage.totalTokens)} tokens)` : "";
+  const category = r.failureAnalysis ? ` [${r.failureAnalysis.category}]` : "";
+  const msg = r.evaluation.message ? ` \u2014 ${r.evaluation.message}` : "";
+  return `${String(idx + 1)}. ${symbol} ${r.task.name} [${status}]${category} ${formatDuration(r.durationMs)}${tokens}${score}${msg}`;
+}
+function usageLine(usage) {
+  return `Tokens: ${String(usage.totalTokens ?? 0)} total (${String(usage.promptTokens ?? 0)} prompt, ${String(usage.completionTokens ?? 0)} completion)`;
+}
+async function writeReport(report, outputDir = ".omega/reports") {
+  await fs2.mkdir(outputDir, { recursive: true });
+  const ts = nowIso();
+  const jsonFile2 = path6.join(outputDir, `benchmark-${ts}.json`);
+  const mdFile = path6.join(outputDir, `benchmark-${ts}.md`);
+  await fs2.writeFile(jsonFile2, JSON.stringify(report, null, 2), "utf-8");
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const md = [
+    "# Omega Benchmark Report",
+    "",
+    `- Suite: ${report.suite}`,
+    `- Timestamp: ${report.timestamp}`,
+    `- Total: ${String(report.total)}`,
+    `- Passed: ${String(report.passed)}`,
+    `- Failed: ${String(report.failed)}`,
+    `- Timeouts: ${String(report.timeouts)}`,
+    `- Pass rate: ${String(passRate2)}%`,
+    `- Total duration: ${formatDuration(report.totalDurationMs)}`,
+    `- ${usageLine(report.totalUsage)}`,
+    "",
+    "## Results",
+    "",
+    ...report.results.map((r, i) => resultLine(r, i)),
+    "",
+    "## Details",
+    "",
+    "```json",
+    JSON.stringify(report, null, 2),
+    "```",
+    ""
+  ].join("\n");
+  await fs2.writeFile(mdFile, md, "utf-8");
+  return jsonFile2;
+}
+function printSummary(report) {
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  console.log(`
+Benchmark: ${report.suite}`);
+  console.log(`Total: ${String(report.total)} | Passed: ${String(report.passed)} | Failed: ${String(report.failed)} | Timeouts: ${String(report.timeouts)}`);
+  console.log(`Pass rate: ${String(passRate2)}%`);
+  console.log(`Duration: ${formatDuration(report.totalDurationMs)}`);
+  console.log(usageLine(report.totalUsage));
+  for (const r of report.results) {
+    console.log(resultLine(r, report.results.indexOf(r)));
+  }
+}
+
+// ../bench/dist/suites/synthetic.js
+import fs4 from "node:fs/promises";
+
+// ../bench/dist/suites/builder.js
+import fs3 from "node:fs/promises";
+import path7 from "node:path";
+function task(spec) {
+  return {
+    id: spec.id,
+    name: spec.name,
+    title: spec.title,
+    description: spec.description,
+    complexity: spec.complexity ?? "simple",
+    setup: spec.setup,
+    evaluate: spec.evaluate ?? expectValidationPassed()
+  };
+}
+function basePackage(extraScripts = {}) {
+  return {
+    name: "synthetic-bench-project",
+    version: "1.0.0",
+    type: "module",
+    scripts: {
+      lint: "node lint.js",
+      test: "node test.js",
+      build: "node build.js",
+      ...extraScripts
+    }
+  };
+}
+async function jsonFile(projectPath, file, data) {
+  await fs3.mkdir(path7.dirname(path7.join(projectPath, file)), { recursive: true });
+  await fs3.writeFile(path7.join(projectPath, file), JSON.stringify(data, null, 2), "utf-8");
+}
+async function codeFile(projectPath, file, content) {
+  await fs3.mkdir(path7.dirname(path7.join(projectPath, file)), { recursive: true });
+  await fs3.writeFile(path7.join(projectPath, file), content, "utf-8");
+}
+async function nodeProject(projectPath, extraScripts = {}) {
+  await jsonFile(projectPath, "package.json", basePackage(extraScripts));
+  await codeFile(projectPath, "lint.js", `console.log('lint ok');
+`);
+  await codeFile(projectPath, "test.js", `console.log('test ok');
+`);
+  await codeFile(projectPath, "build.js", `console.log('build ok');
+`);
+}
+function readValidation(ctx) {
+  if (!ctx.agentRun?.validationSummary)
+    return { allPassed: false };
+  try {
+    const summary = JSON.parse(ctx.agentRun.validationSummary);
+    return { allPassed: Boolean(summary.allPassed), summary };
+  } catch {
+    return { allPassed: false };
+  }
+}
+function expectValidationPassed() {
+  return (ctx) => {
+    const v = readValidation(ctx);
+    return {
+      passed: v.allPassed,
+      message: v.allPassed ? "Validation passed" : "Validation failed"
+    };
+  };
+}
+async function readFile2(projectPath, file) {
+  return fs3.readFile(path7.join(projectPath, file), "utf-8");
+}
+function expectFileContains(file, needle) {
+  return async (ctx) => {
+    const content = await readFile2(ctx.projectPath, file).catch(() => "");
+    const found = typeof needle === "string" ? content.includes(needle) : needle.test(content);
+    return {
+      passed: found,
+      message: found ? `${file} contains required content` : `${file} missing required content`
+    };
+  };
+}
+function combined(...evaluators) {
+  return async (ctx) => {
+    for (const evaluator of evaluators) {
+      const result = await evaluator(ctx);
+      if (!result.passed)
+        return result;
+    }
+    return { passed: true, message: "All checks passed" };
+  };
+}
+
+// ../bench/dist/suites/synthetic.js
+var lintAlwaysPass = `console.log('lint ok');
+`;
+var buildAlwaysPass = `console.log('build ok');
+`;
+function jsTest(name, body) {
+  return `import assert from 'node:assert';
+${body}
+console.log('${name} test passed');
+`;
+}
+function syntheticSuite() {
+  return [
+    // Simple edits
+    task({
+      id: "greet-function",
+      name: "Add a greet function",
+      title: "Add greet.js with a greet function",
+      description: "Create a `greet.js` file that exports a function `greet(name)` returning `Hello, <name>!`.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("greet", `import { greet } from './greet.js';
+assert.strictEqual(greet('World'), 'Hello, World!');`));
+      }
+    }),
+    task({
+      id: "update-config",
+      name: "Update a config value",
+      title: "Update timeout value in config.json",
+      description: "Change `config.json` so that `timeout` is `5000` instead of `1000`.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await jsonFile(projectPath, "config.json", { timeout: 1e3, retries: 3 });
+        await codeFile(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const cfg = JSON.parse(await readFile('./config.json', 'utf-8'));
+if (cfg.timeout !== 5000) { console.error('timeout is ' + cfg.timeout); process.exit(1); }
+console.log('config test passed');
+`);
+      }
+    }),
+    task({
+      id: "add-readme",
+      name: "Add a README",
+      title: "Create README.md with a usage heading",
+      description: "Create a README.md file that contains a `# Usage` heading.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const readme = await readFile('./README.md', 'utf-8').catch(() => '');
+if (!readme.includes('# Usage')) { console.error('README missing Usage heading'); process.exit(1); }
+console.log('readme test passed');
+`);
+      },
+      evaluate: expectFileContains("README.md", "# Usage")
+    }),
+    task({
+      id: "add-npm-script",
+      name: "Add an npm script",
+      title: "Add a `hello` script to package.json",
+      description: 'Add a `hello` script to package.json that prints "hello world".',
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", `import pkg from './package.json' assert { type: 'json' };
+if (pkg.scripts?.hello !== 'echo hello world') { console.error('missing hello script'); process.exit(1); }
+console.log('script test passed');
+`);
+      }
+    }),
+    task({
+      id: "fix-typo",
+      name: "Fix a typo",
+      title: 'Fix spelling of "recieve" to "receive"',
+      description: 'The file src/message.js contains "recieve". Change it to "receive".',
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/message.js", `export const message = 'Please recieve this letter';
+`);
+        await codeFile(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const code = await readFile('./src/message.js', 'utf-8');
+if (code.includes('recieve')) { console.error('typo remains'); process.exit(1); }
+if (!code.includes('receive')) { console.error('correct spelling missing'); process.exit(1); }
+console.log('typo test passed');
+`);
+      }
+    }),
+    task({
+      id: "add-gitignore",
+      name: "Add a .gitignore",
+      title: "Ignore node_modules and .env",
+      description: "Create a .gitignore file that ignores node_modules and .env.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const gitignore = await readFile('./.gitignore', 'utf-8').catch(() => '');
+if (!gitignore.includes('node_modules') || !gitignore.includes('.env')) { console.error('.gitignore incomplete'); process.exit(1); }
+console.log('gitignore test passed');
+`);
+      }
+    }),
+    // Refactoring
+    task({
+      id: "extract-constant",
+      name: "Extract a magic constant",
+      title: "Extract MAX_RETRIES constant",
+      description: "Refactor `src/retry.js` so the magic number `3` is replaced by a top-level constant named `MAX_RETRIES`.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/retry.js", `export function attempts() { return 3; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("constant", `import fs from 'node:fs/promises';
+import { attempts } from './src/retry.js';
+const code = await fs.readFile('./src/retry.js', 'utf-8');
+if (!code.includes('const MAX_RETRIES')) { console.error('MAX_RETRIES constant not found'); process.exit(1); }
+if (attempts() !== 3) { console.error('attempts() changed'); process.exit(1); }`));
+      }
+    }),
+    task({
+      id: "rename-variable",
+      name: "Rename a variable",
+      title: "Rename `usr` to `user` in src/index.js",
+      description: "Rename all occurrences of the variable `usr` to `user` in src/index.js while preserving behavior.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/index.js", `export function welcome(usr) { return 'Welcome, ' + usr; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("rename", `import { readFile } from 'node:fs/promises';
+import { welcome } from './src/index.js';
+const code = await readFile('./src/index.js', 'utf-8');
+if (code.includes('usr')) { console.error('old name remains'); process.exit(1); }
+if (!code.includes('user')) { console.error('new name missing'); process.exit(1); }
+assert.strictEqual(welcome('Ada'), 'Welcome, Ada');`));
+      }
+    }),
+    task({
+      id: "remove-dead-code",
+      name: "Remove dead code",
+      title: "Remove unused helper function",
+      description: "Remove the unused `unusedHelper` function from src/utils.js.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/utils.js", `export function usedHelper() { return 1; }
+function unusedHelper() { return 2; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("dead-code", `import { readFile } from 'node:fs/promises';
+import { usedHelper } from './src/utils.js';
+const code = await readFile('./src/utils.js', 'utf-8');
+if (code.includes('unusedHelper')) { console.error('dead code remains'); process.exit(1); }
+assert.strictEqual(usedHelper(), 1);`));
+      }
+    }),
+    task({
+      id: "split-function",
+      name: "Split a function",
+      title: "Split validateUser into smaller functions",
+      description: "Refactor src/validate.js so there is a separate `isEmailValid` function used by `validateUser`.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/validate.js", `export function validateUser(email) { return email.includes('@'); }
+`);
+        await codeFile(projectPath, "test.js", jsTest("split", `import { readFile } from 'node:fs/promises';
+import { validateUser } from './src/validate.js';
+const code = await readFile('./src/validate.js', 'utf-8');
+if (!code.includes('function isEmailValid')) { console.error('isEmailValid not found'); process.exit(1); }
+assert.strictEqual(validateUser('a@b.com'), true);
+assert.strictEqual(validateUser('bad'), false);`));
+      }
+    }),
+    task({
+      id: "convert-var-to-const",
+      name: "Convert var to const",
+      title: "Replace var with const/let",
+      description: "Update src/index.js to use const and let instead of var.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath, {
+          lint: "node lint.js"
+        });
+        await codeFile(projectPath, "src/index.js", `var x = 1;
+var y = 2;
+export function sum() { return x + y; }
+`);
+        await codeFile(projectPath, "lint.js", `import fs from 'node:fs/promises';
+const code = await fs.readFile('./src/index.js', 'utf-8');
+if (code.includes('var ')) { console.error('var keyword found'); process.exit(1); }
+console.log('lint ok');
+`);
+        await codeFile(projectPath, "test.js", jsTest("var", `import { sum } from './src/index.js';
+assert.strictEqual(sum(), 3);`));
+      }
+    }),
+    // Bug fixes
+    task({
+      id: "fix-null-guard",
+      name: "Add null guard",
+      title: "Guard against null input in getLength",
+      description: "Fix src/strings.js so getLength returns 0 for null/undefined input instead of throwing.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/strings.js", `export function getLength(value) { return value.length; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("null-guard", `import { getLength } from './src/strings.js';
+assert.strictEqual(getLength(null), 0);
+assert.strictEqual(getLength(undefined), 0);
+assert.strictEqual(getLength('abc'), 3);`));
+      }
+    }),
+    task({
+      id: "fix-off-by-one",
+      name: "Fix off-by-one",
+      title: "Fix range to include end value",
+      description: "Fix src/range.js so range(1, 3) returns [1, 2, 3].",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/range.js", `export function range(start, end) { const out = []; for (let i = start; i < end; i++) out.push(i); return out; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("range", `import { range } from './src/range.js';
+assert.deepStrictEqual(range(1, 3), [1, 2, 3]);`));
+      }
+    }),
+    task({
+      id: "fix-async-await",
+      name: "Fix async function",
+      title: "Await the fetch call in loadData",
+      description: "Fix src/api.js so loadData awaits the fetch response before returning JSON.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/api.js", `export async function loadData() { return fetch('data.json').then((r) => r.json()); }
+`);
+        await codeFile(projectPath, "test.js", jsTest("async", `import { readFile } from 'node:fs/promises';
+import { loadData } from './src/api.js';
+const code = await readFile('./src/api.js', 'utf-8');
+if (!code.includes('await')) { console.error('missing await'); process.exit(1); }
+const result = await loadData();
+assert.strictEqual(result.ok, true);`));
+        await jsonFile(projectPath, "data.json", { ok: true });
+      },
+      evaluate: combined(expectFileContains("src/api.js", "await"), expectValidationPassed())
+    }),
+    task({
+      id: "fix-export",
+      name: "Fix missing export",
+      title: "Export the helper function",
+      description: "The function helper in src/lib.js is not exported. Export it so the test can import it.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/lib.js", `function helper() { return 42; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("export", `import { helper } from './src/lib.js';
+assert.strictEqual(helper(), 42);`));
+      }
+    }),
+    task({
+      id: "fix-import-path",
+      name: "Fix import path",
+      title: "Correct the import path in src/main.js",
+      description: "Fix the import path in src/main.js so it points to ./helpers.js.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/helpers.js", `export function greet() { return 'hi'; }
+`);
+        await codeFile(projectPath, "src/main.js", `import { greet } from './helper.js';
+export { greet };
+`);
+        await codeFile(projectPath, "test.js", jsTest("import", `import { greet } from './src/main.js';
+assert.strictEqual(greet(), 'hi');`));
+      }
+    }),
+    // Feature additions
+    task({
+      id: "add-cli-arg-parser",
+      name: "Add CLI arg parser",
+      title: "Add parseArgs function",
+      description: "Create src/cli.js exporting parseArgs that returns an object from process.argv style array.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("cli", `import { parseArgs } from './src/cli.js';
+const args = parseArgs(['node', 'script.js', '--name', 'Ada']);
+assert.strictEqual(args.name, 'Ada');`));
+      }
+    }),
+    task({
+      id: "add-validator",
+      name: "Add an email validator",
+      title: "Add isEmail function",
+      description: "Create src/validators.js exporting isEmail that returns true for valid emails.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("validator", `import { isEmail } from './src/validators.js';
+assert.strictEqual(isEmail('a@b.com'), true);
+assert.strictEqual(isEmail('not-an-email'), false);`));
+      }
+    }),
+    task({
+      id: "add-http-route",
+      name: "Add a simple route",
+      title: "Add /health route handler",
+      description: 'Add a /health route to src/server.js that responds with { status: "ok" }.',
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/server.js", `export function createServer() { return { handle: (req) => ({ statusCode: 404, body: 'not found' }) }; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("route", `import { createServer } from './src/server.js';
+const server = createServer();
+const res = server.handle({ url: '/health', method: 'GET' });
+assert.strictEqual(res.statusCode, 200);
+assert.deepStrictEqual(JSON.parse(res.body), { status: 'ok' });`));
+      }
+    }),
+    task({
+      id: "add-sum-function",
+      name: "Add a sum utility",
+      title: "Add sum(arr) function",
+      description: "Create src/math.js exporting sum that adds all numbers in an array.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("sum", `import { sum } from './src/math.js';
+assert.strictEqual(sum([1, 2, 3]), 6);
+assert.strictEqual(sum([]), 0);`));
+      }
+    }),
+    task({
+      id: "add-logger",
+      name: "Add a logger",
+      title: "Add createLogger function",
+      description: "Create src/logger.js exporting createLogger that returns an object with info/error methods.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("logger", `import { createLogger } from './src/logger.js';
+const logger = createLogger();
+assert.strictEqual(typeof logger.info, 'function');
+assert.strictEqual(typeof logger.error, 'function');`));
+      }
+    }),
+    // Tests
+    task({
+      id: "write-missing-test",
+      name: "Write a missing test",
+      title: "Write a test for double()",
+      description: "Create test.js that tests a src/math.js double() function and make it pass.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/math.js", `export function double(n) { return n * 2; }
+`);
+        await codeFile(projectPath, "test.js", `import assert from 'node:assert';
+import { double } from './src/math.js';
+assert.strictEqual(double(3), 6);
+console.log('double test passed');
+`);
+      }
+    }),
+    task({
+      id: "make-failing-test-pass",
+      name: "Make failing test pass",
+      title: "Implement capitalize()",
+      description: "Implement src/strings.js capitalize() so the provided test passes.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/strings.js", `export function capitalize(s) { return s; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("capitalize", `import { capitalize } from './src/strings.js';
+assert.strictEqual(capitalize('hello'), 'Hello');`));
+      }
+    }),
+    // Config/package
+    task({
+      id: "fix-package-type",
+      name: "Fix package type",
+      title: "Set package type to module",
+      description: 'Update package.json so "type" is "module".',
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await jsonFile(projectPath, "package.json", {
+          name: "synthetic-bench-project",
+          version: "1.0.0",
+          type: "commonjs",
+          scripts: { lint: "node lint.js", test: "node test.js", build: "node build.js" }
+        });
+        await codeFile(projectPath, "lint.js", lintAlwaysPass);
+        await codeFile(projectPath, "test.js", `console.log('test ok');
+`);
+        await codeFile(projectPath, "build.js", buildAlwaysPass);
+      },
+      evaluate: combined(async (ctx) => {
+        const pkg = JSON.parse(await fs4.readFile(`${ctx.projectPath}/package.json`, "utf-8"));
+        return {
+          passed: pkg.type === "module",
+          message: pkg.type === "module" ? "package type is module" : "package type not module"
+        };
+      }, expectValidationPassed())
+    }),
+    task({
+      id: "update-dependency",
+      name: "Update dependency version",
+      title: "Bump lodash to 4.17.21",
+      description: "Update package.json so lodash dependency version is 4.17.21.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await jsonFile(projectPath, "package.json", {
+          name: "synthetic-bench-project",
+          version: "1.0.0",
+          type: "module",
+          dependencies: { lodash: "4.17.20" },
+          scripts: { lint: "node lint.js", test: "node test.js", build: "node build.js" }
+        });
+        await codeFile(projectPath, "lint.js", lintAlwaysPass);
+        await codeFile(projectPath, "test.js", `import pkg from './package.json' assert { type: 'json' };
+if (pkg.dependencies?.lodash !== '4.17.21') { console.error('lodash not updated'); process.exit(1); }
+console.log('dependency test passed');
+`);
+        await codeFile(projectPath, "build.js", buildAlwaysPass);
+      }
+    }),
+    // Multi-file
+    task({
+      id: "extract-shared-utility",
+      name: "Extract shared utility",
+      title: "Move formatDate to src/date.js and update imports",
+      description: "Extract the formatDate function from src/a.js and src/b.js into src/date.js, then update imports.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/a.js", `function formatDate(d) { return d.toISOString().split('T')[0]; }
+export function runA(d) { return formatDate(d); }
+`);
+        await codeFile(projectPath, "src/b.js", `function formatDate(d) { return d.toISOString().split('T')[0]; }
+export function runB(d) { return formatDate(d); }
+`);
+        await codeFile(projectPath, "test.js", jsTest("shared", `import { readFile } from 'node:fs/promises';
+import { runA } from './src/a.js';
+import { runB } from './src/b.js';
+const a = await readFile('./src/a.js', 'utf-8');
+const b = await readFile('./src/b.js', 'utf-8');
+if (!a.includes('./date.js') || !b.includes('./date.js')) { console.error('imports not updated'); process.exit(1); }
+const d = new Date('2024-01-15');
+assert.strictEqual(runA(d), '2024-01-15');
+assert.strictEqual(runB(d), '2024-01-15');`));
+      }
+    }),
+    task({
+      id: "fix-circular-import",
+      name: "Fix circular import",
+      title: "Break circular dependency between a.js and b.js",
+      description: "Refactor src/a.js and src/b.js to remove the circular dependency while keeping both exports working.",
+      complexity: "complex",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/a.js", `import { valueB } from './b.js';
+export const valueA = 'A' + valueB;
+`);
+        await codeFile(projectPath, "src/b.js", `import { valueA } from './a.js';
+export const valueB = 'B' + valueA;
+`);
+        await codeFile(projectPath, "test.js", jsTest("circular", `import { valueA, valueB } from './src/a.js';
+assert.ok(typeof valueA === 'string');
+assert.ok(typeof valueB === 'string');`));
+      }
+    }),
+    // Documentation
+    task({
+      id: "update-readme-example",
+      name: "Update README example",
+      title: "Add a usage example to README.md",
+      description: "Edit README.md to include a code block example showing how to use greet().",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "README.md", `# Project
+
+A demo project.
+`);
+        await codeFile(projectPath, "test.js", [
+          "import { readFile } from 'node:fs/promises';",
+          "const readme = await readFile('./README.md', 'utf-8');",
+          "if (!readme.includes('```')) { console.error('README missing code block'); process.exit(1); }",
+          "if (!readme.includes('greet')) { console.error('README missing greet example'); process.exit(1); }",
+          "console.log('readme test passed');",
+          ""
+        ].join("\n"));
+      }
+    }),
+    // Validation / noop
+    task({
+      id: "noop-validation",
+      name: "Run validation on a clean project",
+      title: "Confirm the project already passes validation",
+      description: "This project already passes lint, test and build. Use the publish tool to confirm validation passes and finish.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+      }
+    }),
+    // Lint / format
+    task({
+      id: "fix-lint-warning",
+      name: "Fix a lint warning",
+      title: "Remove unused variable flagged by linter",
+      description: "The custom linter flags any use of `var`. Update src/index.js so the lint script passes while keeping the greeting behavior.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath, {
+          lint: "node lint.js"
+        });
+        await codeFile(projectPath, "src/index.js", `var greeting = 'Hello';
+export function hi(name) { return greeting + ', ' + name + '!'; }
+`);
+        await codeFile(projectPath, "lint.js", `import fs from 'node:fs/promises';
+const code = await fs.readFile('./src/index.js', 'utf-8');
+if (code.includes('var ')) { console.error('lint failed: var keyword found'); process.exit(1); }
+console.log('lint ok');
+`);
+        await codeFile(projectPath, "test.js", jsTest("hi", `import { hi } from './src/index.js';
+assert.strictEqual(hi('Omega'), 'Hello, Omega!');`));
+      }
+    }),
+    // Complex
+    task({
+      id: "implement-todo-store",
+      name: "Implement a todo store",
+      title: "Implement TodoStore with add/complete/list",
+      description: "Create src/todo.js exporting TodoStore class with add(text), complete(id), and list() methods.",
+      complexity: "complex",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("todo", `import { TodoStore } from './src/todo.js';
+const store = new TodoStore();
+const id = store.add('Buy milk');
+assert.deepStrictEqual(store.list(), [{ id, text: 'Buy milk', done: false }]);
+store.complete(id);
+assert.strictEqual(store.list()[0].done, true);`));
+      }
+    })
+  ];
+}
+
+// ../bench/dist/adapters/deepswe.js
+import fs5 from "node:fs/promises";
+import path8 from "node:path";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+var execFileAsync = promisify(execFile);
+function parseToml(raw) {
+  const result = { task: {}, metadata: {} };
+  let section;
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith("[")) {
+      const name = trimmed.slice(1, -1).split(".")[0];
+      if (name === "task" || name === "metadata")
+        section = name;
+      continue;
+    }
+    if (!section || trimmed.startsWith("#") || !trimmed.includes("="))
+      continue;
+    const eq = trimmed.indexOf("=");
+    const key = trimmed.slice(0, eq).trim();
+    let value = trimmed.slice(eq + 1).trim();
+    if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
+      value = value.slice(1, -1);
+    }
+    if (section === "task") {
+      result.task ??= {};
+      result.task[key] = value;
+    } else {
+      result.metadata ??= {};
+      result.metadata[key] = value;
+    }
+  }
+  return result;
+}
+function mulberry32(seed) {
+  return () => {
+    let t = seed += 1831565813;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+async function readTask(taskDir) {
+  const tomlRaw = await fs5.readFile(path8.join(taskDir, "task.toml"), "utf-8");
+  const instructionRaw = await fs5.readFile(path8.join(taskDir, "instruction.md"), "utf-8");
+  return { toml: parseToml(tomlRaw), instruction: instructionRaw };
+}
+async function cloneRepo(repoUrl, commit, targetPath) {
+  await fs5.mkdir(path8.dirname(targetPath), { recursive: true });
+  await execFileAsync("git", ["clone", repoUrl, targetPath], { timeout: 12e4 });
+  await execFileAsync("git", ["-C", targetPath, "checkout", commit], { timeout: 6e4 });
+}
+async function loadDeepSWESuite(options) {
+  const entries = await fs5.readdir(options.tasksDir, { withFileTypes: true });
+  const taskDirs = entries.filter((e) => e.isDirectory()).map((e) => path8.join(options.tasksDir, e.name));
+  let loaded = [];
+  for (const dir of taskDirs) {
+    try {
+      const data = await readTask(dir);
+      loaded.push({ dir, ...data });
+    } catch {
+    }
+  }
+  if (options.taskIds && options.taskIds.length > 0) {
+    loaded = loaded.filter((t) => {
+      const id = t.toml.metadata?.task_id ?? path8.basename(t.dir);
+      return options.taskIds?.includes(id);
+    });
+  }
+  if (options.nTasks && options.nTasks > 0 && options.nTasks < loaded.length) {
+    const seed = options.sampleSeed ?? 0;
+    const rnd = mulberry32(seed);
+    loaded = loaded.map((t) => ({ t, sort: rnd() })).sort((a, b) => a.sort - b.sort).slice(0, options.nTasks).map((x) => x.t);
+  }
+  const tasks = [];
+  for (const { dir, toml, instruction } of loaded) {
+    const id = toml.metadata?.task_id ?? path8.basename(dir);
+    const title = toml.metadata?.display_title ?? toml.metadata?.original_title ?? id;
+    const repo = toml.metadata?.repository_url;
+    const commit = toml.metadata?.base_commit_hash;
+    tasks.push({
+      id: `deepswe-${id}`,
+      name: id,
+      title,
+      description: instruction,
+      complexity: "complex",
+      setup: async (projectPath) => {
+        if (!repo || !commit) {
+          throw new Error(`DeepSWE task ${id} is missing repository_url or base_commit_hash`);
+        }
+        await cloneRepo(repo, commit, projectPath);
+      },
+      evaluate: (ctx) => {
+        const hasDiff = ctx.diffs.length > 0 && ctx.diffs.some((d) => d.patch.trim().length > 0);
+        const done = ctx.agentRun?.resultStatus === "done";
+        const passed = done && hasDiff;
+        return Promise.resolve({
+          passed,
+          message: passed ? `Agent finished and produced a diff (${String(ctx.diffs[0]?.patch.length ?? 0)} chars)` : `Agent status=${ctx.agentRun?.resultStatus ?? "unknown"}, diff=${hasDiff ? "yes" : "no"}`
+        });
+      }
+    });
+  }
+  return tasks;
+}
+
+// ../bench/dist/optimise.js
+import fs6 from "node:fs/promises";
+import path9 from "node:path";
+async function findLatestReport(outputDir) {
+  const entries = await fs6.readdir(outputDir).catch(() => []);
+  const files = entries.filter((f) => f.startsWith("benchmark-") && f.endsWith(".json")).sort().reverse();
+  if (files.length === 0)
+    return void 0;
+  const raw = await fs6.readFile(path9.join(outputDir, files[0]), "utf-8");
+  return JSON.parse(raw);
+}
+function summariseTraceFlow(report, maxLines = 60) {
+  const lines = report.split("\n");
+  if (lines.length <= maxLines)
+    return report;
+  return lines.slice(0, maxLines).join("\n") + "\n... (truncated)";
+}
+function buildOptimisePrompt(report, failedResult, traceFlowText) {
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const failed = report.results.filter((r) => !r.evaluation.passed);
+  const lines = [];
+  lines.push("# Prompt optimisation task");
+  lines.push("");
+  lines.push(`The latest benchmark run achieved ${String(passRate2)}% pass rate (${String(report.passed)}/${String(report.total)}).`);
+  lines.push("");
+  lines.push("## Failure analysis");
+  const traceFlows = /* @__PURE__ */ new Map();
+  const failureSummary = summariseFailures(report.results, traceFlows);
+  for (const summary of failureSummary) {
+    lines.push(`- ${summary.category}: ${String(summary.count)} task(s)`);
+    for (const example of summary.examples) {
+      lines.push(`  - ${example}`);
+    }
+  }
+  lines.push("");
+  lines.push("## Failed tasks");
+  for (const r of failed.slice(0, 5)) {
+    const analysis = r.failureAnalysis ?? classifyFailure(r, void 0);
+    lines.push(`- ${r.task.name} [${analysis.category}]: ${r.evaluation.message ?? "no message"}`);
+  }
+  lines.push("");
+  if (failedResult) {
+    const analysis = failedResult.failureAnalysis ?? classifyFailure(failedResult, void 0);
+    lines.push(`## Focus task: ${failedResult.task.name}`);
+    lines.push(`Status: ${failedResult.status}`);
+    lines.push(`Category: ${analysis.category}`);
+    lines.push(`Root cause: ${analysis.rootCause}`);
+    if (analysis.evidence.length > 0) {
+      lines.push("Evidence:");
+      for (const ev of analysis.evidence) {
+        lines.push(`- ${ev}`);
+      }
+    }
+    lines.push("");
+  }
+  if (traceFlowText) {
+    lines.push("## Trace flow snapshot");
+    lines.push("```json");
+    lines.push(summariseTraceFlow(traceFlowText));
+    lines.push("```");
+    lines.push("");
+  }
+  lines.push("## Instructions");
+  lines.push("Analyse the failures above and edit `packages/agent/src/prompts.ts` (and optionally `packages/agent/src/planner.ts`) to improve the Omega agent.");
+  lines.push("Rules:");
+  lines.push("- Keep changes minimal and targeted.");
+  lines.push("- Address the diagnosed root cause, not just the symptom.");
+  lines.push("- Do not break existing tool signatures or system behaviour.");
+  lines.push("- Run `pnpm lint` and `pnpm test` after edits.");
+  lines.push("- Finish with a summary of what changed and why it should help.");
+  return lines.join("\n");
+}
+async function loadOptimisationContext(apiUrl, outputDir) {
+  const report = await findLatestReport(outputDir);
+  if (!report)
+    return void 0;
+  const traceFlows = /* @__PURE__ */ new Map();
+  for (const result of report.results) {
+    if (result.evaluation.passed)
+      continue;
+    const traceFlow = result.harnessTaskId ? await getTraceFlow(apiUrl, result.harnessTaskId) : void 0;
+    traceFlows.set(result.harnessTaskId, traceFlow ?? void 0);
+  }
+  const failedResult = pickFocusResult(report.results, traceFlows);
+  let traceFlowText;
+  if (failedResult?.harnessTaskId) {
+    const traceFlow = traceFlows.get(failedResult.harnessTaskId);
+    if (traceFlow) {
+      traceFlowText = JSON.stringify(traceFlow, null, 2);
+    }
+  }
+  return { report, failedResult, traceFlowText };
+}
+async function submitOptimiseTask(apiUrl, projectId, prompt) {
+  const title = "Optimise agent prompts from benchmark trace analysis";
+  const res = await fetch(`${apiUrl}/tasks`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      projectId,
+      title,
+      description: prompt,
+      complexity: "complex",
+      tags: ["self-improve", "benchmark"]
+    })
+  });
+  if (!res.ok)
+    throw new Error(`submit optimise task failed: ${String(res.status)}`);
+  return res.json();
+}
+
+// ../bench/dist/ab.js
+function passRate(report) {
+  return report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+}
+function findResult(report, taskId) {
+  return report.results.find((r) => r.task.id === taskId);
+}
+function buildAbReport(baseline, candidate, baselineReport, candidateReport) {
+  const baselineRate = passRate(baselineReport);
+  const candidateRate = passRate(candidateReport);
+  const taskIds = /* @__PURE__ */ new Set();
+  for (const r of baselineReport.results)
+    taskIds.add(r.task.id);
+  for (const r of candidateReport.results)
+    taskIds.add(r.task.id);
+  const perTask = Array.from(taskIds).map((taskId) => {
+    const base = findResult(baselineReport, taskId);
+    const cand = findResult(candidateReport, taskId);
+    return {
+      taskId,
+      taskName: base?.task.name ?? cand?.task.name ?? taskId,
+      baselinePassed: base?.evaluation.passed ?? false,
+      candidatePassed: cand?.evaluation.passed ?? false,
+      baselineTokens: base?.usage?.totalTokens,
+      candidateTokens: cand?.usage?.totalTokens
+    };
+  });
+  return {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    baseline,
+    candidate,
+    baselineReport,
+    candidateReport,
+    delta: {
+      passRate: candidateRate - baselineRate,
+      passedDelta: candidateReport.passed - baselineReport.passed,
+      durationMsDelta: candidateReport.totalDurationMs - baselineReport.totalDurationMs,
+      promptTokensDelta: (candidateReport.totalUsage.promptTokens ?? 0) - (baselineReport.totalUsage.promptTokens ?? 0),
+      completionTokensDelta: (candidateReport.totalUsage.completionTokens ?? 0) - (baselineReport.totalUsage.completionTokens ?? 0),
+      totalTokensDelta: (candidateReport.totalUsage.totalTokens ?? 0) - (baselineReport.totalUsage.totalTokens ?? 0)
+    },
+    perTask
+  };
+}
+function deltaSymbol(value) {
+  if (value > 0)
+    return "+";
+  if (value < 0)
+    return "";
+  return "";
+}
+function formatDelta(value) {
+  return `${deltaSymbol(value)}${String(value)}`;
+}
+function formatAbReport(report) {
+  const lines = [];
+  lines.push("# Omega A/B Benchmark Report");
+  lines.push("");
+  lines.push(`- Timestamp: ${report.timestamp}`);
+  lines.push(`- Baseline: ${report.baseline.name}`);
+  lines.push(`- Candidate: ${report.candidate.name}`);
+  lines.push("");
+  lines.push("## Summary");
+  lines.push(`| Metric | Baseline | Candidate | Delta |`);
+  lines.push(`|---|---|---|---|`);
+  lines.push(`| Pass rate | ${String(passRate(report.baselineReport))}% | ${String(passRate(report.candidateReport))}% | ${formatDelta(report.delta.passRate)}% |`);
+  lines.push(`| Passed | ${String(report.baselineReport.passed)}/${String(report.baselineReport.total)} | ${String(report.candidateReport.passed)}/${String(report.candidateReport.total)} | ${formatDelta(report.delta.passedDelta)} |`);
+  lines.push(`| Duration | ${formatDuration2(report.baselineReport.totalDurationMs)} | ${formatDuration2(report.candidateReport.totalDurationMs)} | ${formatDuration2(report.delta.durationMsDelta)} |`);
+  lines.push(`| Tokens | ${String(report.baselineReport.totalUsage.totalTokens ?? 0)} | ${String(report.candidateReport.totalUsage.totalTokens ?? 0)} | ${formatDelta(report.delta.totalTokensDelta)} |`);
+  lines.push("");
+  lines.push("## Per-task comparison");
+  lines.push(`| Task | Baseline | Candidate | Token delta |`);
+  lines.push(`|---|---|---|---|`);
+  for (const t of report.perTask) {
+    const tokenDelta = t.baselineTokens !== void 0 && t.candidateTokens !== void 0 ? formatDelta(t.candidateTokens - t.baselineTokens) : "-";
+    lines.push(`| ${t.taskName} | ${t.baselinePassed ? "pass" : "fail"} | ${t.candidatePassed ? "pass" : "fail"} | ${tokenDelta} |`);
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+function formatDuration2(ms) {
+  if (ms < 1e3)
+    return `${String(Math.round(ms))}ms`;
+  return `${(ms / 1e3).toFixed(1)}s`;
+}
+
+// ../../apps/cli/src/commands/bench.ts
+async function waitForApi2(apiUrl, maxMs = 1e4) {
+  const deadline = Date.now() + maxMs;
+  while (Date.now() < deadline) {
+    try {
+      const res = await fetch(`${apiUrl}/projects`);
+      if (res.ok) return;
+    } catch {
+    }
+    await new Promise((r) => setTimeout(r, 300));
+  }
+  throw new Error(`Harness API is not ready at ${apiUrl}. Run \`omega ui\` or \`omega console\` first.`);
+}
+function currentProject(apiUrl) {
+  const cwd = process.cwd();
+  const name = path11.basename(cwd);
+  return ensureProject2(apiUrl, `bench-${name}`, cwd);
+}
+var runCmd = new Command8("run").description("Run a benchmark suite").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").action(async (opts) => {
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const timeoutMs = Number(opts.timeout);
+  let tasks;
+  let suiteName;
+  if (opts.suite === "deep-swe") {
+    if (!opts.path) {
+      throw new Error("--path is required for the deep-swe suite");
+    }
+    tasks = await loadDeepSWESuite({
+      tasksDir: opts.path,
+      nTasks: opts.nTasks,
+      sampleSeed: opts.sampleSeed
+    });
+    suiteName = "deep-swe";
+  } else if (opts.suite === "synthetic") {
+    tasks = syntheticSuite();
+    if (opts.nTasks !== void 0 && opts.nTasks > 0) {
+      tasks = tasks.slice(0, opts.nTasks);
+    }
+    suiteName = "synthetic";
+  } else {
+    throw new Error(`Unknown suite: ${opts.suite}`);
+  }
+  if (tasks.length === 0) {
+    console.log("No benchmark tasks to run.");
+    return;
+  }
+  console.log(`Running ${String(tasks.length)} benchmark tasks against ${apiUrl}`);
+  const report = await runBenchmark(tasks, {
+    apiUrl,
+    suiteName,
+    timeoutMs,
+    onProgress: (result) => {
+      const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
+      console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
+    }
+  });
+  const reportFile = await writeReport(report, opts.outputDir);
+  printSummary(report);
+  console.log(`
+Report written to ${reportFile}`);
+});
+var optimiseCmd = new Command8("optimise").description("Create a self-improve task from the latest benchmark report").option("--output-dir <dir>", "report output directory", ".omega/reports").action(async (opts) => {
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const context = await loadOptimisationContext(apiUrl, opts.outputDir);
+  if (!context) {
+    console.log("No benchmark report found. Run `omega bench run` first.");
+    return;
+  }
+  const { loadCurrentPrompts: loadCurrentPrompts2 } = await Promise.resolve().then(() => (init_prompt_versioning(), prompt_versioning_exports));
+  const current = await loadCurrentPrompts2();
+  const versionRes = await fetch(`${apiUrl}/prompt-versions`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      name: current.name,
+      sourcePath: current.sourcePath,
+      systemPrompt: current.systemPrompt,
+      textToolsPrompt: current.textToolsPrompt,
+      hash: current.hash,
+      metadata: { planningPrompt: current.planningPrompt }
+    })
+  });
+  if (versionRes.ok) {
+    const version = await versionRes.json();
+    console.log(`Saved baseline prompt version: ${version.name} (${version.id})`);
+  } else {
+    console.warn("Could not save prompt version:", await versionRes.text());
+  }
+  const project = await currentProject(apiUrl);
+  const prompt = buildOptimisePrompt(
+    context.report,
+    context.failedResult,
+    context.traceFlowText
+  );
+  const task2 = await submitOptimiseTask(apiUrl, project.id, prompt);
+  console.log(`Created self-improve task ${task2.id}`);
+  console.log(`Run \`omega task run ${task2.id}\` to execute it.`);
+});
+async function loadPromptSpec(source) {
+  if (source === "current") {
+    const { readPromptsSource: readPromptsSource2 } = await Promise.resolve().then(() => (init_prompt_versioning(), prompt_versioning_exports));
+    const prompts = await readPromptsSource2();
+    return { name: "current", ...prompts };
+  }
+  const resolved = path11.resolve(source);
+  const raw = await fs8.readFile(resolved, "utf-8");
+  const parsed = JSON.parse(raw);
+  if (!parsed.name || !parsed.systemPrompt || !parsed.textToolsPrompt) {
+    throw new Error("Prompt file must include name, systemPrompt, and textToolsPrompt");
+  }
+  return parsed;
+}
+async function writePromptFile(spec) {
+  const tmpDir = await fs8.mkdtemp(path11.join(os.tmpdir(), "omega-prompt-"));
+  const file = path11.join(tmpDir, "prompts.json");
+  await fs8.writeFile(file, JSON.stringify(spec, null, 2), "utf-8");
+  return file;
+}
+async function runBenchmarkWithPrompts(tasks, spec, options) {
+  const promptFile = await writePromptFile(spec);
+  const env = {
+    ...process.env,
+    OMEGA_SYSTEM_PROMPT_FILE: promptFile,
+    OMEGA_TEXT_TOOLS_PROMPT_FILE: promptFile
+  };
+  let apiUrl = options.apiUrl;
+  let server;
+  if (!apiUrl) {
+    const started = await startHarnessServer(env);
+    apiUrl = started.apiUrl;
+    server = started.process;
+  }
+  try {
+    await waitForApi2(apiUrl, 3e4);
+    const report = await runBenchmark(tasks, {
+      apiUrl,
+      suiteName: options.suiteName,
+      timeoutMs: options.timeoutMs,
+      onProgress: (result) => {
+        const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
+        console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
+      }
+    });
+    return { report, apiUrl, server };
+  } catch (err) {
+    server?.kill();
+    throw err;
+  }
+}
+var abCmd = new Command8("ab").description("Run an A/B benchmark comparing two prompt versions").option("--baseline <source>", 'baseline prompts: "current" or path to JSON file', "current").option("--candidate <source>", "candidate prompts: path to JSON file").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").action(async (opts) => {
+  if (!opts.candidate) {
+    throw new Error("--candidate is required");
+  }
+  const timeoutMs = Number(opts.timeout);
+  const existingApiUrl = getApiUrl();
+  let tasks;
+  let suiteName;
+  if (opts.suite === "deep-swe") {
+    throw new Error("DeepSWE A/B not yet supported; use --suite synthetic");
+  } else if (opts.suite === "synthetic") {
+    tasks = syntheticSuite();
+    if (opts.nTasks !== void 0 && opts.nTasks > 0) {
+      tasks = tasks.slice(0, opts.nTasks);
+    }
+    suiteName = "synthetic";
+  } else {
+    throw new Error(`Unknown suite: ${opts.suite}`);
+  }
+  const baselineSpec = await loadPromptSpec(opts.baseline);
+  const candidateSpec = await loadPromptSpec(opts.candidate);
+  console.log(`Baseline: ${baselineSpec.name}`);
+  console.log(`Candidate: ${candidateSpec.name}`);
+  console.log(`Running ${String(tasks.length)} tasks for baseline...`);
+  const { report: baselineReport, server: baseServer } = await runBenchmarkWithPrompts(tasks, baselineSpec, {
+    apiUrl: existingApiUrl,
+    suiteName: `${suiteName}-baseline`,
+    timeoutMs
+  });
+  baseServer?.kill();
+  console.log(`Running ${String(tasks.length)} tasks for candidate...`);
+  const { report: candidateReport, server: candServer } = await runBenchmarkWithPrompts(tasks, candidateSpec, {
+    apiUrl: existingApiUrl,
+    suiteName: `${suiteName}-candidate`,
+    timeoutMs
+  });
+  candServer?.kill();
+  const abReport = buildAbReport(baselineSpec, candidateSpec, baselineReport, candidateReport);
+  await fs8.mkdir(opts.outputDir, { recursive: true });
+  const ts = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+  const jsonFile2 = path11.join(opts.outputDir, `ab-${ts}.json`);
+  const mdFile = path11.join(opts.outputDir, `ab-${ts}.md`);
+  await fs8.writeFile(jsonFile2, JSON.stringify(abReport, null, 2), "utf-8");
+  await fs8.writeFile(mdFile, formatAbReport(abReport), "utf-8");
+  console.log("\n" + formatAbReport(abReport));
+  console.log(`
+A/B report written to ${jsonFile2}`);
+});
+var benchCmd = new Command8("bench").description("Run benchmarks and optimise prompts").addCommand(runCmd).addCommand(abCmd).addCommand(optimiseCmd);
+
 // ../../apps/cli/src/index.ts
 program2.name("harness").description("Omega harness CLI").version("0.1.0").option("--api <url>", "API base URL", "http://localhost:4000");
 program2.addCommand(projectCmd);
@@ -825,4 +2511,5 @@ program2.addCommand(uiCmd);
 program2.addCommand(consoleCmd);
 program2.addCommand(skillCmd);
 program2.addCommand(agentCmd);
+program2.addCommand(benchCmd);
 program2.parse();