@castlemilk/omega 0.6.11 → 0.6.13

package/dist/cli.js

@@ -66,20 +66,20 @@ function useTasks(pollMs = 1e3) {
         const data = await res.json();
         setConnected(true);
         const current = {};
-        for (const task of data) {
-          current[task.id] = task;
-          const prev = previousTasks.current[task.id];
+        for (const task2 of data) {
+          current[task2.id] = task2;
+          const prev = previousTasks.current[task2.id];
           if (!prev) {
-            addLog(`New task queued: "${task.title}"`, "info");
-          } else if (prev.status !== task.status) {
-            if (task.status === "in_progress") {
-              const provider = task.provider ?? "unknown";
-              const model = task.model ?? "unknown";
-              addLog(`LLM picked up "${task.title}" \u2192 ${provider}/${model}`, "info");
-            } else if (task.status === "done") {
-              addLog(`Completed: "${task.title}"`, "success");
-            } else if (task.status === "failed") {
-              addLog(`Failed: "${task.title}"${task.error ? ` \u2014 ${task.error}` : ""}`, "error");
+            addLog(`New task queued: "${task2.title}"`, "info");
+          } else if (prev.status !== task2.status) {
+            if (task2.status === "in_progress") {
+              const provider = task2.provider ?? "unknown";
+              const model = task2.model ?? "unknown";
+              addLog(`LLM picked up "${task2.title}" \u2192 ${provider}/${model}`, "info");
+            } else if (task2.status === "done") {
+              addLog(`Completed: "${task2.title}"`, "success");
+            } else if (task2.status === "failed") {
+              addLog(`Failed: "${task2.title}"${task2.error ? ` \u2014 ${task2.error}` : ""}`, "error");
             }
           }
         }
@@ -231,16 +231,16 @@ function TaskList({
       children: [
         header,
         visible.length === 0 && /* @__PURE__ */ jsx(Text, { dimColor: true, children: "No tasks yet. Create one in the web UI." }),
-        visible.map((task) => {
-          const providerText = task.provider ? `${task.provider}/${task.model ?? ""}` : "-";
+        visible.map((task2) => {
+          const providerText = task2.provider ? `${task2.provider}/${task2.model ?? ""}` : "-";
           const titleMax = Math.max(10, columns - 62);
           return /* @__PURE__ */ jsxs(Box, { flexDirection: "row", gap: 1, children: [
-            /* @__PURE__ */ jsx(Text, { color: statusColor(task.status), children: statusSymbol(task.status) }),
-            /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { dimColor: true, children: formatTime(new Date(task.updatedAt)) }) }),
-            /* @__PURE__ */ jsx(Box, { width: 12, children: /* @__PURE__ */ jsx(Text, { color: statusColor(task.status), bold: true, children: task.status.toUpperCase() }) }),
+            /* @__PURE__ */ jsx(Text, { color: statusColor(task2.status), children: statusSymbol(task2.status) }),
+            /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { dimColor: true, children: formatTime(new Date(task2.updatedAt)) }) }),
+            /* @__PURE__ */ jsx(Box, { width: 12, children: /* @__PURE__ */ jsx(Text, { color: statusColor(task2.status), bold: true, children: task2.status.toUpperCase() }) }),
             /* @__PURE__ */ jsx(Box, { width: 24, children: /* @__PURE__ */ jsx(Text, { dimColor: true, children: truncate(providerText, 23) }) }),
-            /* @__PURE__ */ jsx(Box, { flexGrow: 1, children: /* @__PURE__ */ jsx(Text, { children: truncate(task.title, titleMax) }) })
-          ] }, task.id);
+            /* @__PURE__ */ jsx(Box, { flexGrow: 1, children: /* @__PURE__ */ jsx(Text, { children: truncate(task2.title, titleMax) }) })
+          ] }, task2.id);
         })
       ]
     }
@@ -318,6 +318,76 @@ var init_tui = __esm({
   }
 });
 
+// ../agent/dist/prompt-versioning.js
+var prompt_versioning_exports = {};
+__export(prompt_versioning_exports, {
+  envForPrompts: () => envForPrompts,
+  hashPrompts: () => hashPrompts,
+  loadCurrentPrompts: () => loadCurrentPrompts,
+  readPromptsSource: () => readPromptsSource,
+  saveCurrentPrompts: () => saveCurrentPrompts
+});
+import fs7 from "node:fs/promises";
+import path10 from "node:path";
+import crypto from "node:crypto";
+import { fileURLToPath as fileURLToPath4 } from "node:url";
+function hashString(value) {
+  return crypto.createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+function hashPrompts(input) {
+  return hashString([input.systemPrompt, input.textToolsPrompt, input.planningPrompt ?? ""].join("\n---\n"));
+}
+async function readPromptsSource() {
+  const source = await fs7.readFile(PROMPTS_PATH, "utf-8");
+  const systemMatch = /export const AGENT_SYSTEM_PROMPT =\s*(?:loadPromptFromEnv\('OMEGA_SYSTEM_PROMPT'\) \?\?\s*)?`([\s\S]*?)`;/m.exec(source);
+  const textToolsMatch = /export const TEXT_TOOLS_SYSTEM_PROMPT =\s*(?:loadPromptFromEnv\('OMEGA_TEXT_TOOLS_PROMPT'\) \?\?\s*)?`([\s\S]*?)`;/m.exec(source);
+  return {
+    systemPrompt: systemMatch?.[1] ?? "",
+    textToolsPrompt: textToolsMatch?.[1] ?? ""
+  };
+}
+async function loadCurrentPrompts() {
+  const { systemPrompt, textToolsPrompt } = await readPromptsSource();
+  const plannerSource = await fs7.readFile(PLANNER_PATH, "utf-8");
+  const planningMatch = /const PLAN_PROMPT = `([\s\S]*?)`;/m.exec(plannerSource);
+  const planningPrompt = planningMatch?.[1];
+  return {
+    name: `auto-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`,
+    sourcePath: PROMPTS_PATH,
+    systemPrompt,
+    textToolsPrompt,
+    planningPrompt,
+    hash: hashPrompts({ systemPrompt, textToolsPrompt, planningPrompt })
+  };
+}
+async function saveCurrentPrompts(input) {
+  const source = await fs7.readFile(PROMPTS_PATH, "utf-8");
+  const replaceConst = (name, value) => {
+    const pattern = new RegExp(`export const ${name} =\\s*(?:process\\.env\\.OMEGA_[A-Z_]+_PROMPT \\?\\?\\s*)?\`[\\s\\S]*?\`;`, "m");
+    return source.replace(pattern, `export const ${name} =
+  process.env.OMEGA_${name.replace(/([A-Z])/g, "_$1").toUpperCase()}_PROMPT ??
+  \`${value}\`;`);
+  };
+  let updated = replaceConst("AGENT_SYSTEM_PROMPT", input.systemPrompt);
+  updated = replaceConst("TEXT_TOOLS_SYSTEM_PROMPT", input.textToolsPrompt);
+  await fs7.writeFile(PROMPTS_PATH, updated, "utf-8");
+}
+function envForPrompts(input) {
+  return {
+    OMEGA_SYSTEM_PROMPT: input.systemPrompt,
+    OMEGA_TEXT_TOOLS_PROMPT: input.textToolsPrompt
+  };
+}
+var __dirname3, PROMPTS_PATH, PLANNER_PATH;
+var init_prompt_versioning = __esm({
+  "../agent/dist/prompt-versioning.js"() {
+    "use strict";
+    __dirname3 = path10.dirname(fileURLToPath4(import.meta.url));
+    PROMPTS_PATH = path10.resolve(__dirname3, "..", "src", "prompts.ts");
+    PLANNER_PATH = path10.resolve(__dirname3, "..", "src", "planner.ts");
+  }
+});
+
 // ../../apps/cli/src/index.ts
 import { program as program2 } from "commander";
 
@@ -328,10 +398,10 @@ import { Command } from "commander";
 import { program } from "commander";
 function getApiUrl() {
   const opts = program.opts();
-  return opts.api ?? "http://localhost:4000";
+  return opts.api ?? process.env.HARNESS_API_URL ?? "http://localhost:4000";
 }
-async function apiFetch(path10, init) {
-  const url = `${getApiUrl()}${path10}`;
+async function apiFetch(path12, init) {
+  const url = `${getApiUrl()}${path12}`;
   const res = await fetch(url, init);
   const data = await res.json().catch(() => ({}));
   if (!res.ok) {
@@ -446,7 +516,7 @@ var taskFeedCmd = new Command2("feed").description("Feed a task into the harness
 // ../../apps/cli/src/commands/task.ts
 var taskCmd = new Command3("task").description("Manage tasks");
 taskCmd.command("create").description("Create a task").requiredOption("--project <id>", "project id").requiredOption("--title <title>", "task title").option("--description <text>", "task description").option("--complexity <level>", "simple | medium | complex", "simple").option("--tags <tags>", "comma-separated tags").action(async (opts) => {
-  const task = await apiFetch("/tasks", {
+  const task2 = await apiFetch("/tasks", {
     method: "POST",
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify({
@@ -457,7 +527,7 @@ taskCmd.command("create").description("Create a task").requiredOption("--project
       tags: opts.tags ? opts.tags.split(",").map((t) => t.trim()) : []
     })
   });
-  console.log(JSON.stringify(task, null, 2));
+  console.log(JSON.stringify(task2, null, 2));
 });
 taskCmd.command("list").description("List tasks").option("--project <id>", "filter by project id").action(async (opts) => {
   const query = opts.project ? `?projectId=${opts.project}` : "";
@@ -472,14 +542,17 @@ taskCmd.addCommand(taskFeedCmd);
 
 // ../../apps/cli/src/commands/ui.ts
 import { Command as Command4 } from "commander";
+import path3 from "path";
+import open from "open";
+import { render } from "ink";
+import React2 from "react";
+
+// ../../apps/cli/src/lib/server.ts
 import { spawn } from "child_process";
 import { existsSync as existsSync2 } from "fs";
 import net from "net";
 import path2 from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
-import open from "open";
-import { render } from "ink";
-import React2 from "react";
 var __filename2 = fileURLToPath2(import.meta.url);
 var __dirname2 = path2.dirname(__filename2);
 var DEFAULT_PORT = 4e3;
@@ -489,7 +562,7 @@ var MAX_GRPC_PORT = 50061;
 function findAvailablePort(preferred = DEFAULT_PORT, max = MAX_PORT) {
   return new Promise((resolve, reject) => {
     if (preferred > max) {
-      reject(new Error(`No available ports between ${DEFAULT_PORT.toString()} and ${max.toString()}`));
+      reject(new Error(`No available ports between ${preferred.toString()} and ${max.toString()}`));
       return;
     }
     const tester = net.createServer();
@@ -511,11 +584,11 @@ function findAvailablePort(preferred = DEFAULT_PORT, max = MAX_PORT) {
   });
 }
 function startBundledServer(env) {
-  const realServerPath = path2.resolve(__dirname2, "server/dist/index.js");
+  const realServerPath = path2.resolve(__dirname2, "../../server/dist/index.js");
   if (existsSync2(realServerPath)) {
     return spawn(process.execPath, [realServerPath], { stdio: "inherit", env });
   }
-  const legacyServerPath = path2.resolve(__dirname2, "server.js");
+  const legacyServerPath = path2.resolve(__dirname2, "../../server.js");
   if (existsSync2(legacyServerPath)) {
     return spawn(process.execPath, [legacyServerPath], { stdio: "inherit", env });
   }
@@ -537,9 +610,29 @@ async function waitForApi(apiUrl, maxMs = 3e4) {
   }
   throw new Error("Server did not become ready in time");
 }
+async function startHarnessServer(env, options = {}) {
+  const port = await findAvailablePort(options.preferredPort ?? DEFAULT_PORT, MAX_PORT);
+  const grpcPort = await findAvailablePort(options.preferredGrpcPort ?? DEFAULT_GRPC_PORT, MAX_GRPC_PORT);
+  const apiUrl = `http://localhost:${port.toString()}`;
+  const serverEnv = {
+    ...env,
+    PORT: port.toString(),
+    GRPC_PORT: grpcPort.toString(),
+    HARNESS_API_URL: apiUrl
+  };
+  const server = startBundledServer(serverEnv) ?? spawn("pnpm", ["--filter", "@omega/server", "dev"], {
+    stdio: "inherit",
+    shell: true,
+    env: serverEnv
+  });
+  await waitForApi(apiUrl);
+  return { process: server, apiUrl, port, grpcPort };
+}
+
+// ../../apps/cli/src/commands/ui.ts
 async function ensureProject(apiUrl) {
   const cwd = process.cwd();
-  const name = path2.basename(cwd);
+  const name = path3.basename(cwd);
   try {
     const listRes = await fetch(`${apiUrl}/projects`);
     const projects = await listRes.json();
@@ -565,25 +658,16 @@ async function ensureProject(apiUrl) {
 var uiCmd = new Command4("ui").description("Open the harness web UI").option("--no-tui", "Do not open the terminal console alongside the web UI").action(async (options) => {
   const defaultApiUrl = process.env.HARNESS_API_URL ?? `http://localhost:${DEFAULT_PORT.toString()}`;
   let server;
+  let serverStarted = false;
   const alreadyRunning = await isApiReady(defaultApiUrl);
   const useTui = options.tui;
   let apiUrl = defaultApiUrl;
   if (!alreadyRunning) {
-    const port = await findAvailablePort();
-    const grpcPort = await findAvailablePort(DEFAULT_GRPC_PORT, MAX_GRPC_PORT);
-    apiUrl = `http://localhost:${port.toString()}`;
+    const started = await startHarnessServer(process.env);
+    server = started.process;
+    apiUrl = started.apiUrl;
+    serverStarted = true;
     process.env.HARNESS_API_URL = apiUrl;
-    const env = {
-      ...process.env,
-      PORT: port.toString(),
-      GRPC_PORT: grpcPort.toString(),
-      HARNESS_API_URL: apiUrl
-    };
-    server = startBundledServer(env) ?? spawn("pnpm", ["--filter", "@omega/server", "dev"], {
-      stdio: "inherit",
-      shell: true,
-      env
-    });
     server.on("exit", (code) => {
       process.exit(code ?? 0);
     });
@@ -607,7 +691,7 @@ var uiCmd = new Command4("ui").description("Open the harness web UI").option("--
     const { TuiApp: TuiApp2 } = await Promise.resolve().then(() => (init_tui(), tui_exports));
     const { waitUntilExit } = render(React2.createElement(TuiApp2));
     await waitUntilExit();
-    if (server && !alreadyRunning) {
+    if (serverStarted && server) {
       server.kill();
     }
   }
@@ -628,7 +712,7 @@ var consoleCmd = new Command5("console").description("Open the harness TUI conso
 
 // ../../apps/cli/src/commands/skill.ts
 import { Command as Command6 } from "commander";
-import path3 from "path";
+import path4 from "path";
 import { fileURLToPath as fileURLToPath3 } from "url";
 
 // ../skills/src/parse.ts
@@ -695,7 +779,8 @@ export class ${className} {
 
   async execute(args: Record<string, unknown>): Promise<string> {
     const prompt = ${instructionsLiteral} + '\\n\\nArgs:\\n' + JSON.stringify(args, null, 2);
-    return this.provider.send(prompt, { system: ${descriptionLiteral} });
+    const response = await this.provider.send(prompt, { system: ${descriptionLiteral} });
+    return response.content;
   }
 }
 `;
@@ -730,22 +815,22 @@ async function registerSkill(sourcePath, outputDir) {
 // ../../apps/cli/src/commands/skill.ts
 function getOutputDir() {
   if (process.env.OMEGA_HARNESS_ROOT) {
-    return path3.join(path3.resolve(process.env.OMEGA_HARNESS_ROOT), "packages/skills/src");
+    return path4.join(path4.resolve(process.env.OMEGA_HARNESS_ROOT), "packages/skills/src");
   }
   try {
     const __filename3 = fileURLToPath3(import.meta.url);
-    const __dirname3 = path3.dirname(__filename3);
-    return path3.join(path3.resolve(__dirname3, "../../../.."), "packages/skills/src");
+    const __dirname4 = path4.dirname(__filename3);
+    return path4.join(path4.resolve(__dirname4, "../../../.."), "packages/skills/src");
   } catch {
-    return path3.join(process.cwd(), "harness-skills");
+    return path4.join(process.cwd(), "harness-skills");
   }
 }
 var skillCmd = new Command6("skill").description("Manage skill artifacts");
 skillCmd.command("generate").description("Generate a harness adapter from a SKILL.md file").argument("<source>", "path to SKILL.md").action(async (source) => {
   const outputDir = getOutputDir();
-  const manifest = await registerSkill(path3.resolve(source), outputDir);
+  const manifest = await registerSkill(path4.resolve(source), outputDir);
   console.log(`Generated adapter for skill: ${manifest.name}`);
-  console.log(`Output directory: ${path3.join(outputDir, "generated")}`);
+  console.log(`Output directory: ${path4.join(outputDir, "generated")}`);
 });
 
 // ../../apps/cli/src/commands/agent.ts
@@ -755,7 +840,7 @@ agentCmd.command("run").description("Run a single agent task").requiredOption("-
   async (opts) => {
     const tags = ["agent"];
     if (opts.autoPublish) tags.push("publish");
-    const task = await apiFetch("/tasks", {
+    const task2 = await apiFetch("/tasks", {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
@@ -766,8 +851,8 @@ agentCmd.command("run").description("Run a single agent task").requiredOption("-
         tags
       })
     });
-    console.log(JSON.stringify(task, null, 2));
-    const result = await apiFetch(`/tasks/${task.id}/run`, { method: "POST" });
+    console.log(JSON.stringify(task2, null, 2));
+    const result = await apiFetch(`/tasks/${task2.id}/run`, { method: "POST" });
     console.log(JSON.stringify(result, null, 2));
   }
 );
@@ -779,7 +864,7 @@ agentCmd.command("loop").description("Continuously run agent tasks").requiredOpt
       const title = opts.prompt ?? "Improve the Omega harness: fix a TODO, add a test, refactor a function, or improve documentation";
       const tags = ["agent", "self-improve"];
       if (opts.autoPublish) tags.push("publish");
-      const task = await apiFetch("/tasks", {
+      const task2 = await apiFetch("/tasks", {
         method: "POST",
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({
@@ -790,8 +875,8 @@ agentCmd.command("loop").description("Continuously run agent tasks").requiredOpt
           tags
         })
       });
-      console.log(`Created task ${task.id}`);
-      const result = await apiFetch(`/tasks/${task.id}/run`, { method: "POST" });
+      console.log(`Created task ${task2.id}`);
+      const result = await apiFetch(`/tasks/${task2.id}/run`, { method: "POST" });
       console.log(JSON.stringify(result, null, 2));
       return result;
     };
@@ -819,7 +904,9 @@ agentCmd.command("diffs").description("Show diffs for a task").argument("<id>",
 
 // ../../apps/cli/src/commands/bench.ts
 import { Command as Command8 } from "commander";
-import path9 from "node:path";
+import fs8 from "node:fs/promises";
+import path11 from "node:path";
+import os from "node:os";
 
 // ../bench/dist/api-client.js
 var ApiError = class extends Error {
@@ -836,15 +923,15 @@ async function apiFetch2(url, init) {
   }
   return res.json();
 }
-async function ensureProject2(apiUrl, name, path10) {
+async function ensureProject2(apiUrl, name, path12) {
   const res = await fetch(`${apiUrl}/projects`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ name, path: path10 })
+    body: JSON.stringify({ name, path: path12 })
   });
   if (res.status === 409) {
-    const projects = await fetch(`${apiUrl}/projects?path=${encodeURIComponent(path10)}`).then((r) => r.json());
-    const existing = projects.find((p) => p.path === path10);
+    const projects = await fetch(`${apiUrl}/projects?path=${encodeURIComponent(path12)}`).then((r) => r.json());
+    const existing = projects.find((p) => p.path === path12);
     if (existing)
       return existing;
   }
@@ -861,7 +948,9 @@ async function createTask(apiUrl, projectId, title, options = {}) {
       title,
       description: options.description,
       complexity: options.complexity ?? "simple",
-      tags: options.tags ?? ["benchmark"]
+      tags: options.tags ?? ["benchmark"],
+      provider: options.provider,
+      model: options.model
     })
   });
 }
@@ -874,9 +963,9 @@ async function getTask(apiUrl, taskId) {
 async function waitForTask(apiUrl, taskId, timeoutMs = 12e4) {
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
-    const task = await getTask(apiUrl, taskId);
-    if (task.status === "done" || task.status === "failed")
-      return task;
+    const task2 = await getTask(apiUrl, taskId);
+    if (task2.status === "done" || task2.status === "failed")
+      return task2;
     await new Promise((r) => setTimeout(r, 300));
   }
   return { status: "timeout", error: "Task did not finish in time" };
@@ -914,8 +1003,169 @@ function countSpans(node) {
 
 // ../bench/dist/runner.js
 import fs from "node:fs/promises";
-import path4 from "node:path";
+import path5 from "node:path";
 import { execSync } from "node:child_process";
+
+// ../bench/dist/analyse.js
+function walkSpans(traceFlow, callback) {
+  if (!traceFlow)
+    return;
+  function visit(span) {
+    callback?.(span);
+    for (const child of span.children) {
+      visit(child);
+    }
+  }
+  for (const span of traceFlow.spans) {
+    visit(span);
+  }
+}
+function findSpans(traceFlow, predicate) {
+  const matches = [];
+  walkSpans(traceFlow, (span) => {
+    if (predicate(span))
+      matches.push(span);
+  });
+  return matches;
+}
+function spanError(span) {
+  if (span.status !== "error")
+    return void 0;
+  const err = span.attributes?.error;
+  return typeof err === "string" ? err : void 0;
+}
+function toolSpans(traceFlow) {
+  return findSpans(traceFlow, (s) => s.name.startsWith("agent.tool."));
+}
+function providerSpans(traceFlow) {
+  return findSpans(traceFlow, (s) => s.name === "provider.send");
+}
+function planSpan(traceFlow) {
+  return findSpans(traceFlow, (s) => s.name === "agent.plan").shift();
+}
+function validationSummary(result) {
+  if (!result.agentRun?.validationSummary)
+    return void 0;
+  try {
+    return JSON.parse(result.agentRun.validationSummary);
+  } catch {
+    return void 0;
+  }
+}
+function classifyFailure(result, traceFlow) {
+  const evidence = [];
+  if (result.status === "timeout") {
+    return {
+      category: "timeout",
+      rootCause: "Task did not finish within the timeout window.",
+      evidence: [`durationMs: ${String(result.durationMs)}`]
+    };
+  }
+  const validation = validationSummary(result);
+  if (validation?.allPassed === false) {
+    const failedSteps = [];
+    for (const key of ["lint", "test", "build"]) {
+      const step = validation[key];
+      if (step && !step.success) {
+        failedSteps.push(key);
+      }
+    }
+    return {
+      category: "validation_failure",
+      rootCause: `Validation failed: ${failedSteps.length > 0 ? failedSteps.join(", ") : "unspecified step"}.`,
+      evidence: failedSteps.map((s) => `${s} did not pass`)
+    };
+  }
+  const tools = toolSpans(traceFlow);
+  const failedTools = tools.filter((s) => s.status === "error");
+  if (failedTools.length > 0) {
+    const first = failedTools[0];
+    const toolName = first.name.replace("agent.tool.", "");
+    const err = spanError(first) ?? "unknown error";
+    evidence.push(`${toolName} failed: ${err}`);
+    if (toolName === "edit_file" || err.includes("old_string")) {
+      return {
+        category: "tool_misuse",
+        rootCause: "edit_file failed because the old_string did not match. The agent may not have read the file first or copied the exact text.",
+        evidence
+      };
+    }
+    return {
+      category: "tool_misuse",
+      rootCause: `Tool ${toolName} was invoked but failed.`,
+      evidence
+    };
+  }
+  const providers = providerSpans(traceFlow);
+  const failedProviders = providers.filter((s) => s.status === "error");
+  if (failedProviders.length > 0) {
+    const err = spanError(failedProviders[0]) ?? "provider error";
+    if (err.includes("JSON") || err.includes("parse") || err.includes("Unexpected token")) {
+      return {
+        category: "parse_error",
+        rootCause: "Provider response could not be parsed as the expected format.",
+        evidence: [err]
+      };
+    }
+  }
+  const plan = planSpan(traceFlow);
+  if (plan) {
+    const planSteps = plan.attributes?.planSteps;
+    const planStepsCount = typeof planSteps === "number" ? planSteps : 0;
+    if (planStepsCount === 0) {
+      return {
+        category: "plan_error",
+        rootCause: "The planner produced an empty plan with no actionable steps.",
+        evidence: ["planSteps: 0"]
+      };
+    }
+    const providerTurns = providers.length;
+    if (providerTurns > 10 && !result.evaluation.passed) {
+      return {
+        category: "plan_error",
+        rootCause: "The agent took many turns without finishing; the plan may be stuck or too vague.",
+        evidence: [`provider turns: ${String(providerTurns)}`, `plan steps: ${String(planStepsCount)}`]
+      };
+    }
+  }
+  const message = result.evaluation.message ?? result.agentRun?.resultStatus ?? "unknown";
+  return {
+    category: "unknown",
+    rootCause: `Failure cause could not be classified: ${message}.`,
+    evidence: [message]
+  };
+}
+function pickFocusResult(results, traceFlows) {
+  const failed = results.filter((r) => !r.evaluation.passed);
+  if (failed.length === 0)
+    return void 0;
+  const seenCategories = /* @__PURE__ */ new Set();
+  for (const result of failed) {
+    const analysis = result.failureAnalysis ?? classifyFailure(result, traceFlows.get(result.harnessTaskId));
+    if (!seenCategories.has(analysis.category)) {
+      seenCategories.add(analysis.category);
+      return result;
+    }
+  }
+  return failed[0];
+}
+function summariseFailures(results, traceFlows) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    if (result.evaluation.passed)
+      continue;
+    const analysis = result.failureAnalysis ?? classifyFailure(result, traceFlows.get(result.harnessTaskId));
+    const entry = counts.get(analysis.category) ?? { count: 0, examples: [] };
+    entry.count++;
+    if (entry.examples.length < 3) {
+      entry.examples.push(`${result.task.name}: ${analysis.rootCause}`);
+    }
+    counts.set(analysis.category, entry);
+  }
+  return Array.from(counts.entries()).map(([category, data]) => ({ category, ...data })).sort((a, b) => b.count - a.count);
+}
+
+// ../bench/dist/runner.js
 function countAllSpans(traceFlow) {
   if (!traceFlow)
     return 0;
@@ -933,6 +1183,13 @@ function ensureGitRepo(repoPath) {
   execSync("git add .", { cwd: repoPath, stdio: "ignore" });
   execSync('git commit -m "bench init"', { cwd: repoPath, stdio: "ignore" });
 }
+function sumUsage(a, b) {
+  return {
+    promptTokens: (a.promptTokens ?? 0) + (b?.promptTokens ?? 0),
+    completionTokens: (a.completionTokens ?? 0) + (b?.completionTokens ?? 0),
+    totalTokens: (a.totalTokens ?? 0) + (b?.totalTokens ?? 0)
+  };
+}
 async function runBenchmark(tasks, options) {
   const { apiUrl, suiteName, timeoutMs = 12e4, projectPrefix = "bench" } = options;
   const report = {
@@ -943,11 +1200,12 @@ async function runBenchmark(tasks, options) {
     failed: 0,
     timeouts: 0,
     totalDurationMs: 0,
+    totalUsage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
     results: []
   };
-  const baseDir = path4.join("/tmp", `omega-bench-${String(Date.now())}`);
+  const baseDir = path5.join("/tmp", `omega-bench-${String(Date.now())}`);
   await fs.mkdir(baseDir, { recursive: true });
-  for (const task of tasks) {
+  for (const task2 of tasks) {
     const start = Date.now();
     let harnessTaskId = "";
     let status = "failed";
@@ -958,17 +1216,17 @@ async function runBenchmark(tasks, options) {
     let projectId = "";
     let projectPath = "";
     try {
-      projectPath = path4.join(baseDir, task.id);
+      projectPath = path5.join(baseDir, task2.id);
       await fs.mkdir(projectPath, { recursive: true });
-      if (task.setup) {
-        await task.setup(projectPath);
+      if (task2.setup) {
+        await task2.setup(projectPath);
       }
       ensureGitRepo(projectPath);
-      const project = await ensureProject2(apiUrl, `${projectPrefix}-${task.id}`, projectPath);
+      const project = await ensureProject2(apiUrl, `${projectPrefix}-${task2.id}`, projectPath);
       projectId = project.id;
-      const harnessTask = await createTask(apiUrl, project.id, task.title, {
-        description: task.description,
-        complexity: task.complexity ?? "simple",
+      const harnessTask = await createTask(apiUrl, project.id, task2.title, {
+        description: task2.description,
+        complexity: task2.complexity ?? "simple",
         tags: ["benchmark", "agent"]
       });
       harnessTaskId = harnessTask.id;
@@ -980,7 +1238,7 @@ async function runBenchmark(tasks, options) {
         getDiffs(apiUrl, harnessTask.id),
         getTraceFlow(apiUrl, harnessTask.id)
       ]);
-      evaluation = await task.evaluate({
+      evaluation = await task2.evaluate({
         apiUrl,
         taskId: harnessTask.id,
         projectPath,
@@ -996,14 +1254,21 @@ async function runBenchmark(tasks, options) {
       };
     }
     const durationMs = Date.now() - start;
+    const usage = agentRun ? {
+      promptTokens: agentRun.promptTokens,
+      completionTokens: agentRun.completionTokens,
+      totalTokens: agentRun.totalTokens
+    } : void 0;
     const result = {
-      task,
+      task: task2,
       harnessTaskId,
       durationMs,
       status,
       evaluation,
       agentRun,
-      spanCount: countAllSpans(traceFlow)
+      spanCount: countAllSpans(traceFlow),
+      usage,
+      failureAnalysis: evaluation.passed ? void 0 : classifyFailure({ task: task2, harnessTaskId, durationMs, status, evaluation, agentRun, spanCount: countAllSpans(traceFlow), usage }, traceFlow)
     };
     if (status === "timeout")
       report.timeouts++;
@@ -1012,6 +1277,7 @@ async function runBenchmark(tasks, options) {
     else
       report.failed++;
     report.totalDurationMs += durationMs;
+    report.totalUsage = sumUsage(report.totalUsage, usage);
     report.results.push(result);
     options.onProgress?.(result);
   }
@@ -1020,7 +1286,7 @@ async function runBenchmark(tasks, options) {
 
 // ../bench/dist/report.js
 import fs2 from "node:fs/promises";
-import path5 from "node:path";
+import path6 from "node:path";
 function nowIso() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
@@ -1033,16 +1299,21 @@ function resultLine(r, idx) {
   const symbol = r.evaluation.passed ? "\u2713" : "\u2717";
   const status = r.status === "timeout" ? "timeout" : r.status;
   const score = r.evaluation.score !== void 0 ? ` (score ${String(r.evaluation.score)})` : "";
+  const tokens = r.usage?.totalTokens !== void 0 ? ` (${String(r.usage.totalTokens)} tokens)` : "";
+  const category = r.failureAnalysis ? ` [${r.failureAnalysis.category}]` : "";
   const msg = r.evaluation.message ? ` \u2014 ${r.evaluation.message}` : "";
-  return `${String(idx + 1)}. ${symbol} ${r.task.name} [${status}] ${formatDuration(r.durationMs)}${score}${msg}`;
+  return `${String(idx + 1)}. ${symbol} ${r.task.name} [${status}]${category} ${formatDuration(r.durationMs)}${tokens}${score}${msg}`;
+}
+function usageLine(usage) {
+  return `Tokens: ${String(usage.totalTokens ?? 0)} total (${String(usage.promptTokens ?? 0)} prompt, ${String(usage.completionTokens ?? 0)} completion)`;
 }
 async function writeReport(report, outputDir = ".omega/reports") {
   await fs2.mkdir(outputDir, { recursive: true });
   const ts = nowIso();
-  const jsonFile = path5.join(outputDir, `benchmark-${ts}.json`);
-  const mdFile = path5.join(outputDir, `benchmark-${ts}.md`);
-  await fs2.writeFile(jsonFile, JSON.stringify(report, null, 2), "utf-8");
-  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const jsonFile2 = path6.join(outputDir, `benchmark-${ts}.json`);
+  const mdFile = path6.join(outputDir, `benchmark-${ts}.md`);
+  await fs2.writeFile(jsonFile2, JSON.stringify(report, null, 2), "utf-8");
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   const md = [
     "# Omega Benchmark Report",
     "",
@@ -1052,8 +1323,9 @@ async function writeReport(report, outputDir = ".omega/reports") {
     `- Passed: ${String(report.passed)}`,
     `- Failed: ${String(report.failed)}`,
     `- Timeouts: ${String(report.timeouts)}`,
-    `- Pass rate: ${String(passRate)}%`,
+    `- Pass rate: ${String(passRate2)}%`,
     `- Total duration: ${formatDuration(report.totalDurationMs)}`,
+    `- ${usageLine(report.totalUsage)}`,
     "",
     "## Results",
     "",
@@ -1067,31 +1339,69 @@ async function writeReport(report, outputDir = ".omega/reports") {
     ""
   ].join("\n");
   await fs2.writeFile(mdFile, md, "utf-8");
-  return jsonFile;
+  return jsonFile2;
 }
 function printSummary(report) {
-  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   console.log(`
 Benchmark: ${report.suite}`);
   console.log(`Total: ${String(report.total)} | Passed: ${String(report.passed)} | Failed: ${String(report.failed)} | Timeouts: ${String(report.timeouts)}`);
-  console.log(`Pass rate: ${String(passRate)}%`);
+  console.log(`Pass rate: ${String(passRate2)}%`);
   console.log(`Duration: ${formatDuration(report.totalDurationMs)}`);
+  console.log(usageLine(report.totalUsage));
   for (const r of report.results) {
     console.log(resultLine(r, report.results.indexOf(r)));
   }
 }
 
 // ../bench/dist/suites/synthetic.js
+import fs4 from "node:fs/promises";
+
+// ../bench/dist/suites/builder.js
 import fs3 from "node:fs/promises";
-import path6 from "node:path";
-async function writeJson(projectPath, file, data) {
-  await fs3.writeFile(path6.join(projectPath, file), JSON.stringify(data, null, 2), "utf-8");
+import path7 from "node:path";
+function task(spec) {
+  return {
+    id: spec.id,
+    name: spec.name,
+    title: spec.title,
+    description: spec.description,
+    complexity: spec.complexity ?? "simple",
+    setup: spec.setup,
+    evaluate: spec.evaluate ?? expectValidationPassed()
+  };
 }
-async function writeFile2(projectPath, file, content) {
-  await fs3.mkdir(path6.dirname(path6.join(projectPath, file)), { recursive: true });
-  await fs3.writeFile(path6.join(projectPath, file), content, "utf-8");
+function basePackage(extraScripts = {}) {
+  return {
+    name: "synthetic-bench-project",
+    version: "1.0.0",
+    type: "module",
+    scripts: {
+      lint: "node lint.js",
+      test: "node test.js",
+      build: "node build.js",
+      ...extraScripts
+    }
+  };
+}
+async function jsonFile(projectPath, file, data) {
+  await fs3.mkdir(path7.dirname(path7.join(projectPath, file)), { recursive: true });
+  await fs3.writeFile(path7.join(projectPath, file), JSON.stringify(data, null, 2), "utf-8");
+}
+async function codeFile(projectPath, file, content) {
+  await fs3.mkdir(path7.dirname(path7.join(projectPath, file)), { recursive: true });
+  await fs3.writeFile(path7.join(projectPath, file), content, "utf-8");
+}
+async function nodeProject(projectPath, extraScripts = {}) {
+  await jsonFile(projectPath, "package.json", basePackage(extraScripts));
+  await codeFile(projectPath, "lint.js", `console.log('lint ok');
+`);
+  await codeFile(projectPath, "test.js", `console.log('test ok');
+`);
+  await codeFile(projectPath, "build.js", `console.log('build ok');
+`);
 }
-function parseValidation(ctx) {
+function readValidation(ctx) {
   if (!ctx.agentRun?.validationSummary)
     return { allPassed: false };
   try {
@@ -1101,142 +1411,598 @@ function parseValidation(ctx) {
     return { allPassed: false };
   }
 }
-function validate(ctx) {
-  const v = parseValidation(ctx);
-  return {
-    passed: v.allPassed,
-    message: v.allPassed ? "Validation passed" : "Validation failed"
+function expectValidationPassed() {
+  return (ctx) => {
+    const v = readValidation(ctx);
+    return {
+      passed: v.allPassed,
+      message: v.allPassed ? "Validation passed" : "Validation failed"
+    };
   };
 }
-function basePackage() {
-  return {
-    name: "synthetic-bench-project",
-    version: "1.0.0",
-    type: "module",
-    scripts: {
-      lint: "node lint.js",
-      test: "node test.js",
-      build: "node build.js"
+async function readFile2(projectPath, file) {
+  return fs3.readFile(path7.join(projectPath, file), "utf-8");
+}
+function expectFileContains(file, needle) {
+  return async (ctx) => {
+    const content = await readFile2(ctx.projectPath, file).catch(() => "");
+    const found = typeof needle === "string" ? content.includes(needle) : needle.test(content);
+    return {
+      passed: found,
+      message: found ? `${file} contains required content` : `${file} missing required content`
+    };
+  };
+}
+function combined(...evaluators) {
+  return async (ctx) => {
+    for (const evaluator of evaluators) {
+      const result = await evaluator(ctx);
+      if (!result.passed)
+        return result;
     }
+    return { passed: true, message: "All checks passed" };
   };
 }
-var TASKS = [
-  {
-    id: "greet-function",
-    name: "Add a greet function",
-    title: "Add greet.js with a greet function",
-    description: "Create a `greet.js` file that exports a function `greet(name)` returning `Hello, <name>!`. The existing test expects this function.",
-    complexity: "simple",
-    setup: async (projectPath) => {
-      await writeJson(projectPath, "package.json", basePackage());
-      await writeFile2(projectPath, "test.js", `import { greet } from './greet.js';
-import assert from 'node:assert';
-assert.strictEqual(greet('World'), 'Hello, World!');
-console.log('greet test passed');
+
+// ../bench/dist/suites/synthetic.js
+var lintAlwaysPass = `console.log('lint ok');
+`;
+var buildAlwaysPass = `console.log('build ok');
+`;
+function jsTest(name, body) {
+  return `import assert from 'node:assert';
+${body}
+console.log('${name} test passed');
+`;
+}
+function syntheticSuite() {
+  return [
+    // Simple edits
+    task({
+      id: "greet-function",
+      name: "Add a greet function",
+      title: "Add greet.js with a greet function",
+      description: "Create a `greet.js` file that exports a function `greet(name)` returning `Hello, <name>!`.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("greet", `import { greet } from './greet.js';
+assert.strictEqual(greet('World'), 'Hello, World!');`));
+      }
+    }),
+    task({
+      id: "update-config",
+      name: "Update a config value",
+      title: "Update timeout value in config.json",
+      description: "Change `config.json` so that `timeout` is `5000` instead of `1000`.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await jsonFile(projectPath, "config.json", { timeout: 1e3, retries: 3 });
+        await codeFile(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const cfg = JSON.parse(await readFile('./config.json', 'utf-8'));
+if (cfg.timeout !== 5000) { console.error('timeout is ' + cfg.timeout); process.exit(1); }
+console.log('config test passed');
 `);
-      await writeFile2(projectPath, "lint.js", `console.log('lint ok');
+      }
+    }),
+    task({
+      id: "add-readme",
+      name: "Add a README",
+      title: "Create README.md with a usage heading",
+      description: "Create a README.md file that contains a `# Usage` heading.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const readme = await readFile('./README.md', 'utf-8').catch(() => '');
+if (!readme.includes('# Usage')) { console.error('README missing Usage heading'); process.exit(1); }
+console.log('readme test passed');
 `);
-      await writeFile2(projectPath, "build.js", `console.log('build ok');
+      },
+      evaluate: expectFileContains("README.md", "# Usage")
+    }),
+    task({
+      id: "add-npm-script",
+      name: "Add an npm script",
+      title: "Add a `hello` script to package.json",
+      description: 'Add a `hello` script to package.json that prints "hello world".',
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", `import pkg from './package.json' assert { type: 'json' };
+if (pkg.scripts?.hello !== 'echo hello world') { console.error('missing hello script'); process.exit(1); }
+console.log('script test passed');
 `);
-    },
-    evaluate: (ctx) => Promise.resolve(validate(ctx))
-  },
-  {
-    id: "fix-lint-warning",
-    name: "Fix a lint warning",
-    title: "Remove unused variable flagged by linter",
-    description: "The custom linter flags any use of `var`. Update `src/index.js` so the lint script passes while keeping the greeting behavior.",
-    complexity: "simple",
-    setup: async (projectPath) => {
-      await writeJson(projectPath, "package.json", basePackage());
-      await writeFile2(projectPath, "src/index.js", `var greeting = 'Hello';
-export function hi(name) { return greeting + ', ' + name + '!'; }
+      }
+    }),
+    task({
+      id: "fix-typo",
+      name: "Fix a typo",
+      title: 'Fix spelling of "recieve" to "receive"',
+      description: 'The file src/message.js contains "recieve". Change it to "receive".',
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/message.js", `export const message = 'Please recieve this letter';
+`);
+        await codeFile(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const code = await readFile('./src/message.js', 'utf-8');
+if (code.includes('recieve')) { console.error('typo remains'); process.exit(1); }
+if (!code.includes('receive')) { console.error('correct spelling missing'); process.exit(1); }
+console.log('typo test passed');
+`);
+      }
+    }),
+    task({
+      id: "add-gitignore",
+      name: "Add a .gitignore",
+      title: "Ignore node_modules and .env",
+      description: "Create a .gitignore file that ignores node_modules and .env.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const gitignore = await readFile('./.gitignore', 'utf-8').catch(() => '');
+if (!gitignore.includes('node_modules') || !gitignore.includes('.env')) { console.error('.gitignore incomplete'); process.exit(1); }
+console.log('gitignore test passed');
+`);
+      }
+    }),
+    // Refactoring
+    task({
+      id: "extract-constant",
+      name: "Extract a magic constant",
+      title: "Extract MAX_RETRIES constant",
+      description: "Refactor `src/retry.js` so the magic number `3` is replaced by a top-level constant named `MAX_RETRIES`.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/retry.js", `export function attempts() { return 3; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("constant", `import fs from 'node:fs/promises';
+import { attempts } from './src/retry.js';
+const code = await fs.readFile('./src/retry.js', 'utf-8');
+if (!code.includes('const MAX_RETRIES')) { console.error('MAX_RETRIES constant not found'); process.exit(1); }
+if (attempts() !== 3) { console.error('attempts() changed'); process.exit(1); }`));
+      }
+    }),
+    task({
+      id: "rename-variable",
+      name: "Rename a variable",
+      title: "Rename `usr` to `user` in src/index.js",
+      description: "Rename all occurrences of the variable `usr` to `user` in src/index.js while preserving behavior.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/index.js", `export function welcome(usr) { return 'Welcome, ' + usr; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("rename", `import { readFile } from 'node:fs/promises';
+import { welcome } from './src/index.js';
+const code = await readFile('./src/index.js', 'utf-8');
+if (code.includes('usr')) { console.error('old name remains'); process.exit(1); }
+if (!code.includes('user')) { console.error('new name missing'); process.exit(1); }
+assert.strictEqual(welcome('Ada'), 'Welcome, Ada');`));
+      }
+    }),
+    task({
+      id: "remove-dead-code",
+      name: "Remove dead code",
+      title: "Remove unused helper function",
+      description: "Remove the unused `unusedHelper` function from src/utils.js.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/utils.js", `export function usedHelper() { return 1; }
+function unusedHelper() { return 2; }
+`);
+        await codeFile(projectPath, "test.js", jsTest("dead-code", `import { readFile } from 'node:fs/promises';
+import { usedHelper } from './src/utils.js';
+const code = await readFile('./src/utils.js', 'utf-8');
+if (code.includes('unusedHelper')) { console.error('dead code remains'); process.exit(1); }
+assert.strictEqual(usedHelper(), 1);`));
+      }
+    }),
+    task({
+      id: "split-function",
+      name: "Split a function",
+      title: "Split validateUser into smaller functions",
+      description: "Refactor src/validate.js so there is a separate `isEmailValid` function used by `validateUser`.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/validate.js", `export function validateUser(email) { return email.includes('@'); }
 `);
-      await writeFile2(projectPath, "test.js", `import { hi } from './src/index.js';
-import assert from 'node:assert';
-assert.strictEqual(hi('Omega'), 'Hello, Omega!');
-console.log('hi test passed');
+        await codeFile(projectPath, "test.js", jsTest("split", `import { readFile } from 'node:fs/promises';
+import { validateUser } from './src/validate.js';
+const code = await readFile('./src/validate.js', 'utf-8');
+if (!code.includes('function isEmailValid')) { console.error('isEmailValid not found'); process.exit(1); }
+assert.strictEqual(validateUser('a@b.com'), true);
+assert.strictEqual(validateUser('bad'), false);`));
+      }
+    }),
+    task({
+      id: "convert-var-to-const",
+      name: "Convert var to const",
+      title: "Replace var with const/let",
+      description: "Update src/index.js to use const and let instead of var.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath, {
+          lint: "node lint.js"
+        });
+        await codeFile(projectPath, "src/index.js", `var x = 1;
+var y = 2;
+export function sum() { return x + y; }
 `);
-      await writeFile2(projectPath, "lint.js", `import fs from 'node:fs/promises';
+        await codeFile(projectPath, "lint.js", `import fs from 'node:fs/promises';
 const code = await fs.readFile('./src/index.js', 'utf-8');
-if (code.includes('var ')) { console.error('lint failed: var keyword found'); process.exit(1); }
+if (code.includes('var ')) { console.error('var keyword found'); process.exit(1); }
 console.log('lint ok');
 `);
-      await writeFile2(projectPath, "build.js", `console.log('build ok');
+        await codeFile(projectPath, "test.js", jsTest("var", `import { sum } from './src/index.js';
+assert.strictEqual(sum(), 3);`));
+      }
+    }),
+    // Bug fixes
+    task({
+      id: "fix-null-guard",
+      name: "Add null guard",
+      title: "Guard against null input in getLength",
+      description: "Fix src/strings.js so getLength returns 0 for null/undefined input instead of throwing.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/strings.js", `export function getLength(value) { return value.length; }
 `);
-    },
-    evaluate: (ctx) => Promise.resolve(validate(ctx))
-  },
-  {
-    id: "update-config",
-    name: "Update a config value",
-    title: "Update timeout value in config.json",
-    description: "Change `config.json` so that `timeout` is `5000` instead of `1000`. The test will read the file and assert the value.",
-    complexity: "simple",
-    setup: async (projectPath) => {
-      await writeJson(projectPath, "package.json", basePackage());
-      await writeJson(projectPath, "config.json", { timeout: 1e3, retries: 3 });
-      await writeFile2(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
-const cfg = JSON.parse(await readFile('./config.json', 'utf-8'));
-if (cfg.timeout !== 5000) { console.error('timeout is ' + cfg.timeout); process.exit(1); }
-console.log('config test passed');
+        await codeFile(projectPath, "test.js", jsTest("null-guard", `import { getLength } from './src/strings.js';
+assert.strictEqual(getLength(null), 0);
+assert.strictEqual(getLength(undefined), 0);
+assert.strictEqual(getLength('abc'), 3);`));
+      }
+    }),
+    task({
+      id: "fix-off-by-one",
+      name: "Fix off-by-one",
+      title: "Fix range to include end value",
+      description: "Fix src/range.js so range(1, 3) returns [1, 2, 3].",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/range.js", `export function range(start, end) { const out = []; for (let i = start; i < end; i++) out.push(i); return out; }
 `);
-      await writeFile2(projectPath, "lint.js", `console.log('lint ok');
+        await codeFile(projectPath, "test.js", jsTest("range", `import { range } from './src/range.js';
+assert.deepStrictEqual(range(1, 3), [1, 2, 3]);`));
+      }
+    }),
+    task({
+      id: "fix-async-await",
+      name: "Fix async function",
+      title: "Await the fetch call in loadData",
+      description: "Fix src/api.js so loadData awaits the fetch response before returning JSON.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/api.js", `export async function loadData() { return fetch('data.json').then((r) => r.json()); }
 `);
-      await writeFile2(projectPath, "build.js", `console.log('build ok');
+        await codeFile(projectPath, "test.js", jsTest("async", `import { readFile } from 'node:fs/promises';
+import { loadData } from './src/api.js';
+const code = await readFile('./src/api.js', 'utf-8');
+if (!code.includes('await')) { console.error('missing await'); process.exit(1); }
+const result = await loadData();
+assert.strictEqual(result.ok, true);`));
+        await jsonFile(projectPath, "data.json", { ok: true });
+      },
+      evaluate: combined(expectFileContains("src/api.js", "await"), expectValidationPassed())
+    }),
+    task({
+      id: "fix-export",
+      name: "Fix missing export",
+      title: "Export the helper function",
+      description: "The function helper in src/lib.js is not exported. Export it so the test can import it.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/lib.js", `function helper() { return 42; }
 `);
-    },
-    evaluate: (ctx) => Promise.resolve(validate(ctx))
-  },
-  {
-    id: "extract-constant",
-    name: "Extract a magic constant",
-    title: "Extract MAX_RETRIES constant",
-    description: "Refactor `src/retry.js` so the magic number `3` is replaced by a top-level constant named `MAX_RETRIES`. The test checks for the constant.",
-    complexity: "medium",
-    setup: async (projectPath) => {
-      await writeJson(projectPath, "package.json", basePackage());
-      await writeFile2(projectPath, "src/retry.js", `export function attempts() { return 3; }
+        await codeFile(projectPath, "test.js", jsTest("export", `import { helper } from './src/lib.js';
+assert.strictEqual(helper(), 42);`));
+      }
+    }),
+    task({
+      id: "fix-import-path",
+      name: "Fix import path",
+      title: "Correct the import path in src/main.js",
+      description: "Fix the import path in src/main.js so it points to ./helpers.js.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/helpers.js", `export function greet() { return 'hi'; }
 `);
-      await writeFile2(projectPath, "test.js", `import fs from 'node:fs/promises';
-import { attempts } from './src/retry.js';
-const code = await fs.readFile('./src/retry.js', 'utf-8');
-if (!code.includes('const MAX_RETRIES')) { console.error('MAX_RETRIES constant not found'); process.exit(1); }
-if (attempts() !== 3) { console.error('attempts() changed'); process.exit(1); }
-console.log('constant test passed');
+        await codeFile(projectPath, "src/main.js", `import { greet } from './helper.js';
+export { greet };
+`);
+        await codeFile(projectPath, "test.js", jsTest("import", `import { greet } from './src/main.js';
+assert.strictEqual(greet(), 'hi');`));
+      }
+    }),
+    // Feature additions
+    task({
+      id: "add-cli-arg-parser",
+      name: "Add CLI arg parser",
+      title: "Add parseArgs function",
+      description: "Create src/cli.js exporting parseArgs that returns an object from process.argv style array.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("cli", `import { parseArgs } from './src/cli.js';
+const args = parseArgs(['node', 'script.js', '--name', 'Ada']);
+assert.strictEqual(args.name, 'Ada');`));
+      }
+    }),
+    task({
+      id: "add-validator",
+      name: "Add an email validator",
+      title: "Add isEmail function",
+      description: "Create src/validators.js exporting isEmail that returns true for valid emails.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("validator", `import { isEmail } from './src/validators.js';
+assert.strictEqual(isEmail('a@b.com'), true);
+assert.strictEqual(isEmail('not-an-email'), false);`));
+      }
+    }),
+    task({
+      id: "add-http-route",
+      name: "Add a simple route",
+      title: "Add /health route handler",
+      description: 'Add a /health route to src/server.js that responds with { status: "ok" }.',
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/server.js", `export function createServer() { return { handle: (req) => ({ statusCode: 404, body: 'not found' }) }; }
 `);
-      await writeFile2(projectPath, "lint.js", `console.log('lint ok');
+        await codeFile(projectPath, "test.js", jsTest("route", `import { createServer } from './src/server.js';
+const server = createServer();
+const res = server.handle({ url: '/health', method: 'GET' });
+assert.strictEqual(res.statusCode, 200);
+assert.deepStrictEqual(JSON.parse(res.body), { status: 'ok' });`));
+      }
+    }),
+    task({
+      id: "add-sum-function",
+      name: "Add a sum utility",
+      title: "Add sum(arr) function",
+      description: "Create src/math.js exporting sum that adds all numbers in an array.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("sum", `import { sum } from './src/math.js';
+assert.strictEqual(sum([1, 2, 3]), 6);
+assert.strictEqual(sum([]), 0);`));
+      }
+    }),
+    task({
+      id: "add-logger",
+      name: "Add a logger",
+      title: "Add createLogger function",
+      description: "Create src/logger.js exporting createLogger that returns an object with info/error methods.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("logger", `import { createLogger } from './src/logger.js';
+const logger = createLogger();
+assert.strictEqual(typeof logger.info, 'function');
+assert.strictEqual(typeof logger.error, 'function');`));
+      }
+    }),
+    // Tests
+    task({
+      id: "write-missing-test",
+      name: "Write a missing test",
+      title: "Write a test for double()",
+      description: "Create test.js that tests a src/math.js double() function and make it pass.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/math.js", `export function double(n) { return n * 2; }
 `);
-      await writeFile2(projectPath, "build.js", `console.log('build ok');
+        await codeFile(projectPath, "test.js", `import assert from 'node:assert';
+import { double } from './src/math.js';
+assert.strictEqual(double(3), 6);
+console.log('double test passed');
 `);
-    },
-    evaluate: (ctx) => Promise.resolve(validate(ctx))
-  },
-  {
-    id: "noop-validation",
-    name: "Run validation on a clean project",
-    title: "Confirm the project already passes validation",
-    description: "This project already passes lint, test and build. Use the publish tool to confirm validation passes and finish.",
-    complexity: "simple",
-    setup: async (projectPath) => {
-      await writeJson(projectPath, "package.json", basePackage());
-      await writeFile2(projectPath, "test.js", `console.log('test ok');
+      }
+    }),
+    task({
+      id: "make-failing-test-pass",
+      name: "Make failing test pass",
+      title: "Implement capitalize()",
+      description: "Implement src/strings.js capitalize() so the provided test passes.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/strings.js", `export function capitalize(s) { return s; }
 `);
-      await writeFile2(projectPath, "lint.js", `console.log('lint ok');
+        await codeFile(projectPath, "test.js", jsTest("capitalize", `import { capitalize } from './src/strings.js';
+assert.strictEqual(capitalize('hello'), 'Hello');`));
+      }
+    }),
+    // Config/package
+    task({
+      id: "fix-package-type",
+      name: "Fix package type",
+      title: "Set package type to module",
+      description: 'Update package.json so "type" is "module".',
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await jsonFile(projectPath, "package.json", {
+          name: "synthetic-bench-project",
+          version: "1.0.0",
+          type: "commonjs",
+          scripts: { lint: "node lint.js", test: "node test.js", build: "node build.js" }
+        });
+        await codeFile(projectPath, "lint.js", lintAlwaysPass);
+        await codeFile(projectPath, "test.js", `console.log('test ok');
 `);
-      await writeFile2(projectPath, "build.js", `console.log('build ok');
+        await codeFile(projectPath, "build.js", buildAlwaysPass);
+      },
+      evaluate: combined(async (ctx) => {
+        const pkg = JSON.parse(await fs4.readFile(`${ctx.projectPath}/package.json`, "utf-8"));
+        return {
+          passed: pkg.type === "module",
+          message: pkg.type === "module" ? "package type is module" : "package type not module"
+        };
+      }, expectValidationPassed())
+    }),
+    task({
+      id: "update-dependency",
+      name: "Update dependency version",
+      title: "Bump lodash to 4.17.21",
+      description: "Update package.json so lodash dependency version is 4.17.21.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await jsonFile(projectPath, "package.json", {
+          name: "synthetic-bench-project",
+          version: "1.0.0",
+          type: "module",
+          dependencies: { lodash: "4.17.20" },
+          scripts: { lint: "node lint.js", test: "node test.js", build: "node build.js" }
+        });
+        await codeFile(projectPath, "lint.js", lintAlwaysPass);
+        await codeFile(projectPath, "test.js", `import pkg from './package.json' assert { type: 'json' };
+if (pkg.dependencies?.lodash !== '4.17.21') { console.error('lodash not updated'); process.exit(1); }
+console.log('dependency test passed');
 `);
-    },
-    evaluate: (ctx) => Promise.resolve(validate(ctx))
-  }
-];
-function syntheticSuite() {
-  return TASKS;
+        await codeFile(projectPath, "build.js", buildAlwaysPass);
+      }
+    }),
+    // Multi-file
+    task({
+      id: "extract-shared-utility",
+      name: "Extract shared utility",
+      title: "Move formatDate to src/date.js and update imports",
+      description: "Extract the formatDate function from src/a.js and src/b.js into src/date.js, then update imports.",
+      complexity: "medium",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/a.js", `function formatDate(d) { return d.toISOString().split('T')[0]; }
+export function runA(d) { return formatDate(d); }
+`);
+        await codeFile(projectPath, "src/b.js", `function formatDate(d) { return d.toISOString().split('T')[0]; }
+export function runB(d) { return formatDate(d); }
+`);
+        await codeFile(projectPath, "test.js", jsTest("shared", `import { readFile } from 'node:fs/promises';
+import { runA } from './src/a.js';
+import { runB } from './src/b.js';
+const a = await readFile('./src/a.js', 'utf-8');
+const b = await readFile('./src/b.js', 'utf-8');
+if (!a.includes('./date.js') || !b.includes('./date.js')) { console.error('imports not updated'); process.exit(1); }
+const d = new Date('2024-01-15');
+assert.strictEqual(runA(d), '2024-01-15');
+assert.strictEqual(runB(d), '2024-01-15');`));
+      }
+    }),
+    task({
+      id: "fix-circular-import",
+      name: "Fix circular import",
+      title: "Break circular dependency between a.js and b.js",
+      description: "Refactor src/a.js and src/b.js to remove the circular dependency while keeping both exports working.",
+      complexity: "complex",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "src/a.js", `import { valueB } from './b.js';
+export const valueA = 'A' + valueB;
+`);
+        await codeFile(projectPath, "src/b.js", `import { valueA } from './a.js';
+export const valueB = 'B' + valueA;
+`);
+        await codeFile(projectPath, "test.js", jsTest("circular", `import { valueA, valueB } from './src/a.js';
+assert.ok(typeof valueA === 'string');
+assert.ok(typeof valueB === 'string');`));
+      }
+    }),
+    // Documentation
+    task({
+      id: "update-readme-example",
+      name: "Update README example",
+      title: "Add a usage example to README.md",
+      description: "Edit README.md to include a code block example showing how to use greet().",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "README.md", `# Project
+
+A demo project.
+`);
+        await codeFile(projectPath, "test.js", [
+          "import { readFile } from 'node:fs/promises';",
+          "const readme = await readFile('./README.md', 'utf-8');",
+          "if (!readme.includes('```')) { console.error('README missing code block'); process.exit(1); }",
+          "if (!readme.includes('greet')) { console.error('README missing greet example'); process.exit(1); }",
+          "console.log('readme test passed');",
+          ""
+        ].join("\n"));
+      }
+    }),
+    // Validation / noop
+    task({
+      id: "noop-validation",
+      name: "Run validation on a clean project",
+      title: "Confirm the project already passes validation",
+      description: "This project already passes lint, test and build. Use the publish tool to confirm validation passes and finish.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+      }
+    }),
+    // Lint / format
+    task({
+      id: "fix-lint-warning",
+      name: "Fix a lint warning",
+      title: "Remove unused variable flagged by linter",
+      description: "The custom linter flags any use of `var`. Update src/index.js so the lint script passes while keeping the greeting behavior.",
+      complexity: "simple",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath, {
+          lint: "node lint.js"
+        });
+        await codeFile(projectPath, "src/index.js", `var greeting = 'Hello';
+export function hi(name) { return greeting + ', ' + name + '!'; }
+`);
+        await codeFile(projectPath, "lint.js", `import fs from 'node:fs/promises';
+const code = await fs.readFile('./src/index.js', 'utf-8');
+if (code.includes('var ')) { console.error('lint failed: var keyword found'); process.exit(1); }
+console.log('lint ok');
+`);
+        await codeFile(projectPath, "test.js", jsTest("hi", `import { hi } from './src/index.js';
+assert.strictEqual(hi('Omega'), 'Hello, Omega!');`));
+      }
+    }),
+    // Complex
+    task({
+      id: "implement-todo-store",
+      name: "Implement a todo store",
+      title: "Implement TodoStore with add/complete/list",
+      description: "Create src/todo.js exporting TodoStore class with add(text), complete(id), and list() methods.",
+      complexity: "complex",
+      setup: async (projectPath) => {
+        await nodeProject(projectPath);
+        await codeFile(projectPath, "test.js", jsTest("todo", `import { TodoStore } from './src/todo.js';
+const store = new TodoStore();
+const id = store.add('Buy milk');
+assert.deepStrictEqual(store.list(), [{ id, text: 'Buy milk', done: false }]);
+store.complete(id);
+assert.strictEqual(store.list()[0].done, true);`));
+      }
+    })
+  ];
 }
 
 // ../bench/dist/adapters/deepswe.js
-import fs4 from "node:fs/promises";
-import path7 from "node:path";
+import fs5 from "node:fs/promises";
+import path8 from "node:path";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 var execFileAsync = promisify(execFile);
@@ -1278,18 +2044,18 @@ function mulberry32(seed) {
   };
 }
 async function readTask(taskDir) {
-  const tomlRaw = await fs4.readFile(path7.join(taskDir, "task.toml"), "utf-8");
-  const instructionRaw = await fs4.readFile(path7.join(taskDir, "instruction.md"), "utf-8");
+  const tomlRaw = await fs5.readFile(path8.join(taskDir, "task.toml"), "utf-8");
+  const instructionRaw = await fs5.readFile(path8.join(taskDir, "instruction.md"), "utf-8");
   return { toml: parseToml(tomlRaw), instruction: instructionRaw };
 }
 async function cloneRepo(repoUrl, commit, targetPath) {
-  await fs4.mkdir(path7.dirname(targetPath), { recursive: true });
+  await fs5.mkdir(path8.dirname(targetPath), { recursive: true });
   await execFileAsync("git", ["clone", repoUrl, targetPath], { timeout: 12e4 });
   await execFileAsync("git", ["-C", targetPath, "checkout", commit], { timeout: 6e4 });
 }
 async function loadDeepSWESuite(options) {
-  const entries = await fs4.readdir(options.tasksDir, { withFileTypes: true });
-  const taskDirs = entries.filter((e) => e.isDirectory()).map((e) => path7.join(options.tasksDir, e.name));
+  const entries = await fs5.readdir(options.tasksDir, { withFileTypes: true });
+  const taskDirs = entries.filter((e) => e.isDirectory()).map((e) => path8.join(options.tasksDir, e.name));
   let loaded = [];
   for (const dir of taskDirs) {
     try {
@@ -1300,7 +2066,7 @@ async function loadDeepSWESuite(options) {
   }
   if (options.taskIds && options.taskIds.length > 0) {
     loaded = loaded.filter((t) => {
-      const id = t.toml.metadata?.task_id ?? path7.basename(t.dir);
+      const id = t.toml.metadata?.task_id ?? path8.basename(t.dir);
       return options.taskIds?.includes(id);
     });
   }
@@ -1311,7 +2077,7 @@ async function loadDeepSWESuite(options) {
   }
   const tasks = [];
   for (const { dir, toml, instruction } of loaded) {
-    const id = toml.metadata?.task_id ?? path7.basename(dir);
+    const id = toml.metadata?.task_id ?? path8.basename(dir);
     const title = toml.metadata?.display_title ?? toml.metadata?.original_title ?? id;
     const repo = toml.metadata?.repository_url;
     const commit = toml.metadata?.base_commit_hash;
@@ -1342,65 +2108,110 @@ async function loadDeepSWESuite(options) {
 }
 
 // ../bench/dist/optimise.js
-import fs5 from "node:fs/promises";
-import path8 from "node:path";
+import fs6 from "node:fs/promises";
+import path9 from "node:path";
 async function findLatestReport(outputDir) {
-  const entries = await fs5.readdir(outputDir).catch(() => []);
+  const entries = await fs6.readdir(outputDir).catch(() => []);
   const files = entries.filter((f) => f.startsWith("benchmark-") && f.endsWith(".json")).sort().reverse();
   if (files.length === 0)
     return void 0;
-  const raw = await fs5.readFile(path8.join(outputDir, files[0]), "utf-8");
+  const raw = await fs6.readFile(path9.join(outputDir, files[0]), "utf-8");
   return JSON.parse(raw);
 }
-function summariseTraceFlow(report, maxLines = 60) {
-  const lines = report.split("\n");
-  if (lines.length <= maxLines)
-    return report;
-  return lines.slice(0, maxLines).join("\n") + "\n... (truncated)";
+function extractToolErrors(traceFlow) {
+  if (!traceFlow)
+    return [];
+  const errors = [];
+  function visit(span) {
+    if (span.name.startsWith("agent.tool.") && span.status === "error") {
+      const tool = span.name.replace("agent.tool.", "");
+      const message = typeof span.attributes?.error === "string" ? span.attributes.error : "unknown error";
+      errors.push(`- ${tool}: ${message}`);
+    }
+    for (const child of span.children) {
+      visit(child);
+    }
+  }
+  for (const root of traceFlow.spans) {
+    visit(root);
+  }
+  return errors.slice(0, 5);
 }
 function buildOptimisePrompt(report, failedResult, traceFlowText) {
-  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
-  const failed = report.results.filter((r) => !r.evaluation.passed);
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   const lines = [];
-  lines.push("# Prompt optimisation task");
+  lines.push("Task: Improve the Omega agent prompts in packages/agent/src/prompts.ts");
   lines.push("");
-  lines.push(`The latest benchmark run achieved ${String(passRate)}% pass rate (${String(report.passed)}/${String(report.total)}).`);
+  lines.push(`The latest benchmark run achieved ${String(passRate2)}% pass rate (${String(report.passed)}/${String(report.total)}).`);
   lines.push("");
-  lines.push("## Failed tasks");
-  for (const r of failed.slice(0, 5)) {
-    lines.push(`- ${r.task.name}: ${r.evaluation.message ?? "no message"}`);
+  lines.push("## Failure summary");
+  const traceFlows = /* @__PURE__ */ new Map();
+  const failureSummary = summariseFailures(report.results, traceFlows);
+  for (const summary of failureSummary) {
+    lines.push(`- ${summary.category}: ${String(summary.count)} task(s)`);
+    for (const example of summary.examples) {
+      lines.push(`  - ${example}`);
+    }
   }
   lines.push("");
   if (failedResult) {
-    lines.push(`## Focus task: ${failedResult.task.name}`);
-    lines.push(`Status: ${failedResult.status}`);
-    lines.push(`Evaluation: ${failedResult.evaluation.message ?? "none"}`);
+    const analysis = failedResult.failureAnalysis ?? classifyFailure(failedResult, void 0);
+    lines.push(`## Focus failure: ${failedResult.task.name}`);
+    lines.push(`Category: ${analysis.category}`);
+    lines.push(`Root cause: ${analysis.rootCause}`);
+    if (analysis.evidence.length > 0) {
+      lines.push("Evidence:");
+      for (const ev of analysis.evidence) {
+        lines.push(`- ${ev}`);
+      }
+    }
     lines.push("");
   }
   if (traceFlowText) {
-    lines.push("## Trace flow snapshot");
-    lines.push("```json");
-    lines.push(summariseTraceFlow(traceFlowText));
-    lines.push("```");
-    lines.push("");
+    const traceFlow = JSON.parse(traceFlowText);
+    const toolErrors = extractToolErrors(traceFlow);
+    if (toolErrors.length > 0) {
+      lines.push("## Failed tool calls");
+      for (const err of toolErrors) {
+        lines.push(err);
+      }
+      lines.push("");
+    }
   }
   lines.push("## Instructions");
-  lines.push("Analyse the failures above and edit `packages/agent/src/prompts.ts` (and optionally `packages/agent/src/planner.ts`) to improve the Omega agent.");
-  lines.push("Rules:");
-  lines.push("- Keep changes minimal and targeted.");
-  lines.push("- Do not break existing tool signatures or system behaviour.");
-  lines.push("- Run `pnpm lint` and `pnpm test` after edits.");
-  lines.push("- Finish with a summary of what changed and why it should help.");
+  lines.push("1. Read packages/agent/src/prompts.ts.");
+  lines.push("2. Make a small, targeted change to the system prompt or text-tools prompt that fixes the focus failure.");
+  if (failedResult) {
+    const analysis = failedResult.failureAnalysis ?? classifyFailure(failedResult, void 0);
+    const categoryActions = {
+      timeout: "Add a stronger instruction to finish promptly and avoid long reasoning chains.",
+      validation_failure: "Add clearer instructions to run validation commands (pnpm lint, pnpm test) and fix errors before finishing.",
+      tool_misuse: "Add clearer instructions and examples for using edit_file and write_file correctly, including reading the file first.",
+      parse_error: "Add clearer instructions to respond with valid JSON tool_calls and no markdown wrappers.",
+      plan_error: "Add stronger instructions to create a concise plan and execute it without over-thinking or repeating actions.",
+      unknown: "Add stronger general instructions to read the task, act with tools, validate, and finish."
+    };
+    lines.push(`3. Concrete focus: ${categoryActions[analysis.category]}`);
+  }
+  lines.push("4. Run `pnpm lint` and `pnpm test` after edits.");
+  lines.push("5. Finish with a summary of what changed and why it should help.");
   return lines.join("\n");
 }
 async function loadOptimisationContext(apiUrl, outputDir) {
   const report = await findLatestReport(outputDir);
   if (!report)
     return void 0;
-  const failedResult = report.results.find((r) => !r.evaluation.passed);
+  const traceFlows = /* @__PURE__ */ new Map();
+  for (const result of report.results) {
+    if (result.evaluation.passed)
+      continue;
+    const traceFlow = result.harnessTaskId ? await getTraceFlow(apiUrl, result.harnessTaskId) : void 0;
+    traceFlows.set(result.harnessTaskId, traceFlow ?? void 0);
+  }
+  const failedResult = pickFocusResult(report.results, traceFlows);
   let traceFlowText;
   if (failedResult?.harnessTaskId) {
-    const traceFlow = await getTraceFlow(apiUrl, failedResult.harnessTaskId);
+    const traceFlow = traceFlows.get(failedResult.harnessTaskId);
     if (traceFlow) {
       traceFlowText = JSON.stringify(traceFlow, null, 2);
     }
@@ -1425,6 +2236,92 @@ async function submitOptimiseTask(apiUrl, projectId, prompt) {
   return res.json();
 }
 
+// ../bench/dist/ab.js
+function passRate(report) {
+  return report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+}
+function findResult(report, taskId) {
+  return report.results.find((r) => r.task.id === taskId);
+}
+function buildAbReport(baseline, candidate, baselineReport, candidateReport) {
+  const baselineRate = passRate(baselineReport);
+  const candidateRate = passRate(candidateReport);
+  const taskIds = /* @__PURE__ */ new Set();
+  for (const r of baselineReport.results)
+    taskIds.add(r.task.id);
+  for (const r of candidateReport.results)
+    taskIds.add(r.task.id);
+  const perTask = Array.from(taskIds).map((taskId) => {
+    const base = findResult(baselineReport, taskId);
+    const cand = findResult(candidateReport, taskId);
+    return {
+      taskId,
+      taskName: base?.task.name ?? cand?.task.name ?? taskId,
+      baselinePassed: base?.evaluation.passed ?? false,
+      candidatePassed: cand?.evaluation.passed ?? false,
+      baselineTokens: base?.usage?.totalTokens,
+      candidateTokens: cand?.usage?.totalTokens
+    };
+  });
+  return {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    baseline,
+    candidate,
+    baselineReport,
+    candidateReport,
+    delta: {
+      passRate: candidateRate - baselineRate,
+      passedDelta: candidateReport.passed - baselineReport.passed,
+      durationMsDelta: candidateReport.totalDurationMs - baselineReport.totalDurationMs,
+      promptTokensDelta: (candidateReport.totalUsage.promptTokens ?? 0) - (baselineReport.totalUsage.promptTokens ?? 0),
+      completionTokensDelta: (candidateReport.totalUsage.completionTokens ?? 0) - (baselineReport.totalUsage.completionTokens ?? 0),
+      totalTokensDelta: (candidateReport.totalUsage.totalTokens ?? 0) - (baselineReport.totalUsage.totalTokens ?? 0)
+    },
+    perTask
+  };
+}
+function deltaSymbol(value) {
+  if (value > 0)
+    return "+";
+  if (value < 0)
+    return "";
+  return "";
+}
+function formatDelta(value) {
+  return `${deltaSymbol(value)}${String(value)}`;
+}
+function formatAbReport(report) {
+  const lines = [];
+  lines.push("# Omega A/B Benchmark Report");
+  lines.push("");
+  lines.push(`- Timestamp: ${report.timestamp}`);
+  lines.push(`- Baseline: ${report.baseline.name}`);
+  lines.push(`- Candidate: ${report.candidate.name}`);
+  lines.push("");
+  lines.push("## Summary");
+  lines.push(`| Metric | Baseline | Candidate | Delta |`);
+  lines.push(`|---|---|---|---|`);
+  lines.push(`| Pass rate | ${String(passRate(report.baselineReport))}% | ${String(passRate(report.candidateReport))}% | ${formatDelta(report.delta.passRate)}% |`);
+  lines.push(`| Passed | ${String(report.baselineReport.passed)}/${String(report.baselineReport.total)} | ${String(report.candidateReport.passed)}/${String(report.candidateReport.total)} | ${formatDelta(report.delta.passedDelta)} |`);
+  lines.push(`| Duration | ${formatDuration2(report.baselineReport.totalDurationMs)} | ${formatDuration2(report.candidateReport.totalDurationMs)} | ${formatDuration2(report.delta.durationMsDelta)} |`);
+  lines.push(`| Tokens | ${String(report.baselineReport.totalUsage.totalTokens ?? 0)} | ${String(report.candidateReport.totalUsage.totalTokens ?? 0)} | ${formatDelta(report.delta.totalTokensDelta)} |`);
+  lines.push("");
+  lines.push("## Per-task comparison");
+  lines.push(`| Task | Baseline | Candidate | Token delta |`);
+  lines.push(`|---|---|---|---|`);
+  for (const t of report.perTask) {
+    const tokenDelta = t.baselineTokens !== void 0 && t.candidateTokens !== void 0 ? formatDelta(t.candidateTokens - t.baselineTokens) : "-";
+    lines.push(`| ${t.taskName} | ${t.baselinePassed ? "pass" : "fail"} | ${t.candidatePassed ? "pass" : "fail"} | ${tokenDelta} |`);
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+function formatDuration2(ms) {
+  if (ms < 1e3)
+    return `${String(Math.round(ms))}ms`;
+  return `${(ms / 1e3).toFixed(1)}s`;
+}
+
 // ../../apps/cli/src/commands/bench.ts
 async function waitForApi2(apiUrl, maxMs = 1e4) {
   const deadline = Date.now() + maxMs;
@@ -1440,10 +2337,10 @@ async function waitForApi2(apiUrl, maxMs = 1e4) {
 }
 function currentProject(apiUrl) {
   const cwd = process.cwd();
-  const name = path9.basename(cwd);
+  const name = path11.basename(cwd);
   return ensureProject2(apiUrl, `bench-${name}`, cwd);
 }
-var runCmd = new Command8("run").description("Run a benchmark suite").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks (for deep-swe)", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").action(async (opts) => {
+var runCmd = new Command8("run").description("Run a benchmark suite").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").action(async (opts) => {
   const apiUrl = getApiUrl();
   await waitForApi2(apiUrl);
   const timeoutMs = Number(opts.timeout);
@@ -1461,6 +2358,9 @@ var runCmd = new Command8("run").description("Run a benchmark suite").option("--
     suiteName = "deep-swe";
   } else if (opts.suite === "synthetic") {
     tasks = syntheticSuite();
+    if (opts.nTasks !== void 0 && opts.nTasks > 0) {
+      tasks = tasks.slice(0, opts.nTasks);
+    }
     suiteName = "synthetic";
   } else {
     throw new Error(`Unknown suite: ${opts.suite}`);
@@ -1492,17 +2392,254 @@ var optimiseCmd = new Command8("optimise").description("Create a self-improve ta
     console.log("No benchmark report found. Run `omega bench run` first.");
     return;
   }
+  const { loadCurrentPrompts: loadCurrentPrompts2 } = await Promise.resolve().then(() => (init_prompt_versioning(), prompt_versioning_exports));
+  const current = await loadCurrentPrompts2();
+  const versionRes = await fetch(`${apiUrl}/prompt-versions`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      name: current.name,
+      sourcePath: current.sourcePath,
+      systemPrompt: current.systemPrompt,
+      textToolsPrompt: current.textToolsPrompt,
+      hash: current.hash,
+      metadata: { planningPrompt: current.planningPrompt }
+    })
+  });
+  if (versionRes.ok) {
+    const version = await versionRes.json();
+    console.log(`Saved baseline prompt version: ${version.name} (${version.id})`);
+  } else {
+    console.warn("Could not save prompt version:", await versionRes.text());
+  }
   const project = await currentProject(apiUrl);
   const prompt = buildOptimisePrompt(
     context.report,
     context.failedResult,
     context.traceFlowText
   );
-  const task = await submitOptimiseTask(apiUrl, project.id, prompt);
-  console.log(`Created self-improve task ${task.id}`);
-  console.log(`Run \`omega task run ${task.id}\` to execute it.`);
+  const task2 = await submitOptimiseTask(apiUrl, project.id, prompt);
+  console.log(`Created self-improve task ${task2.id}`);
+  console.log(`Run \`omega task run ${task2.id}\` to execute it.`);
+});
+async function loadPromptSpec(source) {
+  if (source === "current") {
+    const { readPromptsSource: readPromptsSource2 } = await Promise.resolve().then(() => (init_prompt_versioning(), prompt_versioning_exports));
+    const prompts = await readPromptsSource2();
+    return { name: "current", ...prompts };
+  }
+  const resolved = path11.resolve(source);
+  const raw = await fs8.readFile(resolved, "utf-8");
+  const parsed = JSON.parse(raw);
+  if (!parsed.name || !parsed.systemPrompt || !parsed.textToolsPrompt) {
+    throw new Error("Prompt file must include name, systemPrompt, and textToolsPrompt");
+  }
+  return parsed;
+}
+async function writePromptFile(spec) {
+  const tmpDir = await fs8.mkdtemp(path11.join(os.tmpdir(), "omega-prompt-"));
+  const file = path11.join(tmpDir, "prompts.json");
+  await fs8.writeFile(file, JSON.stringify(spec, null, 2), "utf-8");
+  return file;
+}
+async function runBenchmarkWithPrompts(tasks, spec, options) {
+  const promptFile = await writePromptFile(spec);
+  const env = {
+    ...process.env,
+    OMEGA_SYSTEM_PROMPT_FILE: promptFile,
+    OMEGA_TEXT_TOOLS_PROMPT_FILE: promptFile
+  };
+  let apiUrl = options.apiUrl;
+  let server;
+  if (!apiUrl) {
+    const started = await startHarnessServer(env);
+    apiUrl = started.apiUrl;
+    server = started.process;
+  }
+  try {
+    await waitForApi2(apiUrl, 3e4);
+    const report = await runBenchmark(tasks, {
+      apiUrl,
+      suiteName: options.suiteName,
+      timeoutMs: options.timeoutMs,
+      onProgress: (result) => {
+        const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
+        console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
+      }
+    });
+    return { report, apiUrl, server };
+  } catch (err) {
+    server?.kill();
+    throw err;
+  }
+}
+var abCmd = new Command8("ab").description("Run an A/B benchmark comparing two prompt versions").option("--baseline <source>", 'baseline prompts: "current" or path to JSON file', "current").option("--candidate <source>", "candidate prompts: path to JSON file").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").action(async (opts) => {
+  if (!opts.candidate) {
+    throw new Error("--candidate is required");
+  }
+  const timeoutMs = Number(opts.timeout);
+  const existingApiUrl = getApiUrl();
+  let tasks;
+  let suiteName;
+  if (opts.suite === "deep-swe") {
+    throw new Error("DeepSWE A/B not yet supported; use --suite synthetic");
+  } else if (opts.suite === "synthetic") {
+    tasks = syntheticSuite();
+    if (opts.nTasks !== void 0 && opts.nTasks > 0) {
+      tasks = tasks.slice(0, opts.nTasks);
+    }
+    suiteName = "synthetic";
+  } else {
+    throw new Error(`Unknown suite: ${opts.suite}`);
+  }
+  const baselineSpec = await loadPromptSpec(opts.baseline);
+  const candidateSpec = await loadPromptSpec(opts.candidate);
+  console.log(`Baseline: ${baselineSpec.name}`);
+  console.log(`Candidate: ${candidateSpec.name}`);
+  console.log(`Running ${String(tasks.length)} tasks for baseline...`);
+  const { report: baselineReport, server: baseServer } = await runBenchmarkWithPrompts(tasks, baselineSpec, {
+    apiUrl: existingApiUrl,
+    suiteName: `${suiteName}-baseline`,
+    timeoutMs
+  });
+  baseServer?.kill();
+  console.log(`Running ${String(tasks.length)} tasks for candidate...`);
+  const { report: candidateReport, server: candServer } = await runBenchmarkWithPrompts(tasks, candidateSpec, {
+    apiUrl: existingApiUrl,
+    suiteName: `${suiteName}-candidate`,
+    timeoutMs
+  });
+  candServer?.kill();
+  const abReport = buildAbReport(baselineSpec, candidateSpec, baselineReport, candidateReport);
+  await fs8.mkdir(opts.outputDir, { recursive: true });
+  const ts = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+  const jsonFile2 = path11.join(opts.outputDir, `ab-${ts}.json`);
+  const mdFile = path11.join(opts.outputDir, `ab-${ts}.md`);
+  await fs8.writeFile(jsonFile2, JSON.stringify(abReport, null, 2), "utf-8");
+  await fs8.writeFile(mdFile, formatAbReport(abReport), "utf-8");
+  console.log("\n" + formatAbReport(abReport));
+  console.log(`
+A/B report written to ${jsonFile2}`);
+});
+async function savePromptVersion(apiUrl, name) {
+  const { loadCurrentPrompts: loadCurrentPrompts2 } = await Promise.resolve().then(() => (init_prompt_versioning(), prompt_versioning_exports));
+  const current = await loadCurrentPrompts2();
+  const versionRes = await fetch(`${apiUrl}/prompt-versions`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      name: name ?? current.name,
+      sourcePath: current.sourcePath,
+      systemPrompt: current.systemPrompt,
+      textToolsPrompt: current.textToolsPrompt,
+      hash: current.hash,
+      metadata: { planningPrompt: current.planningPrompt }
+    })
+  });
+  if (!versionRes.ok) {
+    console.warn("Could not save prompt version:", await versionRes.text());
+    return void 0;
+  }
+  return await versionRes.json();
+}
+async function runLoopIteration(apiUrl, projectId, tasks, suiteName, timeoutMs, outputDir, provider, model) {
+  const beforeVersion = await savePromptVersion(apiUrl, `loop-before-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`);
+  if (beforeVersion) {
+    console.log(`Saved before-loop prompt version: ${beforeVersion.name} (${beforeVersion.id})`);
+  }
+  console.log(`
+Running ${String(tasks.length)} benchmark tasks...`);
+  const report = await runBenchmark(tasks, {
+    apiUrl,
+    suiteName,
+    timeoutMs,
+    onProgress: (result) => {
+      const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
+      console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
+    }
+  });
+  await writeReport(report, outputDir);
+  printSummary(report);
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  if (passRate2 === 100) {
+    console.log("\nAll tasks passed. Loop complete.");
+    return { report, improved: false };
+  }
+  const context = await loadOptimisationContext(apiUrl, outputDir);
+  if (!context) {
+    throw new Error("Benchmark report could not be loaded for optimisation");
+  }
+  const prompt = buildOptimisePrompt(context.report, context.failedResult, context.traceFlowText);
+  const task2 = await submitOptimiseTask(apiUrl, projectId, prompt);
+  if (provider || model) {
+    await fetch(`${apiUrl}/tasks/${task2.id}`, {
+      method: "PATCH",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ provider, model })
+    });
+  }
+  console.log(`
+Created self-improve task ${task2.id}${provider ? ` (provider: ${provider})` : ""}${model ? ` (model: ${model})` : ""}`);
+  console.log("Running agent...");
+  await runTask(apiUrl, task2.id);
+  const finished = await waitForTask(apiUrl, task2.id, timeoutMs * 5);
+  console.log(`Agent finished: ${finished.status}`);
+  if (finished.error) {
+    console.error(`Agent error: ${finished.error}`);
+  }
+  const afterVersion = await savePromptVersion(apiUrl, `loop-after-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`);
+  if (afterVersion) {
+    console.log(`Saved after-loop prompt version: ${afterVersion.name} (${afterVersion.id})`);
+  }
+  const improved = report.passed < context.report.passed + (context.report.results.some((r) => r.evaluation.passed) ? 0 : 0);
+  return { report, improved };
+}
+var loopCmd = new Command8("loop").description("Run a self-improve benchmark loop").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--iterations <n>", "max iterations", "3").option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for self-improve tasks").option("--model <name>", "model to use for self-improve tasks").action(async (opts) => {
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const project = await currentProject(apiUrl);
+  const timeoutMs = Number(opts.timeout);
+  const maxIterations = Number(opts.iterations);
+  let tasks;
+  let suiteName;
+  if (opts.suite === "deep-swe") {
+    if (!opts.path) throw new Error("--path is required for deep-swe");
+    tasks = await loadDeepSWESuite({
+      tasksDir: opts.path,
+      nTasks: opts.nTasks
+    });
+    suiteName = "deep-swe";
+  } else if (opts.suite === "synthetic") {
+    tasks = syntheticSuite();
+    if (opts.nTasks !== void 0 && opts.nTasks > 0) {
+      tasks = tasks.slice(0, opts.nTasks);
+    }
+    suiteName = "synthetic";
+  } else {
+    throw new Error(`Unknown suite: ${opts.suite}`);
+  }
+  if (tasks.length === 0) {
+    console.log("No benchmark tasks to run.");
+    return;
+  }
+  for (let i = 1; i <= maxIterations; i++) {
+    console.log(`
+=== Iteration ${String(i)}/${String(maxIterations)} ===`);
+    const { report } = await runLoopIteration(
+      apiUrl,
+      project.id,
+      tasks,
+      suiteName,
+      timeoutMs,
+      opts.outputDir,
+      opts.provider,
+      opts.model
+    );
+    const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+    if (passRate2 === 100) break;
+  }
 });
-var benchCmd = new Command8("bench").description("Run benchmarks and optimise prompts").addCommand(runCmd).addCommand(optimiseCmd);
+var benchCmd = new Command8("bench").description("Run benchmarks and optimise prompts").addCommand(runCmd).addCommand(abCmd).addCommand(optimiseCmd).addCommand(loopCmd);
 
 // ../../apps/cli/src/index.ts
 program2.name("harness").description("Omega harness CLI").version("0.1.0").option("--api <url>", "API base URL", "http://localhost:4000");