@castlemilk/omega 0.6.9 → 0.6.11

package/dist/cli.js

@@ -330,8 +330,8 @@ function getApiUrl() {
   const opts = program.opts();
   return opts.api ?? "http://localhost:4000";
 }
-async function apiFetch(path4, init) {
-  const url = `${getApiUrl()}${path4}`;
+async function apiFetch(path10, init) {
+  const url = `${getApiUrl()}${path10}`;
   const res = await fetch(url, init);
   const data = await res.json().catch(() => ({}));
   if (!res.ok) {
@@ -817,6 +817,693 @@ agentCmd.command("diffs").description("Show diffs for a task").argument("<id>",
   console.log(JSON.stringify(diffs, null, 2));
 });
 
+// ../../apps/cli/src/commands/bench.ts
+import { Command as Command8 } from "commander";
+import path9 from "node:path";
+
+// ../bench/dist/api-client.js
+var ApiError = class extends Error {
+  status;
+  constructor(status, message) {
+    super(message);
+    this.status = status;
+  }
+};
+async function apiFetch2(url, init) {
+  const res = await fetch(url, init);
+  if (!res.ok) {
+    throw new ApiError(res.status, `${init?.method ?? "GET"} ${url} -> ${String(res.status)}`);
+  }
+  return res.json();
+}
+async function ensureProject2(apiUrl, name, path10) {
+  const res = await fetch(`${apiUrl}/projects`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ name, path: path10 })
+  });
+  if (res.status === 409) {
+    const projects = await fetch(`${apiUrl}/projects?path=${encodeURIComponent(path10)}`).then((r) => r.json());
+    const existing = projects.find((p) => p.path === path10);
+    if (existing)
+      return existing;
+  }
+  if (!res.ok)
+    throw new ApiError(res.status, `create project failed: ${String(res.status)}`);
+  return res.json();
+}
+async function createTask(apiUrl, projectId, title, options = {}) {
+  return apiFetch2(`${apiUrl}/tasks`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      projectId,
+      title,
+      description: options.description,
+      complexity: options.complexity ?? "simple",
+      tags: options.tags ?? ["benchmark"]
+    })
+  });
+}
+async function runTask(apiUrl, taskId) {
+  await apiFetch2(`${apiUrl}/tasks/${taskId}/run`, { method: "POST" });
+}
+async function getTask(apiUrl, taskId) {
+  return apiFetch2(`${apiUrl}/tasks/${taskId}`);
+}
+async function waitForTask(apiUrl, taskId, timeoutMs = 12e4) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const task = await getTask(apiUrl, taskId);
+    if (task.status === "done" || task.status === "failed")
+      return task;
+    await new Promise((r) => setTimeout(r, 300));
+  }
+  return { status: "timeout", error: "Task did not finish in time" };
+}
+async function getAgentRun(apiUrl, taskId) {
+  try {
+    return await apiFetch2(`${apiUrl}/tasks/${taskId}/agent-run`);
+  } catch {
+    return void 0;
+  }
+}
+async function getDiffs(apiUrl, taskId) {
+  try {
+    return await apiFetch2(`${apiUrl}/tasks/${taskId}/diffs`);
+  } catch {
+    return [];
+  }
+}
+async function getTraceFlow(apiUrl, taskId) {
+  try {
+    return await apiFetch2(`${apiUrl}/tasks/${taskId}/trace-flow`);
+  } catch {
+    return void 0;
+  }
+}
+function countSpans(node) {
+  let count = 1;
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) {
+      count += countSpans(child);
+    }
+  }
+  return count;
+}
+
+// ../bench/dist/runner.js
+import fs from "node:fs/promises";
+import path4 from "node:path";
+import { execSync } from "node:child_process";
+function countAllSpans(traceFlow) {
+  if (!traceFlow)
+    return 0;
+  return traceFlow.spans.reduce((acc, span) => acc + countSpans(span), 0);
+}
+function ensureGitRepo(repoPath) {
+  try {
+    execSync("git rev-parse --git-dir", { cwd: repoPath, stdio: "ignore" });
+    return;
+  } catch {
+  }
+  execSync("git init", { cwd: repoPath, stdio: "ignore" });
+  execSync('git config user.email "bench@omega.local"', { cwd: repoPath, stdio: "ignore" });
+  execSync('git config user.name "Omega Bench"', { cwd: repoPath, stdio: "ignore" });
+  execSync("git add .", { cwd: repoPath, stdio: "ignore" });
+  execSync('git commit -m "bench init"', { cwd: repoPath, stdio: "ignore" });
+}
+async function runBenchmark(tasks, options) {
+  const { apiUrl, suiteName, timeoutMs = 12e4, projectPrefix = "bench" } = options;
+  const report = {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    suite: suiteName,
+    total: tasks.length,
+    passed: 0,
+    failed: 0,
+    timeouts: 0,
+    totalDurationMs: 0,
+    results: []
+  };
+  const baseDir = path4.join("/tmp", `omega-bench-${String(Date.now())}`);
+  await fs.mkdir(baseDir, { recursive: true });
+  for (const task of tasks) {
+    const start = Date.now();
+    let harnessTaskId = "";
+    let status = "failed";
+    let agentRun;
+    let diffs = [];
+    let traceFlow;
+    let evaluation = { passed: false, message: "Task did not complete" };
+    let projectId = "";
+    let projectPath = "";
+    try {
+      projectPath = path4.join(baseDir, task.id);
+      await fs.mkdir(projectPath, { recursive: true });
+      if (task.setup) {
+        await task.setup(projectPath);
+      }
+      ensureGitRepo(projectPath);
+      const project = await ensureProject2(apiUrl, `${projectPrefix}-${task.id}`, projectPath);
+      projectId = project.id;
+      const harnessTask = await createTask(apiUrl, project.id, task.title, {
+        description: task.description,
+        complexity: task.complexity ?? "simple",
+        tags: ["benchmark", "agent"]
+      });
+      harnessTaskId = harnessTask.id;
+      await runTask(apiUrl, harnessTask.id);
+      const finished = await waitForTask(apiUrl, harnessTask.id, timeoutMs);
+      status = finished.status === "timeout" ? "timeout" : finished.status;
+      [agentRun, diffs, traceFlow] = await Promise.all([
+        getAgentRun(apiUrl, harnessTask.id),
+        getDiffs(apiUrl, harnessTask.id),
+        getTraceFlow(apiUrl, harnessTask.id)
+      ]);
+      evaluation = await task.evaluate({
+        apiUrl,
+        taskId: harnessTask.id,
+        projectPath,
+        projectId,
+        agentRun,
+        diffs,
+        traceFlow
+      });
+    } catch (err) {
+      evaluation = {
+        passed: false,
+        message: err instanceof Error ? err.message : String(err)
+      };
+    }
+    const durationMs = Date.now() - start;
+    const result = {
+      task,
+      harnessTaskId,
+      durationMs,
+      status,
+      evaluation,
+      agentRun,
+      spanCount: countAllSpans(traceFlow)
+    };
+    if (status === "timeout")
+      report.timeouts++;
+    else if (evaluation.passed)
+      report.passed++;
+    else
+      report.failed++;
+    report.totalDurationMs += durationMs;
+    report.results.push(result);
+    options.onProgress?.(result);
+  }
+  return report;
+}
+
+// ../bench/dist/report.js
+import fs2 from "node:fs/promises";
+import path5 from "node:path";
+function nowIso() {
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+}
+function formatDuration(ms) {
+  if (ms < 1e3)
+    return `${String(Math.round(ms))}ms`;
+  return `${(ms / 1e3).toFixed(1)}s`;
+}
+function resultLine(r, idx) {
+  const symbol = r.evaluation.passed ? "\u2713" : "\u2717";
+  const status = r.status === "timeout" ? "timeout" : r.status;
+  const score = r.evaluation.score !== void 0 ? ` (score ${String(r.evaluation.score)})` : "";
+  const msg = r.evaluation.message ? ` \u2014 ${r.evaluation.message}` : "";
+  return `${String(idx + 1)}. ${symbol} ${r.task.name} [${status}] ${formatDuration(r.durationMs)}${score}${msg}`;
+}
+async function writeReport(report, outputDir = ".omega/reports") {
+  await fs2.mkdir(outputDir, { recursive: true });
+  const ts = nowIso();
+  const jsonFile = path5.join(outputDir, `benchmark-${ts}.json`);
+  const mdFile = path5.join(outputDir, `benchmark-${ts}.md`);
+  await fs2.writeFile(jsonFile, JSON.stringify(report, null, 2), "utf-8");
+  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const md = [
+    "# Omega Benchmark Report",
+    "",
+    `- Suite: ${report.suite}`,
+    `- Timestamp: ${report.timestamp}`,
+    `- Total: ${String(report.total)}`,
+    `- Passed: ${String(report.passed)}`,
+    `- Failed: ${String(report.failed)}`,
+    `- Timeouts: ${String(report.timeouts)}`,
+    `- Pass rate: ${String(passRate)}%`,
+    `- Total duration: ${formatDuration(report.totalDurationMs)}`,
+    "",
+    "## Results",
+    "",
+    ...report.results.map((r, i) => resultLine(r, i)),
+    "",
+    "## Details",
+    "",
+    "```json",
+    JSON.stringify(report, null, 2),
+    "```",
+    ""
+  ].join("\n");
+  await fs2.writeFile(mdFile, md, "utf-8");
+  return jsonFile;
+}
+function printSummary(report) {
+  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  console.log(`
+Benchmark: ${report.suite}`);
+  console.log(`Total: ${String(report.total)} | Passed: ${String(report.passed)} | Failed: ${String(report.failed)} | Timeouts: ${String(report.timeouts)}`);
+  console.log(`Pass rate: ${String(passRate)}%`);
+  console.log(`Duration: ${formatDuration(report.totalDurationMs)}`);
+  for (const r of report.results) {
+    console.log(resultLine(r, report.results.indexOf(r)));
+  }
+}
+
+// ../bench/dist/suites/synthetic.js
+import fs3 from "node:fs/promises";
+import path6 from "node:path";
+async function writeJson(projectPath, file, data) {
+  await fs3.writeFile(path6.join(projectPath, file), JSON.stringify(data, null, 2), "utf-8");
+}
+async function writeFile2(projectPath, file, content) {
+  await fs3.mkdir(path6.dirname(path6.join(projectPath, file)), { recursive: true });
+  await fs3.writeFile(path6.join(projectPath, file), content, "utf-8");
+}
+function parseValidation(ctx) {
+  if (!ctx.agentRun?.validationSummary)
+    return { allPassed: false };
+  try {
+    const summary = JSON.parse(ctx.agentRun.validationSummary);
+    return { allPassed: Boolean(summary.allPassed), summary };
+  } catch {
+    return { allPassed: false };
+  }
+}
+function validate(ctx) {
+  const v = parseValidation(ctx);
+  return {
+    passed: v.allPassed,
+    message: v.allPassed ? "Validation passed" : "Validation failed"
+  };
+}
+function basePackage() {
+  return {
+    name: "synthetic-bench-project",
+    version: "1.0.0",
+    type: "module",
+    scripts: {
+      lint: "node lint.js",
+      test: "node test.js",
+      build: "node build.js"
+    }
+  };
+}
+var TASKS = [
+  {
+    id: "greet-function",
+    name: "Add a greet function",
+    title: "Add greet.js with a greet function",
+    description: "Create a `greet.js` file that exports a function `greet(name)` returning `Hello, <name>!`. The existing test expects this function.",
+    complexity: "simple",
+    setup: async (projectPath) => {
+      await writeJson(projectPath, "package.json", basePackage());
+      await writeFile2(projectPath, "test.js", `import { greet } from './greet.js';
+import assert from 'node:assert';
+assert.strictEqual(greet('World'), 'Hello, World!');
+console.log('greet test passed');
+`);
+      await writeFile2(projectPath, "lint.js", `console.log('lint ok');
+`);
+      await writeFile2(projectPath, "build.js", `console.log('build ok');
+`);
+    },
+    evaluate: (ctx) => Promise.resolve(validate(ctx))
+  },
+  {
+    id: "fix-lint-warning",
+    name: "Fix a lint warning",
+    title: "Remove unused variable flagged by linter",
+    description: "The custom linter flags any use of `var`. Update `src/index.js` so the lint script passes while keeping the greeting behavior.",
+    complexity: "simple",
+    setup: async (projectPath) => {
+      await writeJson(projectPath, "package.json", basePackage());
+      await writeFile2(projectPath, "src/index.js", `var greeting = 'Hello';
+export function hi(name) { return greeting + ', ' + name + '!'; }
+`);
+      await writeFile2(projectPath, "test.js", `import { hi } from './src/index.js';
+import assert from 'node:assert';
+assert.strictEqual(hi('Omega'), 'Hello, Omega!');
+console.log('hi test passed');
+`);
+      await writeFile2(projectPath, "lint.js", `import fs from 'node:fs/promises';
+const code = await fs.readFile('./src/index.js', 'utf-8');
+if (code.includes('var ')) { console.error('lint failed: var keyword found'); process.exit(1); }
+console.log('lint ok');
+`);
+      await writeFile2(projectPath, "build.js", `console.log('build ok');
+`);
+    },
+    evaluate: (ctx) => Promise.resolve(validate(ctx))
+  },
+  {
+    id: "update-config",
+    name: "Update a config value",
+    title: "Update timeout value in config.json",
+    description: "Change `config.json` so that `timeout` is `5000` instead of `1000`. The test will read the file and assert the value.",
+    complexity: "simple",
+    setup: async (projectPath) => {
+      await writeJson(projectPath, "package.json", basePackage());
+      await writeJson(projectPath, "config.json", { timeout: 1e3, retries: 3 });
+      await writeFile2(projectPath, "test.js", `import { readFile } from 'node:fs/promises';
+const cfg = JSON.parse(await readFile('./config.json', 'utf-8'));
+if (cfg.timeout !== 5000) { console.error('timeout is ' + cfg.timeout); process.exit(1); }
+console.log('config test passed');
+`);
+      await writeFile2(projectPath, "lint.js", `console.log('lint ok');
+`);
+      await writeFile2(projectPath, "build.js", `console.log('build ok');
+`);
+    },
+    evaluate: (ctx) => Promise.resolve(validate(ctx))
+  },
+  {
+    id: "extract-constant",
+    name: "Extract a magic constant",
+    title: "Extract MAX_RETRIES constant",
+    description: "Refactor `src/retry.js` so the magic number `3` is replaced by a top-level constant named `MAX_RETRIES`. The test checks for the constant.",
+    complexity: "medium",
+    setup: async (projectPath) => {
+      await writeJson(projectPath, "package.json", basePackage());
+      await writeFile2(projectPath, "src/retry.js", `export function attempts() { return 3; }
+`);
+      await writeFile2(projectPath, "test.js", `import fs from 'node:fs/promises';
+import { attempts } from './src/retry.js';
+const code = await fs.readFile('./src/retry.js', 'utf-8');
+if (!code.includes('const MAX_RETRIES')) { console.error('MAX_RETRIES constant not found'); process.exit(1); }
+if (attempts() !== 3) { console.error('attempts() changed'); process.exit(1); }
+console.log('constant test passed');
+`);
+      await writeFile2(projectPath, "lint.js", `console.log('lint ok');
+`);
+      await writeFile2(projectPath, "build.js", `console.log('build ok');
+`);
+    },
+    evaluate: (ctx) => Promise.resolve(validate(ctx))
+  },
+  {
+    id: "noop-validation",
+    name: "Run validation on a clean project",
+    title: "Confirm the project already passes validation",
+    description: "This project already passes lint, test and build. Use the publish tool to confirm validation passes and finish.",
+    complexity: "simple",
+    setup: async (projectPath) => {
+      await writeJson(projectPath, "package.json", basePackage());
+      await writeFile2(projectPath, "test.js", `console.log('test ok');
+`);
+      await writeFile2(projectPath, "lint.js", `console.log('lint ok');
+`);
+      await writeFile2(projectPath, "build.js", `console.log('build ok');
+`);
+    },
+    evaluate: (ctx) => Promise.resolve(validate(ctx))
+  }
+];
+function syntheticSuite() {
+  return TASKS;
+}
+
+// ../bench/dist/adapters/deepswe.js
+import fs4 from "node:fs/promises";
+import path7 from "node:path";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+var execFileAsync = promisify(execFile);
+function parseToml(raw) {
+  const result = { task: {}, metadata: {} };
+  let section;
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith("[")) {
+      const name = trimmed.slice(1, -1).split(".")[0];
+      if (name === "task" || name === "metadata")
+        section = name;
+      continue;
+    }
+    if (!section || trimmed.startsWith("#") || !trimmed.includes("="))
+      continue;
+    const eq = trimmed.indexOf("=");
+    const key = trimmed.slice(0, eq).trim();
+    let value = trimmed.slice(eq + 1).trim();
+    if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
+      value = value.slice(1, -1);
+    }
+    if (section === "task") {
+      result.task ??= {};
+      result.task[key] = value;
+    } else {
+      result.metadata ??= {};
+      result.metadata[key] = value;
+    }
+  }
+  return result;
+}
+function mulberry32(seed) {
+  return () => {
+    let t = seed += 1831565813;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+async function readTask(taskDir) {
+  const tomlRaw = await fs4.readFile(path7.join(taskDir, "task.toml"), "utf-8");
+  const instructionRaw = await fs4.readFile(path7.join(taskDir, "instruction.md"), "utf-8");
+  return { toml: parseToml(tomlRaw), instruction: instructionRaw };
+}
+async function cloneRepo(repoUrl, commit, targetPath) {
+  await fs4.mkdir(path7.dirname(targetPath), { recursive: true });
+  await execFileAsync("git", ["clone", repoUrl, targetPath], { timeout: 12e4 });
+  await execFileAsync("git", ["-C", targetPath, "checkout", commit], { timeout: 6e4 });
+}
+async function loadDeepSWESuite(options) {
+  const entries = await fs4.readdir(options.tasksDir, { withFileTypes: true });
+  const taskDirs = entries.filter((e) => e.isDirectory()).map((e) => path7.join(options.tasksDir, e.name));
+  let loaded = [];
+  for (const dir of taskDirs) {
+    try {
+      const data = await readTask(dir);
+      loaded.push({ dir, ...data });
+    } catch {
+    }
+  }
+  if (options.taskIds && options.taskIds.length > 0) {
+    loaded = loaded.filter((t) => {
+      const id = t.toml.metadata?.task_id ?? path7.basename(t.dir);
+      return options.taskIds?.includes(id);
+    });
+  }
+  if (options.nTasks && options.nTasks > 0 && options.nTasks < loaded.length) {
+    const seed = options.sampleSeed ?? 0;
+    const rnd = mulberry32(seed);
+    loaded = loaded.map((t) => ({ t, sort: rnd() })).sort((a, b) => a.sort - b.sort).slice(0, options.nTasks).map((x) => x.t);
+  }
+  const tasks = [];
+  for (const { dir, toml, instruction } of loaded) {
+    const id = toml.metadata?.task_id ?? path7.basename(dir);
+    const title = toml.metadata?.display_title ?? toml.metadata?.original_title ?? id;
+    const repo = toml.metadata?.repository_url;
+    const commit = toml.metadata?.base_commit_hash;
+    tasks.push({
+      id: `deepswe-${id}`,
+      name: id,
+      title,
+      description: instruction,
+      complexity: "complex",
+      setup: async (projectPath) => {
+        if (!repo || !commit) {
+          throw new Error(`DeepSWE task ${id} is missing repository_url or base_commit_hash`);
+        }
+        await cloneRepo(repo, commit, projectPath);
+      },
+      evaluate: (ctx) => {
+        const hasDiff = ctx.diffs.length > 0 && ctx.diffs.some((d) => d.patch.trim().length > 0);
+        const done = ctx.agentRun?.resultStatus === "done";
+        const passed = done && hasDiff;
+        return Promise.resolve({
+          passed,
+          message: passed ? `Agent finished and produced a diff (${String(ctx.diffs[0]?.patch.length ?? 0)} chars)` : `Agent status=${ctx.agentRun?.resultStatus ?? "unknown"}, diff=${hasDiff ? "yes" : "no"}`
+        });
+      }
+    });
+  }
+  return tasks;
+}
+
+// ../bench/dist/optimise.js
+import fs5 from "node:fs/promises";
+import path8 from "node:path";
+async function findLatestReport(outputDir) {
+  const entries = await fs5.readdir(outputDir).catch(() => []);
+  const files = entries.filter((f) => f.startsWith("benchmark-") && f.endsWith(".json")).sort().reverse();
+  if (files.length === 0)
+    return void 0;
+  const raw = await fs5.readFile(path8.join(outputDir, files[0]), "utf-8");
+  return JSON.parse(raw);
+}
+function summariseTraceFlow(report, maxLines = 60) {
+  const lines = report.split("\n");
+  if (lines.length <= maxLines)
+    return report;
+  return lines.slice(0, maxLines).join("\n") + "\n... (truncated)";
+}
+function buildOptimisePrompt(report, failedResult, traceFlowText) {
+  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const failed = report.results.filter((r) => !r.evaluation.passed);
+  const lines = [];
+  lines.push("# Prompt optimisation task");
+  lines.push("");
+  lines.push(`The latest benchmark run achieved ${String(passRate)}% pass rate (${String(report.passed)}/${String(report.total)}).`);
+  lines.push("");
+  lines.push("## Failed tasks");
+  for (const r of failed.slice(0, 5)) {
+    lines.push(`- ${r.task.name}: ${r.evaluation.message ?? "no message"}`);
+  }
+  lines.push("");
+  if (failedResult) {
+    lines.push(`## Focus task: ${failedResult.task.name}`);
+    lines.push(`Status: ${failedResult.status}`);
+    lines.push(`Evaluation: ${failedResult.evaluation.message ?? "none"}`);
+    lines.push("");
+  }
+  if (traceFlowText) {
+    lines.push("## Trace flow snapshot");
+    lines.push("```json");
+    lines.push(summariseTraceFlow(traceFlowText));
+    lines.push("```");
+    lines.push("");
+  }
+  lines.push("## Instructions");
+  lines.push("Analyse the failures above and edit `packages/agent/src/prompts.ts` (and optionally `packages/agent/src/planner.ts`) to improve the Omega agent.");
+  lines.push("Rules:");
+  lines.push("- Keep changes minimal and targeted.");
+  lines.push("- Do not break existing tool signatures or system behaviour.");
+  lines.push("- Run `pnpm lint` and `pnpm test` after edits.");
+  lines.push("- Finish with a summary of what changed and why it should help.");
+  return lines.join("\n");
+}
+async function loadOptimisationContext(apiUrl, outputDir) {
+  const report = await findLatestReport(outputDir);
+  if (!report)
+    return void 0;
+  const failedResult = report.results.find((r) => !r.evaluation.passed);
+  let traceFlowText;
+  if (failedResult?.harnessTaskId) {
+    const traceFlow = await getTraceFlow(apiUrl, failedResult.harnessTaskId);
+    if (traceFlow) {
+      traceFlowText = JSON.stringify(traceFlow, null, 2);
+    }
+  }
+  return { report, failedResult, traceFlowText };
+}
+async function submitOptimiseTask(apiUrl, projectId, prompt) {
+  const title = "Optimise agent prompts from benchmark trace analysis";
+  const res = await fetch(`${apiUrl}/tasks`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      projectId,
+      title,
+      description: prompt,
+      complexity: "complex",
+      tags: ["self-improve", "benchmark"]
+    })
+  });
+  if (!res.ok)
+    throw new Error(`submit optimise task failed: ${String(res.status)}`);
+  return res.json();
+}
+
+// ../../apps/cli/src/commands/bench.ts
+async function waitForApi2(apiUrl, maxMs = 1e4) {
+  const deadline = Date.now() + maxMs;
+  while (Date.now() < deadline) {
+    try {
+      const res = await fetch(`${apiUrl}/projects`);
+      if (res.ok) return;
+    } catch {
+    }
+    await new Promise((r) => setTimeout(r, 300));
+  }
+  throw new Error(`Harness API is not ready at ${apiUrl}. Run \`omega ui\` or \`omega console\` first.`);
+}
+function currentProject(apiUrl) {
+  const cwd = process.cwd();
+  const name = path9.basename(cwd);
+  return ensureProject2(apiUrl, `bench-${name}`, cwd);
+}
+var runCmd = new Command8("run").description("Run a benchmark suite").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks (for deep-swe)", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").action(async (opts) => {
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const timeoutMs = Number(opts.timeout);
+  let tasks;
+  let suiteName;
+  if (opts.suite === "deep-swe") {
+    if (!opts.path) {
+      throw new Error("--path is required for the deep-swe suite");
+    }
+    tasks = await loadDeepSWESuite({
+      tasksDir: opts.path,
+      nTasks: opts.nTasks,
+      sampleSeed: opts.sampleSeed
+    });
+    suiteName = "deep-swe";
+  } else if (opts.suite === "synthetic") {
+    tasks = syntheticSuite();
+    suiteName = "synthetic";
+  } else {
+    throw new Error(`Unknown suite: ${opts.suite}`);
+  }
+  if (tasks.length === 0) {
+    console.log("No benchmark tasks to run.");
+    return;
+  }
+  console.log(`Running ${String(tasks.length)} benchmark tasks against ${apiUrl}`);
+  const report = await runBenchmark(tasks, {
+    apiUrl,
+    suiteName,
+    timeoutMs,
+    onProgress: (result) => {
+      const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
+      console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
+    }
+  });
+  const reportFile = await writeReport(report, opts.outputDir);
+  printSummary(report);
+  console.log(`
+Report written to ${reportFile}`);
+});
+var optimiseCmd = new Command8("optimise").description("Create a self-improve task from the latest benchmark report").option("--output-dir <dir>", "report output directory", ".omega/reports").action(async (opts) => {
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const context = await loadOptimisationContext(apiUrl, opts.outputDir);
+  if (!context) {
+    console.log("No benchmark report found. Run `omega bench run` first.");
+    return;
+  }
+  const project = await currentProject(apiUrl);
+  const prompt = buildOptimisePrompt(
+    context.report,
+    context.failedResult,
+    context.traceFlowText
+  );
+  const task = await submitOptimiseTask(apiUrl, project.id, prompt);
+  console.log(`Created self-improve task ${task.id}`);
+  console.log(`Run \`omega task run ${task.id}\` to execute it.`);
+});
+var benchCmd = new Command8("bench").description("Run benchmarks and optimise prompts").addCommand(runCmd).addCommand(optimiseCmd);
+
 // ../../apps/cli/src/index.ts
 program2.name("harness").description("Omega harness CLI").version("0.1.0").option("--api <url>", "API base URL", "http://localhost:4000");
 program2.addCommand(projectCmd);
@@ -825,4 +1512,5 @@ program2.addCommand(uiCmd);
 program2.addCommand(consoleCmd);
 program2.addCommand(skillCmd);
 program2.addCommand(agentCmd);
+program2.addCommand(benchCmd);
 program2.parse();