@castlemilk/omega 0.6.14 → 0.6.15

package/dist/cli.js

@@ -330,8 +330,8 @@ function getApiUrl() {
   const opts = program.opts();
   return opts.api ?? "http://localhost:4000";
 }
-async function apiFetch(path10, init) {
-  const url = `${getApiUrl()}${path10}`;
+async function apiFetch(path11, init) {
+  const url = `${getApiUrl()}${path11}`;
   const res = await fetch(url, init);
   const data = await res.json().catch(() => ({}));
   if (!res.ok) {
@@ -734,8 +734,8 @@ function getOutputDir() {
   }
   try {
     const __filename3 = fileURLToPath3(import.meta.url);
-    const __dirname3 = path3.dirname(__filename3);
-    return path3.join(path3.resolve(__dirname3, "../../../.."), "packages/skills/src");
+    const __dirname4 = path3.dirname(__filename3);
+    return path3.join(path3.resolve(__dirname4, "../../../.."), "packages/skills/src");
   } catch {
     return path3.join(process.cwd(), "harness-skills");
   }
@@ -819,7 +819,8 @@ agentCmd.command("diffs").description("Show diffs for a task").argument("<id>",
 
 // ../../apps/cli/src/commands/bench.ts
 import { Command as Command8 } from "commander";
-import path9 from "node:path";
+import fs8 from "node:fs/promises";
+import path10 from "node:path";
 
 // ../bench/dist/api-client.js
 var ApiError = class extends Error {
@@ -836,15 +837,15 @@ async function apiFetch2(url, init) {
   }
   return res.json();
 }
-async function ensureProject2(apiUrl, name, path10) {
+async function ensureProject2(apiUrl, name, path11) {
   const res = await fetch(`${apiUrl}/projects`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ name, path: path10 })
+    body: JSON.stringify({ name, path: path11 })
   });
   if (res.status === 409) {
-    const projects = await fetch(`${apiUrl}/projects?path=${encodeURIComponent(path10)}`).then((r) => r.json());
-    const existing = projects.find((p) => p.path === path10);
+    const projects = await fetch(`${apiUrl}/projects?path=${encodeURIComponent(path11)}`).then((r) => r.json());
+    const existing = projects.find((p) => p.path === path11);
     if (existing)
       return existing;
   }
@@ -1049,7 +1050,7 @@ async function writeReport(report, outputDir = ".omega/reports") {
   const jsonFile = path5.join(outputDir, `benchmark-${ts}.json`);
   const mdFile = path5.join(outputDir, `benchmark-${ts}.md`);
   await fs2.writeFile(jsonFile, JSON.stringify(report, null, 2), "utf-8");
-  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   const md = [
     "# Omega Benchmark Report",
     "",
@@ -1059,7 +1060,7 @@ async function writeReport(report, outputDir = ".omega/reports") {
     `- Passed: ${String(report.passed)}`,
     `- Failed: ${String(report.failed)}`,
     `- Timeouts: ${String(report.timeouts)}`,
-    `- Pass rate: ${String(passRate)}%`,
+    `- Pass rate: ${String(passRate2)}%`,
     `- Total duration: ${formatDuration(report.totalDurationMs)}`,
     "",
     "## Results",
@@ -1077,11 +1078,11 @@ async function writeReport(report, outputDir = ".omega/reports") {
   return jsonFile;
 }
 function printSummary(report) {
-  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   console.log(`
 Benchmark: ${report.suite}`);
   console.log(`Total: ${String(report.total)} | Passed: ${String(report.passed)} | Failed: ${String(report.failed)} | Timeouts: ${String(report.timeouts)}`);
-  console.log(`Pass rate: ${String(passRate)}%`);
+  console.log(`Pass rate: ${String(passRate2)}%`);
   console.log(`Duration: ${formatDuration(report.totalDurationMs)}`);
   for (const r of report.results) {
     console.log(resultLine(r, report.results.indexOf(r)));
@@ -1289,10 +1290,10 @@ async function readTask(taskDir) {
   const instructionRaw = await fs4.readFile(path7.join(taskDir, "instruction.md"), "utf-8");
   return { toml: parseToml(tomlRaw), instruction: instructionRaw };
 }
-async function cloneRepo(repoUrl, commit, targetPath) {
+async function cloneRepo(repoUrl, commit2, targetPath) {
   await fs4.mkdir(path7.dirname(targetPath), { recursive: true });
   await execFileAsync("git", ["clone", repoUrl, targetPath], { timeout: 12e4 });
-  await execFileAsync("git", ["-C", targetPath, "checkout", commit], { timeout: 6e4 });
+  await execFileAsync("git", ["-C", targetPath, "checkout", commit2], { timeout: 6e4 });
 }
 async function loadDeepSWESuite(options) {
   const entries = await fs4.readdir(options.tasksDir, { withFileTypes: true });
@@ -1321,7 +1322,7 @@ async function loadDeepSWESuite(options) {
     const id = toml.metadata?.task_id ?? path7.basename(dir);
     const title = toml.metadata?.display_title ?? toml.metadata?.original_title ?? id;
     const repo = toml.metadata?.repository_url;
-    const commit = toml.metadata?.base_commit_hash;
+    const commit2 = toml.metadata?.base_commit_hash;
     tasks.push({
       id: `deepswe-${id}`,
       name: id,
@@ -1329,10 +1330,10 @@ async function loadDeepSWESuite(options) {
       description: instruction,
       complexity: "complex",
       setup: async (projectPath) => {
-        if (!repo || !commit) {
+        if (!repo || !commit2) {
           throw new Error(`DeepSWE task ${id} is missing repository_url or base_commit_hash`);
         }
-        await cloneRepo(repo, commit, projectPath);
+        await cloneRepo(repo, commit2, projectPath);
       },
       evaluate: (ctx) => {
         const hasDiff = ctx.diffs.length > 0 && ctx.diffs.some((d) => d.patch.trim().length > 0);
@@ -1366,12 +1367,12 @@ function summariseTraceFlow(report, maxLines = 60) {
   return lines.slice(0, maxLines).join("\n") + "\n... (truncated)";
 }
 function buildOptimisePrompt(report, failedResult, traceFlowText) {
-  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   const failed = report.results.filter((r) => !r.evaluation.passed);
   const lines = [];
   lines.push("# Prompt optimisation task");
   lines.push("");
-  lines.push(`The latest benchmark run achieved ${String(passRate)}% pass rate (${String(report.passed)}/${String(report.total)}).`);
+  lines.push(`The latest benchmark run achieved ${String(passRate2)}% pass rate (${String(report.passed)}/${String(report.total)}).`);
   lines.push("");
   lines.push("## Failed tasks");
   for (const r of failed.slice(0, 5)) {
@@ -1432,6 +1433,335 @@ async function submitOptimiseTask(apiUrl, projectId, prompt) {
   return res.json();
 }
 
+// ../bench/dist/ab.js
+function passRate(report) {
+  return report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+}
+function findResult(report, taskId) {
+  return report.results.find((r) => r.task.id === taskId);
+}
+function buildAbReport(baseline, candidate, baselineReport, candidateReport) {
+  const baselineRate = passRate(baselineReport);
+  const candidateRate = passRate(candidateReport);
+  const taskIds = /* @__PURE__ */ new Set();
+  for (const r of baselineReport.results)
+    taskIds.add(r.task.id);
+  for (const r of candidateReport.results)
+    taskIds.add(r.task.id);
+  const perTask = Array.from(taskIds).map((taskId) => {
+    const base = findResult(baselineReport, taskId);
+    const cand = findResult(candidateReport, taskId);
+    return {
+      taskId,
+      taskName: base?.task.name ?? cand?.task.name ?? taskId,
+      baselinePassed: base?.evaluation.passed ?? false,
+      candidatePassed: cand?.evaluation.passed ?? false,
+      baselineTokens: base?.usage?.totalTokens,
+      candidateTokens: cand?.usage?.totalTokens
+    };
+  });
+  return {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    baseline,
+    candidate,
+    baselineReport,
+    candidateReport,
+    delta: {
+      passRate: candidateRate - baselineRate,
+      passedDelta: candidateReport.passed - baselineReport.passed,
+      durationMsDelta: candidateReport.totalDurationMs - baselineReport.totalDurationMs,
+      promptTokensDelta: (candidateReport.totalUsage?.promptTokens ?? 0) - (baselineReport.totalUsage?.promptTokens ?? 0),
+      completionTokensDelta: (candidateReport.totalUsage?.completionTokens ?? 0) - (baselineReport.totalUsage?.completionTokens ?? 0),
+      totalTokensDelta: (candidateReport.totalUsage?.totalTokens ?? 0) - (baselineReport.totalUsage?.totalTokens ?? 0)
+    },
+    perTask
+  };
+}
+async function runAbBenchmark(tasks, baseline, candidate, options) {
+  const suiteName = options.suiteName ?? "ab";
+  const baselineReport = await runBenchmark(tasks, {
+    ...options,
+    suiteName: `${suiteName}-baseline`
+  });
+  const candidateReport = await runBenchmark(tasks, {
+    ...options,
+    suiteName: `${suiteName}-candidate`
+  });
+  return buildAbReport(baseline, candidate, baselineReport, candidateReport);
+}
+function deltaSymbol(value) {
+  if (value > 0)
+    return "+";
+  if (value < 0)
+    return "";
+  return "";
+}
+function formatDelta(value) {
+  return `${deltaSymbol(value)}${String(value)}`;
+}
+function formatAbReport(report) {
+  const lines = [];
+  lines.push("# Omega A/B Benchmark Report");
+  lines.push("");
+  lines.push(`- Timestamp: ${report.timestamp}`);
+  lines.push(`- Baseline: ${report.baseline.name}`);
+  lines.push(`- Candidate: ${report.candidate.name}`);
+  lines.push("");
+  lines.push("## Summary");
+  lines.push(`| Metric | Baseline | Candidate | Delta |`);
+  lines.push(`|---|---|---|---|`);
+  lines.push(`| Pass rate | ${String(passRate(report.baselineReport))}% | ${String(passRate(report.candidateReport))}% | ${formatDelta(report.delta.passRate)}% |`);
+  lines.push(`| Passed | ${String(report.baselineReport.passed)}/${String(report.baselineReport.total)} | ${String(report.candidateReport.passed)}/${String(report.candidateReport.total)} | ${formatDelta(report.delta.passedDelta)} |`);
+  lines.push(`| Duration | ${formatDuration2(report.baselineReport.totalDurationMs)} | ${formatDuration2(report.candidateReport.totalDurationMs)} | ${formatDuration2(report.delta.durationMsDelta)} |`);
+  lines.push(`| Tokens | ${String(report.baselineReport.totalUsage?.totalTokens ?? 0)} | ${String(report.candidateReport.totalUsage?.totalTokens ?? 0)} | ${formatDelta(report.delta.totalTokensDelta)} |`);
+  lines.push("");
+  lines.push("## Per-task comparison");
+  lines.push(`| Task | Baseline | Candidate | Token delta |`);
+  lines.push(`|---|---|---|---|`);
+  for (const t of report.perTask) {
+    const tokenDelta = t.baselineTokens !== void 0 && t.candidateTokens !== void 0 ? formatDelta(t.candidateTokens - t.baselineTokens) : "-";
+    lines.push(`| ${t.taskName} | ${t.baselinePassed ? "pass" : "fail"} | ${t.candidatePassed ? "pass" : "fail"} | ${tokenDelta} |`);
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+function formatDuration2(ms) {
+  if (ms < 1e3)
+    return `${String(Math.round(ms))}ms`;
+  return `${(ms / 1e3).toFixed(1)}s`;
+}
+
+// ../agent/dist/tool-definitions.js
+var AGENT_TOOLS = [
+  {
+    name: "read_file",
+    description: "Read a file relative to project root.",
+    parameters: {
+      type: "object",
+      properties: { path: { type: "string" } },
+      required: ["path"]
+    }
+  },
+  {
+    name: "write_file",
+    description: "Write content to a file relative to project root.",
+    parameters: {
+      type: "object",
+      properties: { path: { type: "string" }, content: { type: "string" } },
+      required: ["path", "content"]
+    }
+  },
+  {
+    name: "edit_file",
+    description: "Apply a targeted edit to an existing file relative to project root. Replaces one occurrence of old_string with new_string.",
+    parameters: {
+      type: "object",
+      properties: {
+        path: { type: "string" },
+        old_string: { type: "string" },
+        new_string: { type: "string" }
+      },
+      required: ["path", "old_string", "new_string"]
+    }
+  },
+  {
+    name: "run_command",
+    description: "Run a shell command in the project root.",
+    parameters: {
+      type: "object",
+      properties: { command: { type: "string" } },
+      required: ["command"]
+    }
+  },
+  {
+    name: "think",
+    description: "Record reasoning.",
+    parameters: {
+      type: "object",
+      properties: { thought: { type: "string" } },
+      required: ["thought"]
+    }
+  },
+  {
+    name: "finish",
+    description: "Mark the task complete.",
+    parameters: {
+      type: "object",
+      properties: { summary: { type: "string" }, success: { type: "boolean" } },
+      required: ["summary", "success"]
+    }
+  },
+  {
+    name: "publish",
+    description: "Build, validate, and publish the project.",
+    parameters: {
+      type: "object",
+      properties: { version: { type: "string" } },
+      required: []
+    }
+  }
+];
+
+// ../agent/dist/planner.js
+var toolDescriptions = AGENT_TOOLS.map((t) => `- ${t.name}: ${t.description}`).join("\n");
+var PLAN_PROMPT = `You are a planning assistant. Given a task, produce a concise step-by-step plan.
+
+Respond with strict JSON in this exact shape (no markdown):
+{
+  "reasoning": "brief reasoning",
+  "plan": [
+    { "name": "step name", "tool": "optional tool name", "input": { optional tool args } }
+  ]
+}
+
+Available tools:
+${toolDescriptions}
+
+If a step does not need a tool, omit tool/input. Use edit_file for small file changes.`;
+
+// ../agent/dist/tools.js
+import { execFile as execFile2 } from "node:child_process";
+import { promisify as promisify2 } from "node:util";
+var execFileAsync2 = promisify2(execFile2);
+
+// ../agent/dist/validator.js
+import { execFile as execFile3 } from "node:child_process";
+import { promisify as promisify3 } from "node:util";
+var execFileAsync3 = promisify3(execFile3);
+
+// ../agent/dist/publisher.js
+import { execFile as execFile5 } from "node:child_process";
+import { promisify as promisify5 } from "node:util";
+
+// ../agent/dist/git.js
+import { execFile as execFile4 } from "node:child_process";
+import { promisify as promisify4 } from "node:util";
+var execFileAsync4 = promisify4(execFile4);
+
+// ../agent/dist/publisher.js
+var execFileAsync5 = promisify5(execFile5);
+
+// ../agent/dist/prompts.js
+import fs6 from "node:fs";
+function loadPromptFromEnv(name) {
+  const direct = process.env[name];
+  if (direct)
+    return direct;
+  const fileKey = `${name}_FILE`;
+  const filePath = process.env[fileKey];
+  if (filePath) {
+    try {
+      return fs6.readFileSync(filePath, "utf-8");
+    } catch {
+    }
+  }
+  return void 0;
+}
+var AGENT_SYSTEM_PROMPT = loadPromptFromEnv("OMEGA_SYSTEM_PROMPT") ?? `You are Omega, an autonomous software engineering agent running inside a project repository.
+
+Your job is to complete the user's task by calling tools. Do not write prose or explanations outside tool calls. Prefer targeted edits over full rewrites.
+
+Available tools:
+
+- read_file: Read a file relative to project root. Arguments: { "path": "relative/path" }
+- write_file: Overwrite or create a file. Arguments: { "path": "relative/path", "content": "full file content" }
+- edit_file: Replace one exact occurrence of old_string with new_string in an existing file. Use this for small changes. Arguments: { "path": "relative/path", "old_string": "...", "new_string": "..." }
+- run_command: Run a single simple command. No pipes (|), &&, ;, redirects, or globs. Prefer pnpm/npm/node. Arguments: { "command": "pnpm lint" }
+- think: Record a reasoning step. Arguments: { "thought": "..." }
+- finish: Mark the task complete. Arguments: { "summary": "what was done", "success": true }. Use summary, not message.
+- publish: Run build/test/publish validation. Only after your edits pass validation. Arguments: { "version": "optional" }
+
+Rules:
+1. Read the task, then use think to plan.
+2. Use edit_file for small changes; write_file only when creating a file or rewriting most of it.
+3. ALWAYS read a file with read_file before using edit_file on it. Do NOT read a file before write_file when creating a new file.
+4. Before creating JavaScript files, read package.json to check whether "type" is "module" and use ESM (export) or CommonJS (module.exports) accordingly.
+5. You MUST run validation (pnpm lint, pnpm test, or the project's validation command) after every edit using publish or run_command.
+6. Do NOT call finish until validation passes. If validation fails, fix the issue and re-run validation.
+7. Do not expose secrets or run destructive commands.
+8. Finish only when the task is done. Always include summary and success.
+9. Do not repeat the same tool call with the same arguments. If something fails, change your approach.
+10. Do not ask the user for clarification. Do not say "No task was provided". Use the Task and Description above to proceed.
+
+Example tool call sequence for "Add a greet function":
+1. think: { "thought": "I need to create greet.js and export a greet function." }
+2. write_file: { "path": "greet.js", "content": "export function greet(name) { return 'Hello, ' + name + '!'; }
+" }
+3. publish: { }
+4. finish: { "summary": "Created greet.js with greet(name) function and verified it passes tests.", "success": true }`;
+var TEXT_TOOLS_SYSTEM_PROMPT = loadPromptFromEnv("OMEGA_TEXT_TOOLS_PROMPT") ?? `You are Omega, an autonomous software engineering agent running inside a project repository.
+
+You MUST respond with a single JSON object containing a "tool_calls" array. Do not output markdown, explanations, or reasoning outside the JSON.
+
+Available tools (use ONLY these exact names):
+
+- read_file: { "path": "relative/path" }
+- write_file: { "path": "relative/path", "content": "full file content" }
+- edit_file: { "path": "relative/path", "old_string": "...", "new_string": "..." }
+- run_command: { "command": "simple command, no pipes/&&/;/redirects" }
+- think: { "thought": "reasoning text" }
+- finish: { "summary": "what was done", "success": true | false }
+- publish: { "version": "optional" }
+
+Rules:
+- Plan with think, then act.
+- Use edit_file for small changes; write_file only for new files or large rewrites.
+- ALWAYS read a file with read_file before using edit_file on it. Do NOT read a file before write_file when creating a new file.
+- Before creating JavaScript files, read package.json to check whether "type" is "module" and use ESM (export) or CommonJS (module.exports) accordingly.
+- You MUST run validation (pnpm lint, pnpm test, or the project's validation command) after every edit using publish or run_command.
+- Do NOT call finish until validation passes. If validation fails, fix the issue and re-run validation.
+- Do not expose secrets or run destructive commands.
+- Finish only when done. Use summary, not message.
+- Do not repeat the same tool call with the same arguments.
+- Do not ask the user for clarification. Do not say "No task was provided". Use the Task and Description above to proceed.
+
+Example response for "Add a greet function":
+{ "tool_calls": [
+  { "id": "1", "name": "think", "arguments": { "thought": "Create greet.js with greet(name)." } },
+  { "id": "2", "name": "write_file", "arguments": { "path": "greet.js", "content": "export function greet(name) { return 'Hello, ' + name + '!'; }
+" } },
+  { "id": "3", "name": "publish", "arguments": {} },
+  { "id": "4", "name": "finish", "arguments": { "summary": "Created greet.js and verified tests pass.", "success": true } }
+] }`;
+
+// ../agent/dist/prompt-versioning.js
+import fs7 from "node:fs/promises";
+import path9 from "node:path";
+import crypto from "node:crypto";
+import { fileURLToPath as fileURLToPath4 } from "node:url";
+var __dirname3 = path9.dirname(fileURLToPath4(import.meta.url));
+var PROMPTS_PATH = path9.resolve(__dirname3, "..", "src", "prompts.ts");
+var PLANNER_PATH = path9.resolve(__dirname3, "..", "src", "planner.ts");
+function hashString(value) {
+  return crypto.createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+function hashPrompts(input) {
+  return hashString([input.systemPrompt, input.textToolsPrompt, input.planningPrompt ?? ""].join("\n---\n"));
+}
+async function readPromptsSource() {
+  const source = await fs7.readFile(PROMPTS_PATH, "utf-8");
+  const systemMatch = /export const AGENT_SYSTEM_PROMPT =\s*(?:loadPromptFromEnv\('OMEGA_SYSTEM_PROMPT'\) \?\?\s*)?`([\s\S]*?)`;/m.exec(source);
+  const textToolsMatch = /export const TEXT_TOOLS_SYSTEM_PROMPT =\s*(?:loadPromptFromEnv\('OMEGA_TEXT_TOOLS_PROMPT'\) \?\?\s*)?`([\s\S]*?)`;/m.exec(source);
+  return {
+    systemPrompt: systemMatch?.[1] ?? "",
+    textToolsPrompt: textToolsMatch?.[1] ?? ""
+  };
+}
+async function loadCurrentPrompts() {
+  const { systemPrompt, textToolsPrompt } = await readPromptsSource();
+  const plannerSource = await fs7.readFile(PLANNER_PATH, "utf-8");
+  const planningMatch = /const PLAN_PROMPT = `([\s\S]*?)`;/m.exec(plannerSource);
+  const planningPrompt = planningMatch?.[1];
+  return {
+    name: `auto-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`,
+    sourcePath: PROMPTS_PATH,
+    systemPrompt,
+    textToolsPrompt,
+    planningPrompt,
+    hash: hashPrompts({ systemPrompt, textToolsPrompt, planningPrompt })
+  };
+}
+
 // ../../apps/cli/src/commands/bench.ts
 async function waitForApi2(apiUrl, maxMs = 1e4) {
   const deadline = Date.now() + maxMs;
@@ -1447,38 +1777,75 @@ async function waitForApi2(apiUrl, maxMs = 1e4) {
 }
 function currentProject(apiUrl) {
   const cwd = process.cwd();
-  const name = path9.basename(cwd);
+  const name = path10.basename(cwd);
   return ensureProject2(apiUrl, `bench-${name}`, cwd);
 }
-var runCmd = new Command8("run").description("Run a benchmark suite").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks (for deep-swe)", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for benchmark tasks").option("--model <model>", "model to use for benchmark tasks").action(async (opts) => {
-  const apiUrl = getApiUrl();
-  await waitForApi2(apiUrl);
-  const timeoutMs = Number(opts.timeout);
-  let tasks;
-  let suiteName;
+async function savePromptVersion(apiUrl, name) {
+  const current = await loadCurrentPrompts();
+  const versionRes = await fetch(`${apiUrl}/prompt-versions`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      name: name ?? current.name,
+      sourcePath: current.sourcePath,
+      systemPrompt: current.systemPrompt,
+      textToolsPrompt: current.textToolsPrompt,
+      hash: current.hash,
+      metadata: { planningPrompt: current.planningPrompt }
+    })
+  });
+  if (!versionRes.ok) {
+    console.warn("Could not save prompt version:", await versionRes.text());
+    return void 0;
+  }
+  return versionRes.json();
+}
+async function loadPromptSpec(source) {
+  if (source === "current") {
+    const current = await loadCurrentPrompts();
+    return {
+      name: "current",
+      systemPrompt: current.systemPrompt,
+      textToolsPrompt: current.textToolsPrompt
+    };
+  }
+  const raw = await fs8.readFile(path10.resolve(source), "utf-8");
+  const parsed = JSON.parse(raw);
+  if (!parsed.systemPrompt || !parsed.textToolsPrompt) {
+    throw new Error("Prompt spec must include systemPrompt and textToolsPrompt");
+  }
+  return parsed;
+}
+async function resolveTasks(opts) {
   if (opts.suite === "deep-swe") {
     if (!opts.path) {
       throw new Error("--path is required for the deep-swe suite");
     }
-    tasks = await loadDeepSWESuite({
+    const tasks = await loadDeepSWESuite({
       tasksDir: opts.path,
       nTasks: opts.nTasks,
       sampleSeed: opts.sampleSeed
     });
-    suiteName = "deep-swe";
-  } else if (opts.suite === "synthetic") {
-    tasks = syntheticSuite();
+    return { tasks, suiteName: "deep-swe" };
+  }
+  if (opts.suite === "synthetic") {
+    let tasks = syntheticSuite();
     if (opts.nTasks !== void 0 && opts.nTasks > 0) {
       tasks = tasks.slice(0, opts.nTasks);
     }
-    suiteName = "synthetic";
-  } else {
-    throw new Error(`Unknown suite: ${opts.suite}`);
+    return { tasks, suiteName: "synthetic" };
   }
+  throw new Error(`Unknown suite: ${opts.suite}`);
+}
+var runCmd = new Command8("run").description("Run a benchmark suite").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for benchmark tasks").option("--model <model>", "model to use for benchmark tasks").action(async (opts) => {
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const { tasks, suiteName } = await resolveTasks(opts);
   if (tasks.length === 0) {
     console.log("No benchmark tasks to run.");
     return;
   }
+  const timeoutMs = Number(opts.timeout);
   console.log(`Running ${String(tasks.length)} benchmark tasks against ${apiUrl}`);
   const report = await runBenchmark(tasks, {
     apiUrl,
@@ -1514,7 +1881,113 @@ var optimiseCmd = new Command8("optimise").description("Create a self-improve ta
   console.log(`Created self-improve task ${task.id}`);
   console.log(`Run \`omega task run ${task.id}\` to execute it.`);
 });
-var benchCmd = new Command8("bench").description("Run benchmarks and optimise prompts").addCommand(runCmd).addCommand(optimiseCmd);
+var abCmd = new Command8("ab").description("Run an A/B benchmark comparing two prompt versions").option("--baseline <source>", 'baseline prompts: "current" or path to JSON file', "current").option("--candidate <source>", "candidate prompts: path to JSON file").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for benchmark tasks").option("--model <model>", "model to use for benchmark tasks").action(async (opts) => {
+  if (!opts.candidate) {
+    throw new Error("--candidate is required");
+  }
+  if (opts.suite === "deep-swe") {
+    throw new Error("DeepSWE A/B not yet supported; use --suite synthetic");
+  }
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const { tasks, suiteName } = await resolveTasks(opts);
+  if (tasks.length === 0) {
+    console.log("No benchmark tasks to run.");
+    return;
+  }
+  const timeoutMs = Number(opts.timeout);
+  const baselineSpec = await loadPromptSpec(opts.baseline);
+  const candidateSpec = await loadPromptSpec(opts.candidate);
+  console.log(`Baseline: ${baselineSpec.name}`);
+  console.log(`Candidate: ${candidateSpec.name}`);
+  const report = await runAbBenchmark(tasks, baselineSpec, candidateSpec, {
+    apiUrl,
+    suiteName,
+    timeoutMs,
+    provider: opts.provider,
+    model: opts.model,
+    onProgress: (result) => {
+      const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
+      console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
+    }
+  });
+  await fs8.mkdir(opts.outputDir, { recursive: true });
+  const ts = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+  const jsonFile = path10.join(opts.outputDir, `ab-${ts}.json`);
+  const mdFile = path10.join(opts.outputDir, `ab-${ts}.md`);
+  await fs8.writeFile(jsonFile, JSON.stringify(report, null, 2), "utf-8");
+  await fs8.writeFile(mdFile, formatAbReport(report), "utf-8");
+  console.log("\n" + formatAbReport(report));
+  console.log(`
+A/B report written to ${jsonFile}`);
+});
+var loopCmd = new Command8("loop").description("Run a self-improve benchmark loop").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--iterations <n>", "max iterations", "1").option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for self-improve tasks").option("--model <model>", "model to use for self-improve tasks").action(async (opts) => {
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const { tasks, suiteName } = await resolveTasks(opts);
+  if (tasks.length === 0) {
+    console.log("No benchmark tasks to run.");
+    return;
+  }
+  const timeoutMs = Number(opts.timeout);
+  const maxIterations = Number(opts.iterations);
+  const project = await currentProject(apiUrl);
+  for (let i = 1; i <= maxIterations; i++) {
+    console.log(`
+=== Iteration ${String(i)}/${String(maxIterations)} ===`);
+    const beforeVersion = await savePromptVersion(apiUrl, `loop-before-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`);
+    if (beforeVersion) {
+      console.log(`Saved before-loop prompt version: ${beforeVersion.name} (${beforeVersion.id})`);
+    }
+    console.log(`
+Running ${String(tasks.length)} benchmark tasks...`);
+    const report = await runBenchmark(tasks, {
+      apiUrl,
+      suiteName,
+      timeoutMs,
+      provider: opts.provider,
+      model: opts.model,
+      onProgress: (result) => {
+        const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
+        console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
+      }
+    });
+    await writeReport(report, opts.outputDir);
+    printSummary(report);
+    const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+    if (passRate2 === 100) {
+      console.log("\nAll tasks passed. Loop complete.");
+      break;
+    }
+    const context = await loadOptimisationContext(apiUrl, opts.outputDir);
+    if (!context) {
+      throw new Error("Benchmark report could not be loaded for optimisation");
+    }
+    const prompt = buildOptimisePrompt(context.report, context.failedResult, context.traceFlowText);
+    const task = await submitOptimiseTask(apiUrl, project.id, prompt);
+    if (opts.provider || opts.model) {
+      await fetch(`${apiUrl}/tasks/${task.id}`, {
+        method: "PATCH",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ provider: opts.provider, model: opts.model })
+      });
+    }
+    console.log(`
+Created self-improve task ${task.id}${opts.provider ? ` (provider: ${opts.provider})` : ""}${opts.model ? ` (model: ${opts.model})` : ""}`);
+    console.log("Running agent...");
+    await runTask(apiUrl, task.id);
+    const finished = await waitForTask(apiUrl, task.id, timeoutMs * 5);
+    console.log(`Agent finished: ${finished.status}`);
+    if (finished.error) {
+      console.error(`Agent error: ${finished.error}`);
+    }
+    const afterVersion = await savePromptVersion(apiUrl, `loop-after-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`);
+    if (afterVersion) {
+      console.log(`Saved after-loop prompt version: ${afterVersion.name} (${afterVersion.id})`);
+    }
+  }
+});
+var benchCmd = new Command8("bench").description("Run benchmarks and optimise prompts").addCommand(runCmd).addCommand(abCmd).addCommand(optimiseCmd).addCommand(loopCmd);
 
 // ../../apps/cli/src/index.ts
 program2.name("harness").description("Omega harness CLI").version("0.1.0").option("--api <url>", "API base URL", "http://localhost:4000");

package/dist/server/pglite-data/postmaster.pid

@@ -1,6 +1,6 @@
 -42
 /pglite/data
-1783134189
+1783143868
 5432
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@castlemilk/omega",
-  "version": "0.6.14",
+  "version": "0.6.15",
   "description": "Omega Harness CLI - installable via npx",
   "type": "module",
   "main": "./dist/cli.js",