@castlemilk/omega 0.6.19 → 0.6.21

package/dist/cli.js

@@ -320,7 +320,6 @@ var init_tui = __esm({
 
 // ../../apps/cli/src/index.ts
 import { program as program2 } from "commander";
-import { createRequire } from "node:module";
 
 // ../../apps/cli/src/commands/project.ts
 import { Command } from "commander";
@@ -331,8 +330,8 @@ function getApiUrl() {
   const opts = program.opts();
   return opts.api ?? "http://localhost:4000";
 }
-async function apiFetch(path11, init) {
-  const url = `${getApiUrl()}${path11}`;
+async function apiFetch(path10, init) {
+  const url = `${getApiUrl()}${path10}`;
   const res = await fetch(url, init);
   const data = await res.json().catch(() => ({}));
   if (!res.ok) {
@@ -735,8 +734,8 @@ function getOutputDir() {
   }
   try {
     const __filename3 = fileURLToPath3(import.meta.url);
-    const __dirname4 = path3.dirname(__filename3);
-    return path3.join(path3.resolve(__dirname4, "../../../.."), "packages/skills/src");
+    const __dirname3 = path3.dirname(__filename3);
+    return path3.join(path3.resolve(__dirname3, "../../../.."), "packages/skills/src");
   } catch {
     return path3.join(process.cwd(), "harness-skills");
   }
@@ -820,8 +819,7 @@ agentCmd.command("diffs").description("Show diffs for a task").argument("<id>",
 
 // ../../apps/cli/src/commands/bench.ts
 import { Command as Command8 } from "commander";
-import fs8 from "node:fs/promises";
-import path10 from "node:path";
+import path9 from "node:path";
 
 // ../bench/dist/api-client.js
 var ApiError = class extends Error {
@@ -838,15 +836,15 @@ async function apiFetch2(url, init) {
   }
   return res.json();
 }
-async function ensureProject2(apiUrl, name, path11) {
+async function ensureProject2(apiUrl, name, path10) {
   const res = await fetch(`${apiUrl}/projects`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ name, path: path11 })
+    body: JSON.stringify({ name, path: path10 })
   });
   if (res.status === 409) {
-    const projects = await fetch(`${apiUrl}/projects?path=${encodeURIComponent(path11)}`).then((r) => r.json());
-    const existing = projects.find((p) => p.path === path11);
+    const projects = await fetch(`${apiUrl}/projects?path=${encodeURIComponent(path10)}`).then((r) => r.json());
+    const existing = projects.find((p) => p.path === path10);
     if (existing)
       return existing;
   }
@@ -904,6 +902,13 @@ async function getTraceFlow(apiUrl, taskId) {
     return void 0;
   }
 }
+async function getPromptVersion(apiUrl, id) {
+  try {
+    return await apiFetch2(`${apiUrl}/prompt-versions/${id}`);
+  } catch {
+    return void 0;
+  }
+}
 function countSpans(node) {
   let count = 1;
   if (Array.isArray(node.children)) {
@@ -936,7 +941,7 @@ function ensureGitRepo(repoPath) {
   execSync('git commit -m "bench init"', { cwd: repoPath, stdio: "ignore" });
 }
 async function runBenchmark(tasks, options) {
-  const { apiUrl, suiteName, timeoutMs = 12e4, projectPrefix = "bench", provider, model } = options;
+  const { apiUrl, suiteName, timeoutMs = 12e4, projectPrefix = "bench" } = options;
   const report = {
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     suite: suiteName,
@@ -959,6 +964,7 @@ async function runBenchmark(tasks, options) {
     let evaluation = { passed: false, message: "Task did not complete" };
     let projectId = "";
     let projectPath = "";
+    let promptVersion = void 0;
     try {
       projectPath = path4.join(baseDir, task.id);
       await fs.mkdir(projectPath, { recursive: true });
@@ -974,21 +980,24 @@ async function runBenchmark(tasks, options) {
         tags: ["benchmark", "agent"]
       });
       harnessTaskId = harnessTask.id;
-      if (provider || model) {
+      if (options.provider || options.model) {
         await fetch(`${apiUrl}/tasks/${harnessTaskId}`, {
           method: "PATCH",
           headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({ provider, model })
+          body: JSON.stringify({ provider: options.provider, model: options.model })
         });
       }
       await runTask(apiUrl, harnessTask.id);
       const finished = await waitForTask(apiUrl, harnessTask.id, timeoutMs);
-      const harnessStatus = finished.status === "timeout" ? "timeout" : finished.status;
+      status = finished.status === "timeout" ? "timeout" : finished.status;
       [agentRun, diffs, traceFlow] = await Promise.all([
         getAgentRun(apiUrl, harnessTask.id),
         getDiffs(apiUrl, harnessTask.id),
         getTraceFlow(apiUrl, harnessTask.id)
       ]);
+      if (agentRun?.promptVersionId) {
+        promptVersion = await getPromptVersion(apiUrl, agentRun.promptVersionId);
+      }
       evaluation = await task.evaluate({
         apiUrl,
         taskId: harnessTask.id,
@@ -998,13 +1007,11 @@ async function runBenchmark(tasks, options) {
         diffs,
         traceFlow
       });
-      status = harnessStatus === "timeout" ? "timeout" : evaluation.passed ? "done" : "failed";
     } catch (err) {
       evaluation = {
         passed: false,
         message: err instanceof Error ? err.message : String(err)
       };
-      status = "failed";
     }
     const durationMs = Date.now() - start;
     const result = {
@@ -1014,7 +1021,9 @@ async function runBenchmark(tasks, options) {
       status,
       evaluation,
       agentRun,
-      spanCount: countAllSpans(traceFlow)
+      spanCount: countAllSpans(traceFlow),
+      promptVersionId: agentRun?.promptVersionId,
+      promptHash: promptVersion?.hash
     };
     if (status === "timeout")
       report.timeouts++;
@@ -1026,6 +1035,11 @@ async function runBenchmark(tasks, options) {
     report.results.push(result);
     options.onProgress?.(result);
   }
+  if (report.results.length > 0) {
+    const latestResult = report.results[report.results.length - 1];
+    report.promptVersionId = latestResult.promptVersionId;
+    report.promptHash = latestResult.promptHash;
+  }
   return report;
 }
 
@@ -1053,7 +1067,7 @@ async function writeReport(report, outputDir = ".omega/reports") {
   const jsonFile = path5.join(outputDir, `benchmark-${ts}.json`);
   const mdFile = path5.join(outputDir, `benchmark-${ts}.md`);
   await fs2.writeFile(jsonFile, JSON.stringify(report, null, 2), "utf-8");
-  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   const md = [
     "# Omega Benchmark Report",
     "",
@@ -1063,7 +1077,7 @@ async function writeReport(report, outputDir = ".omega/reports") {
     `- Passed: ${String(report.passed)}`,
     `- Failed: ${String(report.failed)}`,
     `- Timeouts: ${String(report.timeouts)}`,
-    `- Pass rate: ${String(passRate2)}%`,
+    `- Pass rate: ${String(passRate)}%`,
     `- Total duration: ${formatDuration(report.totalDurationMs)}`,
     "",
     "## Results",
@@ -1081,11 +1095,11 @@ async function writeReport(report, outputDir = ".omega/reports") {
   return jsonFile;
 }
 function printSummary(report) {
-  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   console.log(`
 Benchmark: ${report.suite}`);
   console.log(`Total: ${String(report.total)} | Passed: ${String(report.passed)} | Failed: ${String(report.failed)} | Timeouts: ${String(report.timeouts)}`);
-  console.log(`Pass rate: ${String(passRate2)}%`);
+  console.log(`Pass rate: ${String(passRate)}%`);
   console.log(`Duration: ${formatDuration(report.totalDurationMs)}`);
   for (const r of report.results) {
     console.log(resultLine(r, report.results.indexOf(r)));
@@ -1131,28 +1145,6 @@ function basePackage() {
     }
   };
 }
-function simpleTask(id, name, title, description, files, complexity = "simple") {
-  return {
-    id,
-    name,
-    title,
-    description,
-    complexity,
-    setup: async (projectPath) => {
-      await writeJson(projectPath, "package.json", basePackage());
-      for (const [file, content] of Object.entries(files)) {
-        await writeFile2(projectPath, file, content);
-      }
-      if (!files["lint.js"])
-        await writeFile2(projectPath, "lint.js", `console.log('lint ok');
-`);
-      if (!files["build.js"])
-        await writeFile2(projectPath, "build.js", `console.log('build ok');
-`);
-    },
-    evaluate: (ctx) => Promise.resolve(validate(ctx))
-  };
-}
 var TASKS = [
   {
     id: "greet-function",
@@ -1245,259 +1237,6 @@ console.log('constant test passed');
     },
     evaluate: (ctx) => Promise.resolve(validate(ctx))
   },
-  simpleTask("sum-function", "Add a sum function", "Add math.js with a sum function", "Create math.js that exports sum(a, b) returning the numeric sum of the two arguments.", {
-    "test.js": `import { sum } from './math.js';
-import assert from 'node:assert';
-assert.strictEqual(sum(2, 3), 5);
-assert.strictEqual(sum(-1, 1), 0);
-console.log('sum test passed');
-`
-  }),
-  simpleTask("reverse-string", "Add reverse string function", "Add string.js with reverse function", "Create string.js that exports reverse(str) returning the input string reversed.", {
-    "test.js": `import { reverse } from './string.js';
-import assert from 'node:assert';
-assert.strictEqual(reverse('abc'), 'cba');
-assert.strictEqual(reverse('Hello'), 'olleH');
-console.log('reverse test passed');
-`
-  }),
-  simpleTask("is-even", "Add isEven helper", "Add helpers.js with isEven function", "Create helpers.js that exports isEven(n) returning true when n is divisible by 2.", {
-    "test.js": `import { isEven } from './helpers.js';
-import assert from 'node:assert';
-assert.strictEqual(isEven(4), true);
-assert.strictEqual(isEven(3), false);
-console.log('isEven test passed');
-`
-  }),
-  simpleTask("uppercase-first", "Uppercase first letter", "Add capitalize.js with capitalize function", "Create capitalize.js that exports capitalize(str) returning the string with the first character upper-cased.", {
-    "test.js": `import { capitalize } from './capitalize.js';
-import assert from 'node:assert';
-assert.strictEqual(capitalize('hello'), 'Hello');
-assert.strictEqual(capitalize(''), '');
-console.log('capitalize test passed');
-`
-  }),
-  simpleTask("remove-console-log", "Remove debug console.log", "Clean up debug logging", "Remove the console.log call from src/main.js so the linter passes. Do not change the exported function behavior.", {
-    "src/main.js": `export function run() { console.log('debug'); return 'ok'; }
-`,
-    "test.js": `import { run } from './src/main.js';
-import assert from 'node:assert';
-assert.strictEqual(run(), 'ok');
-console.log('run test passed');
-`,
-    "lint.js": `import fs from 'node:fs/promises';
-const code = await fs.readFile('./src/main.js', 'utf-8');
-if (code.includes('console.log')) { console.error('lint failed: console.log found'); process.exit(1); }
-console.log('lint ok');
-`
-  }),
-  simpleTask("fix-indentation", "Fix indentation", "Normalize indentation in src/code.js", "Convert the mixed tabs/spaces indentation in src/code.js to two spaces so the lint script passes. Keep behavior.", {
-    "src/code.js": `export function add(a, b) {
-	return a + b;
-}
-`,
-    "test.js": `import { add } from './src/code.js';
-import assert from 'node:assert';
-assert.strictEqual(add(1, 2), 3);
-console.log('add test passed');
-`,
-    "lint.js": `import fs from 'node:fs/promises';
-const code = await fs.readFile('./src/code.js', 'utf-8');
-if (code.includes('\\t')) { console.error('lint failed: tabs found'); process.exit(1); }
-console.log('lint ok');
-`
-  }),
-  simpleTask("arrow-function", "Convert to arrow function", "Refactor src/math.js to use arrow function", "Rewrite src/math.js so double is defined as a const arrow function. Do not change behavior.", {
-    "src/math.js": `export function double(n) { return n * 2; }
-`,
-    "test.js": `import fs from 'node:fs/promises';
-import { double } from './src/math.js';
-import assert from 'node:assert';
-const code = await fs.readFile('./src/math.js', 'utf-8');
-assert.strictEqual(double(5), 10);
-if (!code.includes('=>')) { console.error('lint failed: not arrow function'); process.exit(1); }
-console.log('arrow test passed');
-`
-  }),
-  simpleTask("default-export", "Add a default export", "Make greet the default export", "Update greet.js so the greet function is the default export instead of a named export. Update the test accordingly.", {
-    "greet.js": `export function greet(name) { return 'Hello, ' + name + '!'; }
-`,
-    "test.js": `import greet from './greet.js';
-import assert from 'node:assert';
-assert.strictEqual(greet('World'), 'Hello, World!');
-console.log('default export test passed');
-`
-  }),
-  simpleTask("update-env", "Update environment variable default", "Set PORT in .env", "Create or update .env so it contains PORT=3000.", {
-    "test.js": `import { readFile } from 'node:fs/promises';
-const env = await readFile('./.env', 'utf-8');
-if (!env.includes('PORT=3000')) { console.error('PORT=3000 not found'); process.exit(1); }
-console.log('env test passed');
-`
-  }),
-  simpleTask("add-package-script", "Add package script", "Add a format script to package.json", "Edit package.json to add a format script with value prettier --write .. Do not remove existing scripts.", {
-    "test.js": `import { readFile } from 'node:fs/promises';
-const pkg = JSON.parse(await readFile('./package.json', 'utf-8'));
-if (pkg.scripts?.format !== 'prettier --write .') { console.error('format script missing'); process.exit(1); }
-console.log('package script test passed');
-`
-  }),
-  simpleTask("update-readme", "Update README heading", "Fix the README title", "Update README.md so the first line is # Omega Synthetic Bench instead of # Old Title.", {
-    "README.md": `# Old Title
-
-This is a sample project.
-`,
-    "test.js": `import { readFile } from 'node:fs/promises';
-const readme = await readFile('./README.md', 'utf-8');
-if (!readme.startsWith('# Omega Synthetic Bench')) { console.error('README heading wrong'); process.exit(1); }
-console.log('readme test passed');
-`
-  }),
-  simpleTask("rename-variable", "Rename a variable", "Rename x to value", "In src/calc.js, rename the parameter x to value everywhere. Do not change behavior.", {
-    "src/calc.js": `export function calc(x) { return x * 2; }
-`,
-    "test.js": `import { calc } from './src/calc.js';
-import assert from 'node:assert';
-assert.strictEqual(calc(4), 8);
-console.log('rename test passed');
-`,
-    "lint.js": `import fs from 'node:fs/promises';
-const code = await fs.readFile('./src/calc.js', 'utf-8');
-if (code.includes('(x)') || code.includes(' x ')) { console.error('old variable name found'); process.exit(1); }
-console.log('lint ok');
-`
-  }),
-  simpleTask("add-input-validation", "Add input validation", "Validate input in divide function", "Update src/divide.js so divide(a, b) throws an error when b is 0.", {
-    "src/divide.js": `export function divide(a, b) { return a / b; }
-`,
-    "test.js": `import { divide } from './src/divide.js';
-import assert from 'node:assert';
-assert.strictEqual(divide(10, 2), 5);
-try { divide(1, 0); console.error('should throw'); process.exit(1); } catch { }
-console.log('divide test passed');
-`
-  }),
-  simpleTask("handle-empty-array", "Handle empty array", "Make sumAll handle empty arrays", "Update src/sumAll.js so sumAll([]) returns 0 instead of throwing.", {
-    "src/sumAll.js": `export function sumAll(numbers) { return numbers.reduce((a, b) => a + b); }
-`,
-    "test.js": `import { sumAll } from './src/sumAll.js';
-import assert from 'node:assert';
-assert.strictEqual(sumAll([1, 2, 3]), 6);
-assert.strictEqual(sumAll([]), 0);
-console.log('sumAll test passed');
-`
-  }),
-  simpleTask("add-unit-test", "Add a unit test", "Write a test for multiply", "Create src/multiply.js exporting multiply(a, b). Then create test.js that asserts multiply(3, 4) === 12.", {
-    "src/multiply.js": `export function multiply(a, b) { return a * b; }
-`,
-    "test.js": `// replace with proper tests
-`,
-    "lint.js": `import fs from 'node:fs/promises';
-const test = await fs.readFile('./test.js', 'utf-8');
-if (!test.includes('multiply')) { console.error('test does not cover multiply'); process.exit(1); }
-console.log('lint ok');
-`
-  }),
-  simpleTask("create-module-index", "Create module index", "Export all helpers from index.js", "Create src/index.js that re-exports foo from ./foo.js and bar from ./bar.js.", {
-    "src/foo.js": `export const foo = 'foo';
-`,
-    "src/bar.js": `export const bar = 'bar';
-`,
-    "test.js": `import { foo, bar } from './src/index.js';
-import assert from 'node:assert';
-assert.strictEqual(foo, 'foo');
-assert.strictEqual(bar, 'bar');
-console.log('index test passed');
-`
-  }),
-  simpleTask("add-async-function", "Add async fetch wrapper", "Create fetchText function", "Create src/fetch.js exporting fetchText(url) that returns a Promise resolving to the string fetched: <url>.", {
-    "test.js": `import { fetchText } from './src/fetch.js';
-import assert from 'node:assert';
-const result = await fetchText('https://example.com');
-assert.strictEqual(result, 'fetched: https://example.com');
-console.log('fetch test passed');
-`
-  }),
-  simpleTask("add-try-catch", "Wrap parsing in try/catch", "Make safeParse not throw", "Update src/parse.js so safeParse(str) returns null when JSON parsing fails instead of throwing.", {
-    "src/parse.js": `export function safeParse(str) { return JSON.parse(str); }
-`,
-    "test.js": `import { safeParse } from './src/parse.js';
-import assert from 'node:assert';
-assert.deepStrictEqual(safeParse('{"a":1}'), { a: 1 });
-assert.strictEqual(safeParse('not json'), null);
-console.log('safeParse test passed');
-`
-  }),
-  simpleTask("timeout-promise", "Add delay helper", "Create sleep(ms) helper", "Create src/time.js exporting sleep(ms) that returns a Promise resolving after ms milliseconds.", {
-    "test.js": `import { sleep } from './src/time.js';
-const start = Date.now();
-await sleep(50);
-if (Date.now() - start < 40) { console.error('sleep too short'); process.exit(1); }
-console.log('sleep test passed');
-`
-  }),
-  simpleTask("read-test-path", "Create file at path from test", "Create lib/helpers.js with capitalize function", "Create a file that exports a capitalize(str) function. Read the existing test to find the exact import path and expected output.", {
-    "test.js": `import { capitalize } from './lib/helpers.js';
-import assert from 'node:assert';
-assert.strictEqual(capitalize('hello'), 'Hello');
-console.log('capitalize test passed');
-`
-  }, "medium"),
-  simpleTask("add-cli-args-parser", "Add CLI argument parser", "Create parseArgs function", "Create src/args.js exporting parseArgs(argv) that returns an object from --key value pairs.", {
-    "test.js": `import { parseArgs } from './src/args.js';
-import assert from 'node:assert';
-assert.deepStrictEqual(parseArgs(['--name', 'x', '--count', '2']), { name: 'x', count: '2' });
-console.log('args test passed');
-`
-  }, "medium"),
-  simpleTask("create-middleware", "Create logging middleware", "Add logger middleware", "Create src/middleware.js exporting logger(req, next) that calls next(), increments req.logCount and returns the result.", {
-    "test.js": `import { logger } from './src/middleware.js';
-import assert from 'node:assert';
-const req = { logCount: 0 };
-const result = logger(req, () => 'done');
-assert.strictEqual(result, 'done');
-assert.strictEqual(req.logCount, 1);
-console.log('middleware test passed');
-`
-  }, "medium"),
-  simpleTask("add-throttle", "Add throttle helper", "Create throttle function", "Create src/throttle.js exporting throttle(fn, ms) that returns a throttled function (only calls fn at most once per ms).", {
-    "test.js": `import { throttle } from './src/throttle.js';
-import assert from 'node:assert';
-let calls = 0;
-const fn = throttle(() => calls++, 100);
-fn(); fn(); fn();
-assert.strictEqual(calls, 1);
-console.log('throttle test passed');
-`
-  }, "medium"),
-  simpleTask("memoize-function", "Add memoize helper", "Create memoize function", "Create src/memoize.js exporting memoize(fn) that caches results by first argument.", {
-    "test.js": `import { memoize } from './src/memoize.js';
-import assert from 'node:assert';
-let calls = 0;
-const fn = memoize((n) => { calls++; return n * 2; });
-assert.strictEqual(fn(3), 6);
-assert.strictEqual(fn(3), 6);
-assert.strictEqual(calls, 1);
-console.log('memoize test passed');
-`
-  }, "medium"),
-  simpleTask("fix-race-condition", "Fix a race condition", "Make counter increment atomic", "Update src/counter.js so increment() returns a Promise and uses a local variable correctly. Keep the async interface.", {
-    "src/counter.js": `let count = 0;
-export async function increment() {
-  const current = count;
-  await Promise.resolve();
-  count = current + 1;
-  return count;
-}
-`,
-    "test.js": `import { increment } from './src/counter.js';
-import assert from 'node:assert';
-await Promise.all([increment(), increment(), increment()]);
-const final = await increment();
-assert.strictEqual(final, 4);
-console.log('counter test passed');
-`
-  }, "medium"),
   {
     id: "noop-validation",
     name: "Run validation on a clean project",
@@ -1568,10 +1307,214 @@ async function readTask(taskDir) {
   const instructionRaw = await fs4.readFile(path7.join(taskDir, "instruction.md"), "utf-8");
   return { toml: parseToml(tomlRaw), instruction: instructionRaw };
 }
-async function cloneRepo(repoUrl, commit2, targetPath) {
+async function cloneRepo(repoUrl, commit, targetPath) {
   await fs4.mkdir(path7.dirname(targetPath), { recursive: true });
   await execFileAsync("git", ["clone", repoUrl, targetPath], { timeout: 12e4 });
-  await execFileAsync("git", ["-C", targetPath, "checkout", commit2], { timeout: 6e4 });
+  await execFileAsync("git", ["-C", targetPath, "checkout", commit], { timeout: 6e4 });
+}
+async function writeFile3(filePath, content) {
+  await fs4.mkdir(path7.dirname(filePath), { recursive: true });
+  await fs4.writeFile(filePath, content, "utf-8");
+}
+async function generateModelPatch(projectPath, baseCommit) {
+  const { stdout } = await execFileAsync("git", ["-C", projectPath, "diff", "--binary", baseCommit, "HEAD"], {
+    timeout: 6e4
+  });
+  return stdout;
+}
+function normalisePatch(patch) {
+  if (patch.length === 0)
+    return patch;
+  return patch.endsWith("\n") ? patch : `${patch}
+`;
+}
+async function rewriteConfig(taskDir, copiedTestsDir, appDir, verifierDir, artifactsDir) {
+  const configPath = path7.join(taskDir, "tests", "config.json");
+  const raw = await fs4.readFile(configPath, "utf-8");
+  const config = JSON.parse(raw);
+  const replacer = (value) => {
+    if (typeof value === "string") {
+      return value.replace(/\/logs\/verifier/g, verifierDir).replace(/\/logs\/artifacts/g, artifactsDir).replace(/\/tests/g, copiedTestsDir).replace(/\/app\b/g, appDir).replace(/\/app\//g, `${appDir}/`);
+    }
+    if (Array.isArray(value)) {
+      return value.map(replacer);
+    }
+    if (value && typeof value === "object") {
+      const out = {};
+      for (const [k, v] of Object.entries(value)) {
+        out[k] = replacer(v);
+      }
+      return out;
+    }
+    return value;
+  };
+  const rewritten = replacer(config);
+  await writeFile3(path7.join(copiedTestsDir, "config.json"), JSON.stringify(rewritten, null, 2));
+  return rewritten;
+}
+function replaceTestShPaths(script) {
+  let replaced = script;
+  replaced = replaced.replace(/\/logs\/verifier/g, "${VERIFIER_DIR}");
+  replaced = replaced.replace(/\/logs\/artifacts/g, "${ARTIFACTS_DIR}");
+  replaced = replaced.replace(/\/tests/g, "${TESTS_DIR}");
+  replaced = replaced.replace(/\/app\b/g, "${APP_DIR}");
+  replaced = replaced.replace(/\/app\//g, "${APP_DIR}/");
+  return replaced;
+}
+async function runCommand(command, args, options) {
+  try {
+    const { stdout, stderr } = await execFileAsync(command, args, {
+      cwd: options.cwd,
+      env: options.env,
+      timeout: options.timeout ?? 6e5
+    });
+    return { stdout, stderr, exitCode: 0 };
+  } catch (err) {
+    const e = err;
+    return {
+      stdout: e.stdout ?? "",
+      stderr: e.stderr ?? "",
+      exitCode: e.code ?? 1
+    };
+  }
+}
+async function dockerAvailable() {
+  try {
+    const { exitCode } = await runCommand("docker", ["info"], { timeout: 1e4 });
+    return exitCode === 0;
+  } catch {
+    return false;
+  }
+}
+async function imageExists(tag) {
+  const { exitCode } = await runCommand("docker", ["image", "inspect", tag], { timeout: 1e4 });
+  return exitCode === 0;
+}
+async function buildDeepSWEImage(taskDir, taskName) {
+  const dockerfileDir = path7.join(taskDir, "environment");
+  const dockerfilePath = path7.join(dockerfileDir, "Dockerfile");
+  try {
+    await fs4.access(dockerfilePath);
+  } catch {
+    return void 0;
+  }
+  const tag = `omega-deepswe-${taskName.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
+  if (await imageExists(tag)) {
+    return tag;
+  }
+  const build = await runCommand("docker", ["build", "-t", tag, "-f", dockerfilePath, dockerfileDir], {
+    timeout: 12e5
+  });
+  if (build.exitCode !== 0) {
+    throw new Error(`Docker build failed for ${taskName}:
+${build.stderr}
+${build.stdout}`);
+  }
+  return tag;
+}
+async function runDeepSWEVerifierDocker(projectPath, taskDir, baseCommit, taskName, modelPatchArg) {
+  const absoluteTaskDir = path7.resolve(taskDir);
+  const testsDir = path7.join(absoluteTaskDir, "tests");
+  const workDir = path7.join("/tmp", `deepswe-${path7.basename(taskDir)}-${String(Date.now())}`);
+  const verifierDir = path7.join(workDir, "logs", "verifier");
+  const artifactsDir = path7.join(workDir, "logs", "artifacts");
+  const logFile = path7.join(workDir, "verifier.log");
+  await fs4.mkdir(verifierDir, { recursive: true });
+  await fs4.mkdir(artifactsDir, { recursive: true });
+  const modelPatch = normalisePatch(modelPatchArg ?? await generateModelPatch(projectPath, baseCommit));
+  await writeFile3(path7.join(artifactsDir, "model.patch"), modelPatch);
+  const image = await buildDeepSWEImage(absoluteTaskDir, taskName);
+  if (!image) {
+    throw new Error(`No Dockerfile found for DeepSWE task ${taskName}`);
+  }
+  const logLines = [];
+  function log(line) {
+    logLines.push(line);
+  }
+  log(`Using Docker image ${image}`);
+  const args = [
+    "run",
+    "--rm",
+    "-v",
+    `${testsDir}:/tests:ro`,
+    "-v",
+    `${artifactsDir}:/logs/artifacts`,
+    "-v",
+    `${verifierDir}:/logs/verifier`,
+    image,
+    "bash",
+    "/tests/test.sh"
+  ];
+  const testRun = await runCommand("docker", args, { timeout: 6e5 });
+  log(`=== test.sh stdout ===
+${testRun.stdout}`);
+  log(`=== test.sh stderr ===
+${testRun.stderr}`);
+  let reward = {};
+  try {
+    const rewardRaw = await fs4.readFile(path7.join(verifierDir, "reward.json"), "utf-8");
+    reward = JSON.parse(rewardRaw);
+  } catch {
+  }
+  const logs = logLines.join("\n");
+  await fs4.writeFile(logFile, logs, "utf-8").catch(() => {
+  });
+  return { reward, logs, logFile, exitCode: testRun.exitCode };
+}
+async function runDeepSWEVerifier(projectPath, taskDir, baseCommit, useDocker, modelPatchArg) {
+  if (useDocker && await dockerAvailable()) {
+    return runDeepSWEVerifierDocker(projectPath, taskDir, baseCommit, path7.basename(taskDir), modelPatchArg);
+  }
+  const testsDir = path7.join(taskDir, "tests");
+  const workDir = path7.join("/tmp", `deepswe-${path7.basename(taskDir)}-${String(Date.now())}`);
+  const verifierDir = path7.join(workDir, "logs", "verifier");
+  const artifactsDir = path7.join(workDir, "logs", "artifacts");
+  const copiedTestsDir = path7.join(workDir, "tests");
+  const logFile = path7.join(workDir, "verifier.log");
+  await fs4.mkdir(verifierDir, { recursive: true });
+  await fs4.mkdir(artifactsDir, { recursive: true });
+  await execFileAsync("cp", ["-R", testsDir, copiedTestsDir], { timeout: 6e4 });
+  await rewriteConfig(taskDir, copiedTestsDir, projectPath, verifierDir, artifactsDir);
+  const modelPatch = normalisePatch(modelPatchArg ?? await generateModelPatch(projectPath, baseCommit));
+  await writeFile3(path7.join(artifactsDir, "model.patch"), modelPatch);
+  await execFileAsync("git", ["-C", projectPath, "checkout", "-f", baseCommit], { timeout: 6e4 });
+  const env = {
+    ...process.env,
+    APP_DIR: projectPath,
+    TESTS_DIR: copiedTestsDir,
+    VERIFIER_DIR: verifierDir,
+    ARTIFACTS_DIR: artifactsDir,
+    PATH: `${process.env.PATH ?? ""}:${process.env.HOME ?? "/Users/benebsworth"}/go/bin`
+  };
+  const logLines = [];
+  function log(line) {
+    logLines.push(line);
+  }
+  const testShPath = path7.join(copiedTestsDir, "test.sh");
+  const testShRaw = await fs4.readFile(testShPath, "utf-8");
+  const rewritten = replaceTestShPaths(testShRaw);
+  const localTestSh = path7.join(workDir, "test.sh");
+  await writeFile3(localTestSh, rewritten);
+  await fs4.chmod(localTestSh, 493);
+  const testRun = await runCommand("bash", [localTestSh], {
+    cwd: projectPath,
+    env,
+    timeout: 6e5
+  });
+  log(`=== test.sh stdout ===
+${testRun.stdout}`);
+  log(`=== test.sh stderr ===
+${testRun.stderr}`);
+  let reward = {};
+  try {
+    const rewardRaw = await fs4.readFile(path7.join(verifierDir, "reward.json"), "utf-8");
+    reward = JSON.parse(rewardRaw);
+  } catch {
+  }
+  const logs = logLines.join("\n");
+  await fs4.writeFile(logFile, logs, "utf-8").catch(() => {
+  });
+  return { reward, logs, logFile, exitCode: testRun.exitCode };
 }
 async function loadDeepSWESuite(options) {
   const entries = await fs4.readdir(options.tasksDir, { withFileTypes: true });
@@ -1600,7 +1543,7 @@ async function loadDeepSWESuite(options) {
     const id = toml.metadata?.task_id ?? path7.basename(dir);
     const title = toml.metadata?.display_title ?? toml.metadata?.original_title ?? id;
     const repo = toml.metadata?.repository_url;
-    const commit2 = toml.metadata?.base_commit_hash;
+    const commit = toml.metadata?.base_commit_hash;
     tasks.push({
       id: `deepswe-${id}`,
       name: id,
@@ -1608,19 +1551,37 @@ async function loadDeepSWESuite(options) {
       description: instruction,
       complexity: "complex",
       setup: async (projectPath) => {
-        if (!repo || !commit2) {
+        if (!repo || !commit) {
           throw new Error(`DeepSWE task ${id} is missing repository_url or base_commit_hash`);
         }
-        await cloneRepo(repo, commit2, projectPath);
+        await cloneRepo(repo, commit, projectPath);
       },
-      evaluate: (ctx) => {
-        const hasDiff = ctx.diffs.length > 0 && ctx.diffs.some((d) => d.patch.trim().length > 0);
-        const done = ctx.agentRun?.resultStatus === "done";
-        const passed = done && hasDiff;
-        return Promise.resolve({
+      evaluate: async (ctx) => {
+        if (!commit) {
+          return { passed: false, message: "Missing base_commit_hash" };
+        }
+        const storedPatch = ctx.diffs.length > 0 ? ctx.diffs[0].patch : void 0;
+        const { reward, logs, logFile, exitCode } = await runDeepSWEVerifier(ctx.projectPath, dir, commit, options.useDocker ?? false, storedPatch);
+        const passed = reward.reward === 1;
+        const metrics = {
+          f2p_passed: reward.f2p_passed ?? 0,
+          f2p_total: reward.f2p_total ?? 0,
+          p2p_passed: reward.p2p_passed ?? 0,
+          p2p_total: reward.p2p_total ?? 0,
+          partial: reward.partial ?? 0,
+          verifier_exit_code: exitCode,
+          verifier_log_file: logFile
+        };
+        if (reward.apply_failed) {
+          metrics.apply_failed = 1;
+        }
+        metrics.verifier_logs = logs.slice(-4096);
+        return {
           passed,
-          message: passed ? `Agent finished and produced a diff (${String(ctx.diffs[0]?.patch.length ?? 0)} chars)` : `Agent status=${ctx.agentRun?.resultStatus ?? "unknown"}, diff=${hasDiff ? "yes" : "no"}`
-        });
+          score: reward.partial,
+          message: passed ? `DeepSWE verifier passed (f2p ${String(reward.f2p_passed ?? 0)}/${String(reward.f2p_total ?? 0)}, p2p ${String(reward.p2p_passed ?? 0)}/${String(reward.p2p_total ?? 0)})` : `DeepSWE verifier failed (reward=${String(reward.reward ?? "missing")}, f2p ${String(reward.f2p_passed ?? 0)}/${String(reward.f2p_total ?? 0)}, p2p ${String(reward.p2p_passed ?? 0)}/${String(reward.p2p_total ?? 0)})`,
+          metrics
+        };
       }
     });
   }
@@ -1645,12 +1606,12 @@ function summariseTraceFlow(report, maxLines = 60) {
   return lines.slice(0, maxLines).join("\n") + "\n... (truncated)";
 }
 function buildOptimisePrompt(report, failedResult, traceFlowText) {
-  const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
+  const passRate = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
   const failed = report.results.filter((r) => !r.evaluation.passed);
   const lines = [];
   lines.push("# Prompt optimisation task");
   lines.push("");
-  lines.push(`The latest benchmark run achieved ${String(passRate2)}% pass rate (${String(report.passed)}/${String(report.total)}).`);
+  lines.push(`The latest benchmark run achieved ${String(passRate)}% pass rate (${String(report.passed)}/${String(report.total)}).`);
   lines.push("");
   lines.push("## Failed tasks");
   for (const r of failed.slice(0, 5)) {
@@ -1671,14 +1632,11 @@ function buildOptimisePrompt(report, failedResult, traceFlowText) {
     lines.push("");
   }
   lines.push("## Instructions");
-  lines.push("Analyse the failures above and edit ONLY `packages/agent/src/prompts.ts` to improve the Omega agent.");
-  lines.push("Your goal is to change the system prompt text so the agent avoids the observed failures.");
+  lines.push("Analyse the failures above and edit `packages/agent/src/prompts.ts` (and optionally `packages/agent/src/planner.ts`) to improve the Omega agent.");
   lines.push("Rules:");
-  lines.push("- ONLY edit `packages/agent/src/prompts.ts`. Do NOT edit any other files, including schema files, migrations, or unrelated code.");
-  lines.push("- Keep changes minimal and targeted to the failure pattern.");
-  lines.push("- Use edit_file with an exact old_string from the file. Read the file first if needed.");
+  lines.push("- Keep changes minimal and targeted.");
   lines.push("- Do not break existing tool signatures or system behaviour.");
-  lines.push("- After editing, run `pnpm lint` and `pnpm test` to verify.");
+  lines.push("- Run `pnpm lint` and `pnpm test` after edits.");
   lines.push("- Finish with a summary of what changed and why it should help.");
   return lines.join("\n");
 }
@@ -1714,359 +1672,6 @@ async function submitOptimiseTask(apiUrl, projectId, prompt) {
   return res.json();
 }
 
-// ../bench/dist/ab.js
-function passRate(report) {
-  return report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
-}
-function findResult(report, taskId) {
-  return report.results.find((r) => r.task.id === taskId);
-}
-function buildAbReport(baseline, candidate, baselineReport, candidateReport) {
-  const baselineRate = passRate(baselineReport);
-  const candidateRate = passRate(candidateReport);
-  const taskIds = /* @__PURE__ */ new Set();
-  for (const r of baselineReport.results)
-    taskIds.add(r.task.id);
-  for (const r of candidateReport.results)
-    taskIds.add(r.task.id);
-  const perTask = Array.from(taskIds).map((taskId) => {
-    const base = findResult(baselineReport, taskId);
-    const cand = findResult(candidateReport, taskId);
-    return {
-      taskId,
-      taskName: base?.task.name ?? cand?.task.name ?? taskId,
-      baselinePassed: base?.evaluation.passed ?? false,
-      candidatePassed: cand?.evaluation.passed ?? false,
-      baselineTokens: base?.usage?.totalTokens,
-      candidateTokens: cand?.usage?.totalTokens
-    };
-  });
-  return {
-    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-    baseline,
-    candidate,
-    baselineReport,
-    candidateReport,
-    delta: {
-      passRate: candidateRate - baselineRate,
-      passedDelta: candidateReport.passed - baselineReport.passed,
-      durationMsDelta: candidateReport.totalDurationMs - baselineReport.totalDurationMs,
-      promptTokensDelta: (candidateReport.totalUsage?.promptTokens ?? 0) - (baselineReport.totalUsage?.promptTokens ?? 0),
-      completionTokensDelta: (candidateReport.totalUsage?.completionTokens ?? 0) - (baselineReport.totalUsage?.completionTokens ?? 0),
-      totalTokensDelta: (candidateReport.totalUsage?.totalTokens ?? 0) - (baselineReport.totalUsage?.totalTokens ?? 0)
-    },
-    perTask
-  };
-}
-async function runAbBenchmark(tasks, baseline, candidate, options) {
-  const suiteName = options.suiteName ?? "ab";
-  const baselineReport = await runBenchmark(tasks, {
-    ...options,
-    suiteName: `${suiteName}-baseline`
-  });
-  const candidateReport = await runBenchmark(tasks, {
-    ...options,
-    suiteName: `${suiteName}-candidate`
-  });
-  return buildAbReport(baseline, candidate, baselineReport, candidateReport);
-}
-function deltaSymbol(value) {
-  if (value > 0)
-    return "+";
-  if (value < 0)
-    return "";
-  return "";
-}
-function formatDelta(value) {
-  return `${deltaSymbol(value)}${String(value)}`;
-}
-function formatAbReport(report) {
-  const lines = [];
-  lines.push("# Omega A/B Benchmark Report");
-  lines.push("");
-  lines.push(`- Timestamp: ${report.timestamp}`);
-  lines.push(`- Baseline: ${report.baseline.name}`);
-  lines.push(`- Candidate: ${report.candidate.name}`);
-  lines.push("");
-  lines.push("## Summary");
-  lines.push(`| Metric | Baseline | Candidate | Delta |`);
-  lines.push(`|---|---|---|---|`);
-  lines.push(`| Pass rate | ${String(passRate(report.baselineReport))}% | ${String(passRate(report.candidateReport))}% | ${formatDelta(report.delta.passRate)}% |`);
-  lines.push(`| Passed | ${String(report.baselineReport.passed)}/${String(report.baselineReport.total)} | ${String(report.candidateReport.passed)}/${String(report.candidateReport.total)} | ${formatDelta(report.delta.passedDelta)} |`);
-  lines.push(`| Duration | ${formatDuration2(report.baselineReport.totalDurationMs)} | ${formatDuration2(report.candidateReport.totalDurationMs)} | ${formatDuration2(report.delta.durationMsDelta)} |`);
-  lines.push(`| Tokens | ${String(report.baselineReport.totalUsage?.totalTokens ?? 0)} | ${String(report.candidateReport.totalUsage?.totalTokens ?? 0)} | ${formatDelta(report.delta.totalTokensDelta)} |`);
-  lines.push("");
-  lines.push("## Per-task comparison");
-  lines.push(`| Task | Baseline | Candidate | Token delta |`);
-  lines.push(`|---|---|---|---|`);
-  for (const t of report.perTask) {
-    const tokenDelta = t.baselineTokens !== void 0 && t.candidateTokens !== void 0 ? formatDelta(t.candidateTokens - t.baselineTokens) : "-";
-    lines.push(`| ${t.taskName} | ${t.baselinePassed ? "pass" : "fail"} | ${t.candidatePassed ? "pass" : "fail"} | ${tokenDelta} |`);
-  }
-  lines.push("");
-  return lines.join("\n");
-}
-function formatDuration2(ms) {
-  if (ms < 1e3)
-    return `${String(Math.round(ms))}ms`;
-  return `${(ms / 1e3).toFixed(1)}s`;
-}
-
-// ../agent/dist/tool-definitions.js
-var AGENT_TOOLS = [
-  {
-    name: "read_file",
-    description: "Read a file relative to project root.",
-    parameters: {
-      type: "object",
-      properties: { path: { type: "string" } },
-      required: ["path"]
-    }
-  },
-  {
-    name: "write_file",
-    description: "Write content to a file relative to project root.",
-    parameters: {
-      type: "object",
-      properties: { path: { type: "string" }, content: { type: "string" } },
-      required: ["path", "content"]
-    }
-  },
-  {
-    name: "edit_file",
-    description: "Apply a targeted edit to an existing file relative to project root. Replaces one occurrence of old_string with new_string.",
-    parameters: {
-      type: "object",
-      properties: {
-        path: { type: "string" },
-        old_string: { type: "string" },
-        new_string: { type: "string" }
-      },
-      required: ["path", "old_string", "new_string"]
-    }
-  },
-  {
-    name: "run_command",
-    description: "Run a shell command in the project root.",
-    parameters: {
-      type: "object",
-      properties: { command: { type: "string" } },
-      required: ["command"]
-    }
-  },
-  {
-    name: "think",
-    description: "Record reasoning.",
-    parameters: {
-      type: "object",
-      properties: { thought: { type: "string" } },
-      required: ["thought"]
-    }
-  },
-  {
-    name: "finish",
-    description: "Mark the task complete.",
-    parameters: {
-      type: "object",
-      properties: { summary: { type: "string" }, success: { type: "boolean" } },
-      required: ["summary", "success"]
-    }
-  },
-  {
-    name: "publish",
-    description: "Build, validate, and publish the project.",
-    parameters: {
-      type: "object",
-      properties: { version: { type: "string" } },
-      required: []
-    }
-  }
-];
-
-// ../agent/dist/planner.js
-var toolDescriptions = AGENT_TOOLS.map((t) => `- ${t.name}: ${t.description}`).join("\n");
-var PLAN_PROMPT = `You are a planning assistant. Given a task, produce a concise step-by-step plan.
-
-Respond with strict JSON in this exact shape (no markdown):
-{
-  "reasoning": "brief reasoning",
-  "plan": [
-    { "name": "step name", "tool": "optional tool name", "input": { optional tool args } }
-  ]
-}
-
-Available tools:
-${toolDescriptions}
-
-If a step does not need a tool, omit tool/input. Use edit_file for small file changes.`;
-
-// ../agent/dist/tools.js
-import { execFile as execFile2 } from "node:child_process";
-import { promisify as promisify2 } from "node:util";
-var execFileAsync2 = promisify2(execFile2);
-
-// ../agent/dist/validator.js
-import { execFile as execFile3 } from "node:child_process";
-import { promisify as promisify3 } from "node:util";
-var execFileAsync3 = promisify3(execFile3);
-
-// ../agent/dist/publisher.js
-import { execFile as execFile5 } from "node:child_process";
-import { promisify as promisify5 } from "node:util";
-
-// ../agent/dist/git.js
-import { execFile as execFile4 } from "node:child_process";
-import { promisify as promisify4 } from "node:util";
-var execFileAsync4 = promisify4(execFile4);
-
-// ../agent/dist/publisher.js
-var execFileAsync5 = promisify5(execFile5);
-
-// ../agent/dist/prompts.js
-import fs6 from "node:fs";
-function loadPromptFromEnv(name) {
-  const direct = process.env[name];
-  if (direct)
-    return direct;
-  const fileKey = `${name}_FILE`;
-  const filePath = process.env[fileKey];
-  if (filePath) {
-    try {
-      return fs6.readFileSync(filePath, "utf-8");
-    } catch {
-    }
-  }
-  return void 0;
-}
-var AGENT_SYSTEM_PROMPT = loadPromptFromEnv("OMEGA_SYSTEM_PROMPT") ?? `You are Omega, an autonomous software engineering agent running inside a project repository.
-
-Your job is to complete the user's task by calling tools. Do not write prose or explanations outside tool calls. Prefer targeted edits over full rewrites.
-
-Available tools:
-
-- read_file: Read a file relative to project root. Arguments: { "path": "relative/path" }
-- write_file: Overwrite or create a file. Arguments: { "path": "relative/path", "content": "full file content" }
-- edit_file: Replace one exact occurrence of old_string with new_string in an existing file. Use this for small changes. Arguments: { "path": "relative/path", "old_string": "...", "new_string": "..." }
-- run_command: Run a single simple command. No pipes (|), &&, ;, redirects, or globs. Prefer pnpm/npm/node. Arguments: { "command": "pnpm lint" }
-- think: Record a reasoning step. Arguments: { "thought": "..." }
-- finish: Mark the task complete. Arguments: { "summary": "what was done", "success": true }. Use summary, not message.
-- publish: Run lint/test/build validation in one step. This is the preferred validation tool. Arguments: { "version": "optional" }
-
-Rules:
-1. Read the task, then use think to plan.
-2. Use edit_file for small changes; write_file only when creating a file or rewriting most of it.
-3. ALWAYS read a file with read_file before using edit_file on it. Do NOT read a file before write_file when creating a new file.
-4. Before creating JavaScript files, read package.json to check whether "type" is "module" and use ESM (export) or CommonJS (module.exports) accordingly.
-5. Immediately after making any edit, call publish to run validation. Do not read more files, do not think, and do not call finish before validating.
-6. If validation fails, fix the issue and call publish again. Repeat until validation passes.
-7. Do NOT call finish until validation passes. If you call finish without validation, the task will fail.
-8. Do not expose secrets or run destructive commands.
-9. Finish only when the task is done. Always include summary and success.
-10. Do not repeat the same tool call with the same arguments. If something fails, change your approach.
-11. Do not ask the user for clarification. Do not say "No task was provided". Use the Task and Description above to proceed.
-12. You have a limited number of steps. If you spend too many steps reading or planning without validating, the task will fail.
-13. NEVER read the same file twice in a row. Once you have read a file, the next step must be edit_file, write_file, run_command, or publish.
-14. If you already know what change to make, do not read additional files. Edit and validate immediately.
-15. The think tool is for planning only. After thinking, you must use a concrete action tool next, not think or read_file again.
-16. When publish reports allPassed=true, call finish with success=true immediately. Do not make additional edits or run extra commands.
-17. Do NOT modify test files (test.js, *.test.js, *.spec.js) unless the task explicitly tells you to update the test.
-18. If a task asks you to read a test to find an import path or expected output, read the test file once and then create the required source file. Do not modify the test.
-19. Create files at the EXACT path specified in the task. If the task says "create capitalize.js", create it at the project root as "capitalize.js", not inside src/ or another folder.
-20. Export functions with the EXACT name requested as a named export (e.g., export function capitalize(...)). Do not use default exports unless explicitly instructed.
-21. Guard string/array operations against empty inputs and null/undefined values (e.g., str ? str[0].toUpperCase() + str.slice(1) : '').
-
-Example tool call sequence for "Add a greet function":
-1. think: { "thought": "I need to create greet.js and export a greet function." }
-2. write_file: { "path": "greet.js", "content": "export function greet(name) { return 'Hello, ' + name + '!'; }\\n" }
-3. publish: { }
-4. finish: { "summary": "Created greet.js with greet(name) function and verified it passes tests.", "success": true }
-
-Example tool call sequence for "Fix a lint warning":
-1. think: { "thought": "The linter flags var. I need to read src/index.js and replace var with const or let." }
-2. read_file: { "path": "src/index.js" }
-3. edit_file: { "path": "src/index.js", "old_string": "var greeting = 'Hello';", "new_string": "const greeting = 'Hello';" }
-4. publish: { }
-5. finish: { "summary": "Replaced var with const and verified lint passes.", "success": true }`;
-var TEXT_TOOLS_SYSTEM_PROMPT = loadPromptFromEnv("OMEGA_TEXT_TOOLS_PROMPT") ?? `You are Omega, an autonomous software engineering agent running inside a project repository.
-
-You MUST respond with a single JSON object containing a "tool_calls" array. Do not output markdown, explanations, or reasoning outside the JSON.
-
-Available tools (use ONLY these exact names):
-
-- read_file: { "path": "relative/path" }
-- write_file: { "path": "relative/path", "content": "full file content" }
-- edit_file: { "path": "relative/path", "old_string": "...", "new_string": "..." }
-- run_command: { "command": "simple command, no pipes/&&/;/redirects" }
-- think: { "thought": "reasoning text" }
-- finish: { "summary": "what was done", "success": true | false }
-- publish: { "version": "optional" }
-
-Rules:
-- Plan with think, then act.
-- Use edit_file for small changes; write_file only for new files or large rewrites.
-- ALWAYS read a file with read_file before using edit_file on it. Do NOT read a file before write_file when creating a new file.
-- Before creating JavaScript files, read package.json to check whether "type" is "module" and use ESM (export) or CommonJS (module.exports) accordingly.
-- Immediately after making any edit, call publish to run validation. Do not read more files or call finish before validating.
-- If validation fails, fix the issue and call publish again.
-- Do NOT call finish until validation passes.
-- Do not expose secrets or run destructive commands.
-- Finish only when done. Use summary, not message.
-- Do not repeat the same tool call with the same arguments.
-- Do not ask the user for clarification. Do not say "No task was provided". Use the Task and Description above to proceed.
-- NEVER read the same file twice in a row. Once read, the next step must edit, write, run_command, or publish.
-- The think tool is for planning only. After thinking, use a concrete action tool next.
-- When publish reports allPassed=true, call finish with success=true immediately.
-- Do NOT modify test files unless the task explicitly tells you to update the test.
-- Create files at the EXACT path specified in the task. If the task says "create capitalize.js", create it at the project root as "capitalize.js", not inside src/ or another folder.
-- Export functions with the EXACT name requested as a named export (e.g., export function capitalize(...)). Do not use default exports unless explicitly instructed.
-- Guard string/array operations against empty inputs and null/undefined values.
-
-Example response for "Add a greet function":
-{ "tool_calls": [
-  { "id": "1", "name": "think", "arguments": { "thought": "Create greet.js with greet(name)." } },
-  { "id": "2", "name": "write_file", "arguments": { "path": "greet.js", "content": "export function greet(name) { return 'Hello, ' + name + '!'; }\\n" } },
-  { "id": "3", "name": "publish", "arguments": {} },
-  { "id": "4", "name": "finish", "arguments": { "summary": "Created greet.js and verified tests pass.", "success": true } }
-] }`;
-
-// ../agent/dist/prompt-versioning.js
-import fs7 from "node:fs/promises";
-import path9 from "node:path";
-import crypto from "node:crypto";
-import { fileURLToPath as fileURLToPath4 } from "node:url";
-var __dirname3 = path9.dirname(fileURLToPath4(import.meta.url));
-var PROMPTS_PATH = path9.resolve(__dirname3, "..", "src", "prompts.ts");
-var PLANNER_PATH = path9.resolve(__dirname3, "..", "src", "planner.ts");
-function hashString(value) {
-  return crypto.createHash("sha256").update(value).digest("hex").slice(0, 16);
-}
-function hashPrompts(input) {
-  return hashString([input.systemPrompt, input.textToolsPrompt, input.planningPrompt ?? ""].join("\n---\n"));
-}
-async function readPromptsSource() {
-  const source = await fs7.readFile(PROMPTS_PATH, "utf-8");
-  const systemMatch = /export const AGENT_SYSTEM_PROMPT =\s*(?:loadPromptFromEnv\('OMEGA_SYSTEM_PROMPT'\) \?\?\s*)?`([\s\S]*?)`;/m.exec(source);
-  const textToolsMatch = /export const TEXT_TOOLS_SYSTEM_PROMPT =\s*(?:loadPromptFromEnv\('OMEGA_TEXT_TOOLS_PROMPT'\) \?\?\s*)?`([\s\S]*?)`;/m.exec(source);
-  return {
-    systemPrompt: systemMatch?.[1] ?? "",
-    textToolsPrompt: textToolsMatch?.[1] ?? ""
-  };
-}
-async function loadCurrentPrompts() {
-  const { systemPrompt, textToolsPrompt } = await readPromptsSource();
-  const plannerSource = await fs7.readFile(PLANNER_PATH, "utf-8");
-  const planningMatch = /const PLAN_PROMPT = `([\s\S]*?)`;/m.exec(plannerSource);
-  const planningPrompt = planningMatch?.[1];
-  return {
-    name: `auto-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`,
-    sourcePath: PROMPTS_PATH,
-    systemPrompt,
-    textToolsPrompt,
-    planningPrompt,
-    hash: hashPrompts({ systemPrompt, textToolsPrompt, planningPrompt })
-  };
-}
-
 // ../../apps/cli/src/commands/bench.ts
 async function waitForApi2(apiUrl, maxMs = 1e4) {
   const deadline = Date.now() + maxMs;
@@ -2082,75 +1687,36 @@ async function waitForApi2(apiUrl, maxMs = 1e4) {
 }
 function currentProject(apiUrl) {
   const cwd = process.cwd();
-  const name = path10.basename(cwd);
+  const name = path9.basename(cwd);
   return ensureProject2(apiUrl, `bench-${name}`, cwd);
 }
-async function savePromptVersion(apiUrl, name) {
-  const current = await loadCurrentPrompts();
-  const versionRes = await fetch(`${apiUrl}/prompt-versions`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({
-      name: name ?? current.name,
-      sourcePath: current.sourcePath,
-      systemPrompt: current.systemPrompt,
-      textToolsPrompt: current.textToolsPrompt,
-      hash: current.hash,
-      metadata: { planningPrompt: current.planningPrompt }
-    })
-  });
-  if (!versionRes.ok) {
-    console.warn("Could not save prompt version:", await versionRes.text());
-    return void 0;
-  }
-  return versionRes.json();
-}
-async function loadPromptSpec(source) {
-  if (source === "current") {
-    const current = await loadCurrentPrompts();
-    return {
-      name: "current",
-      systemPrompt: current.systemPrompt,
-      textToolsPrompt: current.textToolsPrompt
-    };
-  }
-  const raw = await fs8.readFile(path10.resolve(source), "utf-8");
-  const parsed = JSON.parse(raw);
-  if (!parsed.systemPrompt || !parsed.textToolsPrompt) {
-    throw new Error("Prompt spec must include systemPrompt and textToolsPrompt");
-  }
-  return parsed;
-}
-async function resolveTasks(opts) {
+var runCmd = new Command8("run").description("Run a benchmark suite").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks (for deep-swe)", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for benchmark tasks").option("--model <model>", "model to use for benchmark tasks").option("--docker", "run DeepSWE verifiers in Docker (required for most Node.js tasks)").action(async (opts) => {
+  const apiUrl = getApiUrl();
+  await waitForApi2(apiUrl);
+  const timeoutMs = Number(opts.timeout);
+  let tasks;
+  let suiteName;
   if (opts.suite === "deep-swe") {
     if (!opts.path) {
       throw new Error("--path is required for the deep-swe suite");
     }
-    const tasks = await loadDeepSWESuite({
+    tasks = await loadDeepSWESuite({
       tasksDir: opts.path,
       nTasks: opts.nTasks,
-      sampleSeed: opts.sampleSeed
+      sampleSeed: opts.sampleSeed,
+      useDocker: opts.docker
     });
-    return { tasks, suiteName: "deep-swe" };
-  }
-  if (opts.suite === "synthetic") {
-    let tasks = syntheticSuite();
-    if (opts.nTasks !== void 0 && opts.nTasks > 0) {
-      tasks = tasks.slice(0, opts.nTasks);
-    }
-    return { tasks, suiteName: "synthetic" };
+    suiteName = "deep-swe";
+  } else if (opts.suite === "synthetic") {
+    tasks = syntheticSuite();
+    suiteName = "synthetic";
+  } else {
+    throw new Error(`Unknown suite: ${opts.suite}`);
   }
-  throw new Error(`Unknown suite: ${opts.suite}`);
-}
-var runCmd = new Command8("run").description("Run a benchmark suite").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for benchmark tasks").option("--model <model>", "model to use for benchmark tasks").action(async (opts) => {
-  const apiUrl = getApiUrl();
-  await waitForApi2(apiUrl);
-  const { tasks, suiteName } = await resolveTasks(opts);
   if (tasks.length === 0) {
     console.log("No benchmark tasks to run.");
     return;
   }
-  const timeoutMs = Number(opts.timeout);
   console.log(`Running ${String(tasks.length)} benchmark tasks against ${apiUrl}`);
   const report = await runBenchmark(tasks, {
     apiUrl,
@@ -2186,133 +1752,10 @@ var optimiseCmd = new Command8("optimise").description("Create a self-improve ta
   console.log(`Created self-improve task ${task.id}`);
   console.log(`Run \`omega task run ${task.id}\` to execute it.`);
 });
-var abCmd = new Command8("ab").description("Run an A/B benchmark comparing two prompt versions").option("--baseline <source>", 'baseline prompts: "current" or path to JSON file', "current").option("--candidate <source>", "candidate prompts: path to JSON file").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for benchmark tasks").option("--model <model>", "model to use for benchmark tasks").action(async (opts) => {
-  if (!opts.candidate) {
-    throw new Error("--candidate is required");
-  }
-  if (opts.suite === "deep-swe") {
-    throw new Error("DeepSWE A/B not yet supported; use --suite synthetic");
-  }
-  const apiUrl = getApiUrl();
-  await waitForApi2(apiUrl);
-  const { tasks, suiteName } = await resolveTasks(opts);
-  if (tasks.length === 0) {
-    console.log("No benchmark tasks to run.");
-    return;
-  }
-  const timeoutMs = Number(opts.timeout);
-  const baselineSpec = await loadPromptSpec(opts.baseline);
-  const candidateSpec = await loadPromptSpec(opts.candidate);
-  console.log(`Baseline: ${baselineSpec.name}`);
-  console.log(`Candidate: ${candidateSpec.name}`);
-  const report = await runAbBenchmark(tasks, baselineSpec, candidateSpec, {
-    apiUrl,
-    suiteName,
-    timeoutMs,
-    provider: opts.provider,
-    model: opts.model,
-    onProgress: (result) => {
-      const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
-      console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
-    }
-  });
-  await fs8.mkdir(opts.outputDir, { recursive: true });
-  const ts = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-  const jsonFile = path10.join(opts.outputDir, `ab-${ts}.json`);
-  const mdFile = path10.join(opts.outputDir, `ab-${ts}.md`);
-  await fs8.writeFile(jsonFile, JSON.stringify(report, null, 2), "utf-8");
-  await fs8.writeFile(mdFile, formatAbReport(report), "utf-8");
-  console.log("\n" + formatAbReport(report));
-  console.log(`
-A/B report written to ${jsonFile}`);
-});
-var loopCmd = new Command8("loop").description("Run a self-improve benchmark loop").option("--suite <name>", "suite name: synthetic | deep-swe", "synthetic").option("--path <dir>", "path to DeepSWE tasks directory (for deep-swe suite)").option("--n-tasks <n>", "limit number of tasks", parseInt).option("--sample-seed <n>", "seed for deterministic sampling (for deep-swe)", parseInt).option("--iterations <n>", "max iterations", "1").option("--timeout <ms>", "per-task timeout in ms", "120000").option("--output-dir <dir>", "report output directory", ".omega/reports").option("--provider <name>", "provider to use for self-improve tasks").option("--model <model>", "model to use for self-improve tasks").action(async (opts) => {
-  const apiUrl = getApiUrl();
-  await waitForApi2(apiUrl);
-  const { tasks, suiteName } = await resolveTasks(opts);
-  if (tasks.length === 0) {
-    console.log("No benchmark tasks to run.");
-    return;
-  }
-  const timeoutMs = Number(opts.timeout);
-  const maxIterations = Number(opts.iterations);
-  const project = await currentProject(apiUrl);
-  for (let i = 1; i <= maxIterations; i++) {
-    console.log(`
-=== Iteration ${String(i)}/${String(maxIterations)} ===`);
-    const beforeVersion = await savePromptVersion(
-      apiUrl,
-      `loop-before-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
-    );
-    if (beforeVersion) {
-      console.log(`Saved before-loop prompt version: ${beforeVersion.name} (${beforeVersion.id})`);
-    }
-    console.log(`
-Running ${String(tasks.length)} benchmark tasks...`);
-    const report = await runBenchmark(tasks, {
-      apiUrl,
-      suiteName,
-      timeoutMs,
-      provider: opts.provider,
-      model: opts.model,
-      onProgress: (result) => {
-        const symbol = result.evaluation.passed ? "\u2713" : "\u2717";
-        console.log(`${symbol} ${result.task.name} [${result.status}] ${String(result.durationMs)}ms`);
-      }
-    });
-    await writeReport(report, opts.outputDir);
-    printSummary(report);
-    const passRate2 = report.total > 0 ? Math.round(report.passed / report.total * 100) : 0;
-    if (passRate2 === 100) {
-      console.log("\nAll tasks passed. Loop complete.");
-      break;
-    }
-    const context = await loadOptimisationContext(apiUrl, opts.outputDir);
-    if (!context) {
-      throw new Error("Benchmark report could not be loaded for optimisation");
-    }
-    const prompt = buildOptimisePrompt(context.report, context.failedResult, context.traceFlowText);
-    const task = await submitOptimiseTask(apiUrl, project.id, prompt);
-    if (opts.provider || opts.model) {
-      await fetch(`${apiUrl}/tasks/${task.id}`, {
-        method: "PATCH",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ provider: opts.provider, model: opts.model })
-      });
-    }
-    console.log(
-      `
-Created self-improve task ${task.id}${opts.provider ? ` (provider: ${opts.provider})` : ""}${opts.model ? ` (model: ${opts.model})` : ""}`
-    );
-    console.log("Running agent...");
-    await runTask(apiUrl, task.id);
-    const finished = await waitForTask(apiUrl, task.id, timeoutMs * 5);
-    console.log(`Agent finished: ${finished.status}`);
-    if (finished.error) {
-      console.error(`Agent error: ${finished.error}`);
-    }
-    const afterVersion = await savePromptVersion(
-      apiUrl,
-      `loop-after-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
-    );
-    if (afterVersion) {
-      console.log(`Saved after-loop prompt version: ${afterVersion.name} (${afterVersion.id})`);
-    }
-  }
-});
-var benchCmd = new Command8("bench").description("Run benchmarks and optimise prompts").addCommand(runCmd).addCommand(abCmd).addCommand(optimiseCmd).addCommand(loopCmd);
+var benchCmd = new Command8("bench").description("Run benchmarks and optimise prompts").addCommand(runCmd).addCommand(optimiseCmd);
 
 // ../../apps/cli/src/index.ts
-function getVersion() {
-  try {
-    const req = createRequire(import.meta.url);
-    const pkg = req("../package.json");
-    return pkg.version ?? "0.0.0";
-  } catch {
-    return "0.0.0";
-  }
-}
-program2.name("harness").description("Omega harness CLI").version(getVersion()).option("--api <url>", "API base URL", "http://localhost:4000");
+program2.name("harness").description("Omega harness CLI").version("0.1.0").option("--api <url>", "API base URL", "http://localhost:4000");
 program2.addCommand(projectCmd);
 program2.addCommand(taskCmd);
 program2.addCommand(uiCmd);