@wrongstack/bench 0.260.0

package/dist/index.js

@@ -0,0 +1,1050 @@
+import * as fs3 from 'fs/promises';
+import { spawn, spawnSync } from 'child_process';
+import { createHash } from 'crypto';
+import * as path from 'path';
+import * as os from 'os';
+import { resolveWstackPaths } from '@wrongstack/core';
+import { fileURLToPath } from 'url';
+
+// src/aggregate.ts
+function aggregateCell(cell, results) {
+  const taskCount = results.length;
+  if (taskCount === 0) {
+    return {
+      cell,
+      taskCount: 0,
+      gradedCount: 0,
+      passRate: 0,
+      editApplyRate: 1,
+      avgCostUsd: 0,
+      avgTokensIn: 0,
+      avgTokensOut: 0,
+      p50Iterations: 0,
+      p50ElapsedMs: 0,
+      timeoutRate: 0,
+      totalRateLimitRetries: 0
+    };
+  }
+  const graded = results.filter((r) => r.grade.graded !== false);
+  const passed = graded.filter((r) => r.grade.passed).length;
+  const timeouts = results.filter((r) => r.run.status === "timeout").length;
+  const editCalls = sum(results, (r) => r.tools.editCalls);
+  const editErrors = sum(results, (r) => r.tools.editErrors);
+  const editApplyRate = editCalls === 0 ? 1 : (editCalls - editErrors) / editCalls;
+  return {
+    cell,
+    taskCount,
+    gradedCount: graded.length,
+    passRate: graded.length === 0 ? 0 : passed / graded.length,
+    editApplyRate,
+    avgCostUsd: sum(results, (r) => r.run.costUsd) / taskCount,
+    avgTokensIn: sum(results, (r) => r.run.tokensIn) / taskCount,
+    avgTokensOut: sum(results, (r) => r.run.tokensOut) / taskCount,
+    p50Iterations: median(results.map((r) => r.run.iterations)),
+    p50ElapsedMs: median(results.map((r) => r.run.elapsedMs)),
+    timeoutRate: timeouts / taskCount,
+    totalRateLimitRetries: sum(results, (r) => r.tools.rateLimitRetries)
+  };
+}
+function aggregateAll(cells, results) {
+  return cells.map(
+    (cell) => aggregateCell(
+      cell,
+      results.filter((r) => r.cell.label === cell.label)
+    )
+  );
+}
+function median(values) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+function sum(items, pick) {
+  let total = 0;
+  for (const item of items) total += pick(item);
+  return total;
+}
+var DEFAULTS = {
+  maxIterations: 40,
+  concurrency: 4,
+  timeoutMs: 6e5
+};
+function parseBenchConfig(raw) {
+  if (typeof raw !== "object" || raw === null) {
+    throw new Error("bench config must be a JSON object");
+  }
+  const obj = raw;
+  const cellsRaw = obj["cells"];
+  if (!Array.isArray(cellsRaw) || cellsRaw.length === 0) {
+    throw new Error('bench config "cells" must be a non-empty array');
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const cells = cellsRaw.map((c, i) => {
+    if (typeof c !== "object" || c === null) {
+      throw new Error(`cells[${i}] must be an object`);
+    }
+    const cell = c;
+    const provider = cell["provider"];
+    const model = cell["model"];
+    if (typeof provider !== "string" || provider.length === 0) {
+      throw new Error(`cells[${i}].provider must be a non-empty string`);
+    }
+    if (typeof model !== "string" || model.length === 0) {
+      throw new Error(`cells[${i}].model must be a non-empty string`);
+    }
+    const label = typeof cell["label"] === "string" && cell["label"].length > 0 ? cell["label"] : `${provider}/${model}`;
+    if (seen.has(label)) {
+      throw new Error(`duplicate cell label "${label}" \u2014 labels must be unique`);
+    }
+    seen.add(label);
+    return { label, provider, model };
+  });
+  const maxIterations = positiveInt(obj["maxIterations"], DEFAULTS.maxIterations, "maxIterations");
+  const concurrency = positiveInt(obj["concurrency"], DEFAULTS.concurrency, "concurrency");
+  const timeoutMs = positiveInt(obj["timeoutMs"], DEFAULTS.timeoutMs, "timeoutMs");
+  return { maxIterations, concurrency, timeoutMs, cells };
+}
+async function loadBenchConfig(path7) {
+  let raw;
+  try {
+    raw = await fs3.readFile(path7, "utf8");
+  } catch (err) {
+    throw new Error(
+      `cannot read bench config at ${path7}: ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(
+      `bench config at ${path7} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+  return parseBenchConfig(parsed);
+}
+function positiveInt(value, fallback, name) {
+  if (value === void 0) return fallback;
+  if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
+    throw new Error(`${name} must be a positive number`);
+  }
+  return Math.floor(value);
+}
+function execCommand(opts) {
+  const useShell = opts.shell ?? true;
+  return new Promise((resolve) => {
+    let child;
+    try {
+      if (useShell) {
+        const line = [opts.command, ...opts.args.map(shellQuote)].join(" ");
+        child = spawn(line, {
+          cwd: opts.cwd,
+          env: { ...process.env, ...opts.env },
+          windowsHide: true,
+          shell: true
+        });
+      } else {
+        child = spawn(opts.command, opts.args, {
+          cwd: opts.cwd,
+          env: { ...process.env, ...opts.env },
+          windowsHide: true
+        });
+      }
+    } catch (err) {
+      resolve({
+        exitCode: null,
+        stdout: "",
+        stderr: err instanceof Error ? err.message : String(err),
+        timedOut: false
+      });
+      return;
+    }
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    let settled = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      try {
+        child.kill("SIGKILL");
+      } catch {
+      }
+    }, opts.timeoutMs);
+    child.stdout?.on("data", (d) => {
+      stdout += d.toString("utf8");
+    });
+    child.stderr?.on("data", (d) => {
+      stderr += d.toString("utf8");
+    });
+    const done = (exitCode) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve({ exitCode, stdout, stderr, timedOut });
+    };
+    child.on("error", (err) => {
+      stderr += `
+${err.message}`;
+      done(null);
+    });
+    child.on("close", (code) => done(code));
+  });
+}
+function shellQuote(arg) {
+  if (arg.length > 0 && !/\s/.test(arg)) return arg;
+  return `"${arg.replace(/"/g, '\\"')}"`;
+}
+function computeHarnessFingerprint(input) {
+  const toolNames = [...input.toolNames].sort((a, b) => a.localeCompare(b));
+  const canonical = JSON.stringify({
+    cliVersion: input.cliVersion,
+    toolNames,
+    maxIterations: input.maxIterations,
+    yolo: input.yolo,
+    subsetId: input.subsetId
+  });
+  const hash = createHash("sha256").update(canonical).digest("hex").slice(0, 12);
+  return {
+    cliVersion: input.cliVersion,
+    toolNames,
+    maxIterations: input.maxIterations,
+    yolo: input.yolo,
+    subsetId: input.subsetId,
+    hash
+  };
+}
+function fingerprintLabel(fp) {
+  const parts = [`wrongstack@${fp.cliVersion}`, `fp:${fp.hash}`, `maxIter=${fp.maxIterations}`];
+  if (fp.yolo) parts.push("yolo");
+  return parts.join(" \xB7 ");
+}
+
+// src/graders/polyglot-grader.ts
+async function gradePolyglot(opts) {
+  const meta = opts.task.meta;
+  if (meta.setupCommand) {
+    const setup = await execCommand({
+      command: meta.setupCommand.command,
+      args: meta.setupCommand.args,
+      cwd: opts.workdir,
+      timeoutMs: opts.timeoutMs
+    });
+    if (setup.exitCode !== 0) {
+      return {
+        passed: false,
+        detail: `setup failed (${meta.setupCommand.command}): ${tail(setup.stderr || setup.stdout)}`
+      };
+    }
+  }
+  const test = await execCommand({
+    command: meta.testCommand.command,
+    args: meta.testCommand.args,
+    cwd: opts.workdir,
+    timeoutMs: opts.timeoutMs
+  });
+  if (test.timedOut) {
+    return { passed: false, detail: "test command timed out" };
+  }
+  if (test.exitCode === 0) {
+    return { passed: true };
+  }
+  return { passed: false, detail: tail(test.stdout + "\n" + test.stderr) };
+}
+function tail(s) {
+  const clean = s.trim();
+  return clean.length > 500 ? `\u2026${clean.slice(-500)}` : clean;
+}
+async function writePredictionsJsonl(outDir, cellLabel, predictions) {
+  await fs3.mkdir(outDir, { recursive: true });
+  const file = path.join(outDir, `predictions-${slug(cellLabel)}.jsonl`);
+  const body = predictions.map((p) => JSON.stringify(p)).join("\n");
+  await fs3.writeFile(file, body + (body ? "\n" : ""), "utf8");
+  return file;
+}
+async function writeInstancePrediction(predictionsDir, cellLabel, prediction) {
+  const dir = path.join(predictionsDir, slug(cellLabel));
+  await fs3.mkdir(dir, { recursive: true });
+  await fs3.writeFile(
+    path.join(dir, `${slug(prediction.instance_id)}.json`),
+    JSON.stringify(prediction),
+    "utf8"
+  );
+}
+async function collectCellPredictions(predictionsDir, cellLabel) {
+  const dir = path.join(predictionsDir, slug(cellLabel));
+  let entries;
+  try {
+    entries = await fs3.readdir(dir);
+  } catch {
+    return [];
+  }
+  const out = [];
+  for (const name of entries.filter((e) => e.endsWith(".json")).sort()) {
+    try {
+      out.push(JSON.parse(await fs3.readFile(path.join(dir, name), "utf8")));
+    } catch {
+    }
+  }
+  return out;
+}
+function parseResolvedIds(reportJson) {
+  const resolved = /* @__PURE__ */ new Set();
+  if (typeof reportJson !== "object" || reportJson === null) return resolved;
+  const obj = reportJson;
+  if (Array.isArray(obj["resolved_ids"])) {
+    for (const id of obj["resolved_ids"]) if (typeof id === "string") resolved.add(id);
+    return resolved;
+  }
+  for (const [id, v] of Object.entries(obj)) {
+    if (v && typeof v === "object" && v["resolved"] === true) {
+      resolved.add(id);
+    }
+  }
+  return resolved;
+}
+function slug(s) {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "cell";
+}
+
+// src/suites/swebench-patch.ts
+async function extractModelPatch(opts) {
+  const exec = opts.exec ?? execCommand;
+  await exec({
+    command: "git",
+    args: ["add", "-A"],
+    cwd: opts.workdir,
+    timeoutMs: opts.timeoutMs,
+    shell: false
+  });
+  const diff = await exec({
+    command: "git",
+    args: ["diff", "--cached", "--no-color"],
+    cwd: opts.workdir,
+    timeoutMs: opts.timeoutMs,
+    shell: false
+  });
+  const raw = diff.stdout;
+  const testPaths = opts.testPatch ? extractPatchPaths(opts.testPatch) : /* @__PURE__ */ new Set();
+  return filterPatchSections(
+    raw,
+    (a, b) => testPaths.has(a) || testPaths.has(b) || isHarnessArtifact(a) || isHarnessArtifact(b)
+  );
+}
+function isHarnessArtifact(p) {
+  return p === ".gitignore" || p.split("/")[0] === ".wrongstack";
+}
+function extractPatchPaths(patch) {
+  const paths = /* @__PURE__ */ new Set();
+  for (const line of patch.split("\n")) {
+    const git = /^diff --git a\/(.+?) b\/(.+)$/.exec(line);
+    if (git) {
+      paths.add(git[1]);
+      paths.add(git[2]);
+      continue;
+    }
+    const minus = /^--- (?:a\/)?(.+)$/.exec(line);
+    if (minus && minus[1] !== "/dev/null") paths.add(stripTimestamp(minus[1]));
+    const plus = /^\+\+\+ (?:b\/)?(.+)$/.exec(line);
+    if (plus && plus[1] !== "/dev/null") paths.add(stripTimestamp(plus[1]));
+  }
+  return paths;
+}
+function filterPatchExcludingPaths(patch, exclude) {
+  if (exclude.size === 0) return patch;
+  return filterPatchSections(patch, (a, b) => exclude.has(a) || exclude.has(b));
+}
+function filterPatchSections(patch, shouldDrop) {
+  const lines = patch.split("\n");
+  const out = [];
+  let skipping = false;
+  for (const line of lines) {
+    const header = /^diff --git a\/(.+?) b\/(.+)$/.exec(line);
+    if (header) {
+      skipping = shouldDrop(header[1], header[2]);
+    }
+    if (!skipping) out.push(line);
+  }
+  return out.join("\n");
+}
+function stripTimestamp(p) {
+  const tab = p.indexOf("	");
+  return tab === -1 ? p : p.slice(0, tab);
+}
+
+// src/graders/swebench-grader.ts
+async function gradeSwebench(opts) {
+  const meta = opts.task.meta;
+  const patch = await extractModelPatch({
+    workdir: opts.workdir,
+    testPatch: meta.testPatch,
+    timeoutMs: opts.timeoutMs,
+    exec: opts.exec
+  });
+  await writeInstancePrediction(opts.predictionsDir, opts.cell.label, {
+    instance_id: meta.instanceId,
+    model_name_or_path: opts.cell.label,
+    model_patch: patch
+  });
+  if (patch.trim().length === 0) {
+    return { passed: false, graded: true, detail: "empty patch (agent made no edits)" };
+  }
+  if (opts.externalGrade) {
+    const verdict = await opts.externalGrade({
+      instanceId: meta.instanceId,
+      patch,
+      image: meta.image,
+      failToPass: meta.failToPass,
+      passToPass: meta.passToPass,
+      testPatch: meta.testPatch,
+      workdir: opts.workdir,
+      timeoutMs: opts.timeoutMs
+    });
+    if (verdict !== void 0) {
+      return { passed: verdict, graded: true };
+    }
+  }
+  return {
+    passed: false,
+    graded: false,
+    detail: `patch exported (${patch.length} bytes) \u2014 grade with the official SWE-bench harness`
+  };
+}
+async function createSandbox(opts) {
+  const base = opts.baseDir ?? await fs3.mkdtemp(path.join(os.tmpdir(), "wstack-bench-"));
+  await fs3.mkdir(base, { recursive: true });
+  const homeDir = path.join(base, "home");
+  const workRoot = path.join(base, "work");
+  await fs3.mkdir(homeDir, { recursive: true });
+  await fs3.mkdir(workRoot, { recursive: true });
+  const config = {
+    yolo: opts.yolo,
+    tools: { maxIterations: opts.maxIterations },
+    session: { auditLevel: "standard" }
+  };
+  await fs3.writeFile(path.join(homeDir, "config.json"), JSON.stringify(config, null, 2), "utf8");
+  return { root: base, homeDir, workRoot };
+}
+async function prepareWorkdir(sandbox, templateDir, taskId, cellLabel, exclude) {
+  const safe = `${slug2(cellLabel)}__${slug2(taskId)}`;
+  const dest = path.join(sandbox.workRoot, safe);
+  await fs3.rm(dest, { recursive: true, force: true });
+  const excludeSet = new Set(exclude ?? []);
+  await fs3.cp(templateDir, dest, {
+    recursive: true,
+    // Drop any path whose segments include an excluded name (e.g. `.meta`),
+    // so the reference solution never reaches the agent's workdir.
+    filter: excludeSet.size === 0 ? void 0 : (src) => !src.split(/[\\/]/).some((seg) => excludeSet.has(seg))
+  });
+  return dest;
+}
+async function cleanupSandbox(sandbox) {
+  await fs3.rm(sandbox.root, { recursive: true, force: true }).catch(() => void 0);
+}
+function slug2(s) {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "x";
+}
+async function runWstack(opts) {
+  const args = [
+    opts.wstackEntry,
+    "--prompt",
+    opts.prompt,
+    "--provider",
+    opts.cell.provider,
+    "--model",
+    opts.cell.model,
+    "--output-json",
+    "--no-tui",
+    "--no-interactive",
+    "--no-banner",
+    "--yolo",
+    "--no-models-refresh",
+    "--skip-index",
+    ...opts.extraArgs ?? []
+  ];
+  const startedAt = Date.now();
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn(opts.nodeBin, args, {
+        cwd: opts.workdir,
+        env: { ...process.env, ...opts.env, WRONGSTACK_HOME: opts.homeDir },
+        // windowsHide + no detached: per the repo spawn convention, detached on
+        // win32 voids CREATE_NO_WINDOW and pops visible consoles for grandchildren.
+        windowsHide: true
+      });
+    } catch (err) {
+      resolve(
+        crashed(startedAt, `spawn failed: ${err instanceof Error ? err.message : String(err)}`)
+      );
+      return;
+    }
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    let settled = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      treeKill(child);
+    }, opts.timeoutMs);
+    child.stdout?.on("data", (d) => {
+      stdout += d.toString("utf8");
+    });
+    child.stderr?.on("data", (d) => {
+      stderr += d.toString("utf8");
+    });
+    const finish = (run) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(run);
+    };
+    child.on("error", (err) => {
+      finish(crashed(startedAt, `process error: ${err.message}`));
+    });
+    child.on("close", (code) => {
+      const elapsedMs = Date.now() - startedAt;
+      if (timedOut) {
+        finish({
+          status: "timeout",
+          finalText: null,
+          iterations: 0,
+          tokensIn: 0,
+          tokensOut: 0,
+          costUsd: 0,
+          elapsedMs,
+          exitCode: code
+        });
+        return;
+      }
+      const parsed = parseOutputJson(stdout);
+      if (!parsed) {
+        finish({
+          status: "crashed",
+          finalText: null,
+          iterations: 0,
+          tokensIn: 0,
+          tokensOut: 0,
+          costUsd: 0,
+          elapsedMs,
+          exitCode: code
+          // stderr is intentionally not surfaced in RawRun; the caller can read
+          // the session log. Keep the shape minimal.
+        });
+        return;
+      }
+      finish({ ...parsed, elapsedMs, exitCode: code });
+    });
+  });
+}
+function parseOutputJson(stdout) {
+  const lines = stdout.split("\n");
+  for (let i = lines.length - 1; i >= 0; i--) {
+    const line = lines[i]?.trim();
+    if (!line?.startsWith("{")) continue;
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (typeof obj["status"] !== "string") continue;
+    const usage = obj["usage"] ?? {};
+    return {
+      status: normalizeStatus(obj["status"]),
+      finalText: typeof obj["finalText"] === "string" ? obj["finalText"] : null,
+      iterations: num(usage["iterations"]),
+      tokensIn: num(usage["input"]),
+      tokensOut: num(usage["output"]),
+      costUsd: num(usage["cost"])
+    };
+  }
+  return void 0;
+}
+function normalizeStatus(s) {
+  switch (s) {
+    case "completed":
+    case "failed":
+    case "aborted":
+    case "max_iterations":
+      return s;
+    default:
+      return "failed";
+  }
+}
+function num(v) {
+  return typeof v === "number" && Number.isFinite(v) ? v : 0;
+}
+function crashed(startedAt, _reason) {
+  return {
+    status: "crashed",
+    finalText: null,
+    iterations: 0,
+    tokensIn: 0,
+    tokensOut: 0,
+    costUsd: 0,
+    elapsedMs: Date.now() - startedAt,
+    exitCode: null
+  };
+}
+function treeKill(child) {
+  if (child.pid === void 0) return;
+  if (process.platform === "win32") {
+    spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { windowsHide: true });
+    return;
+  }
+  try {
+    child.kill("SIGTERM");
+  } catch {
+  }
+  setTimeout(() => {
+    try {
+      child.kill("SIGKILL");
+    } catch {
+    }
+  }, 2e3).unref();
+}
+async function mapWithConcurrency(items, concurrency, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const limit = Math.max(1, Math.min(concurrency, items.length || 1));
+  const workers = Array.from({ length: limit }, async () => {
+    while (true) {
+      const i = next++;
+      if (i >= items.length) return;
+      results[i] = await fn(items[i], i);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+var EDIT_TOOLS = /* @__PURE__ */ new Set([
+  "edit",
+  "write",
+  "multiedit",
+  "multi_edit",
+  "str_replace",
+  "apply_patch"
+]);
+async function readToolMetrics(opts) {
+  const empty = {
+    totalCalls: 0,
+    editCalls: 0,
+    editErrors: 0,
+    rateLimitRetries: 0
+  };
+  let jsonlPath;
+  try {
+    const sessionsDir = resolveWstackPaths({
+      projectRoot: opts.workdir,
+      globalRoot: opts.homeDir
+    }).projectSessions;
+    jsonlPath = await newestJsonl(sessionsDir);
+  } catch {
+    return empty;
+  }
+  if (!jsonlPath) return empty;
+  let raw;
+  try {
+    raw = await fs3.readFile(jsonlPath, "utf8");
+  } catch {
+    return empty;
+  }
+  const metrics = { ...empty };
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) continue;
+    let event;
+    try {
+      event = JSON.parse(trimmed);
+    } catch {
+      continue;
+    }
+    const type = event["type"];
+    if (type === "tool_call_end") {
+      metrics.totalCalls++;
+      const name = typeof event["name"] === "string" ? event["name"].toLowerCase() : "";
+      if (EDIT_TOOLS.has(name)) {
+        metrics.editCalls++;
+        if (event["ok"] === false) metrics.editErrors++;
+      }
+    } else if (type === "provider_retry" || type === "provider_error") {
+      metrics.rateLimitRetries++;
+    }
+  }
+  return metrics;
+}
+async function newestJsonl(dir) {
+  let entries;
+  try {
+    entries = await fs3.readdir(dir);
+  } catch {
+    return void 0;
+  }
+  const jsonls = entries.filter((e) => e.endsWith(".jsonl"));
+  if (jsonls.length === 0) return void 0;
+  let newest;
+  for (const name of jsonls) {
+    const full = path.join(dir, name);
+    try {
+      const stat2 = await fs3.stat(full);
+      if (!newest || stat2.mtimeMs > newest.mtime) {
+        newest = { path: full, mtime: stat2.mtimeMs };
+      }
+    } catch {
+    }
+  }
+  return newest?.path;
+}
+
+// src/orchestrate.ts
+async function runBenchmark(opts) {
+  const progress = opts.onProgress ?? (() => {
+  });
+  const nowFn = opts.now ?? (() => (/* @__PURE__ */ new Date()).toISOString());
+  const tasks = await opts.suite.loadTasks({ limit: opts.limit });
+  if (tasks.length === 0) {
+    throw new Error(`suite "${opts.suite.id}" produced no tasks (check the data directory)`);
+  }
+  const subsetId = opts.suite.subsetId(tasks);
+  const fingerprint = computeHarnessFingerprint({
+    cliVersion: opts.cliVersion,
+    toolNames: opts.toolNames,
+    maxIterations: opts.config.maxIterations,
+    yolo: true,
+    subsetId
+  });
+  progress(
+    `suite=${opts.suite.id} tasks=${tasks.length} cells=${opts.config.cells.length} fp=${fingerprint.hash}`
+  );
+  const sandbox = await createSandbox({
+    baseDir: opts.sandboxBaseDir,
+    maxIterations: opts.config.maxIterations,
+    yolo: true
+  });
+  const units = [];
+  for (const task of tasks) {
+    for (const cell of opts.config.cells) {
+      units.push({ task, cell });
+    }
+  }
+  try {
+    const results = await mapWithConcurrency(units, opts.config.concurrency, async (unit) => {
+      const { task, cell } = unit;
+      const workdir = await prepareWorkdir(
+        sandbox,
+        task.templateDir,
+        task.id,
+        cell.label,
+        task.templateExclude
+      );
+      const run = await runWstack({
+        nodeBin: opts.nodeBin,
+        wstackEntry: opts.wstackEntry,
+        homeDir: sandbox.homeDir,
+        workdir,
+        cell,
+        prompt: task.prompt,
+        timeoutMs: opts.config.timeoutMs,
+        env: opts.env
+      });
+      const tools = await readToolMetrics({ homeDir: sandbox.homeDir, workdir });
+      let grade;
+      try {
+        grade = await opts.grade({ workdir, task, cell, timeoutMs: opts.config.timeoutMs });
+      } catch (err) {
+        grade = {
+          passed: false,
+          detail: `grader error: ${err instanceof Error ? err.message : String(err)}`
+        };
+      }
+      progress(
+        `  ${cell.label} \xB7 ${task.id} \u2192 ${grade.passed ? "PASS" : "fail"} (${run.status}, ${run.iterations} it, $${run.costUsd.toFixed(3)})`
+      );
+      const result = { taskId: task.id, cell, run, grade, tools };
+      return result;
+    });
+    const cells = aggregateAll(opts.config.cells, results);
+    return {
+      suite: opts.suite.id,
+      finishedAt: nowFn(),
+      fingerprint,
+      cells,
+      results
+    };
+  } finally {
+    if (!opts.keepSandbox) await cleanupSandbox(sandbox);
+  }
+}
+async function writeJsonArtifacts(outDir, report) {
+  await fs3.mkdir(outDir, { recursive: true });
+  const jsonl = report.results.map((r) => JSON.stringify(r)).join("\n");
+  await fs3.writeFile(path.join(outDir, "results.jsonl"), jsonl + (jsonl ? "\n" : ""), "utf8");
+  const summary = {
+    suite: report.suite,
+    finishedAt: report.finishedAt,
+    fingerprint: report.fingerprint,
+    cells: report.cells
+  };
+  await fs3.writeFile(path.join(outDir, "summary.json"), JSON.stringify(summary, null, 2), "utf8");
+}
+async function readSummary(outDir) {
+  const raw = await fs3.readFile(path.join(outDir, "summary.json"), "utf8");
+  return JSON.parse(raw);
+}
+
+// src/report/markdown.ts
+function renderMarkdownReport(report) {
+  const { suite, finishedAt, fingerprint, cells } = report;
+  const lines = [];
+  lines.push(`# WrongStack benchmark \u2014 ${suite}`);
+  lines.push("");
+  lines.push(`**Harness:** ${fingerprintLabel(fingerprint)}`);
+  lines.push(`**Finished:** ${finishedAt}`);
+  lines.push(`**Tasks/cell:** ${cells[0]?.taskCount ?? 0}`);
+  lines.push("");
+  lines.push(
+    "Grading is deterministic (the suite's own tests decide pass/fail \u2014 no LLM judge). The only variable across rows is the model; everything else is fixed by the harness fingerprint."
+  );
+  lines.push("");
+  const sorted = [...cells].sort((a, b) => b.passRate - a.passRate);
+  lines.push(
+    "| Model | Pass@1 | Edit-apply | $/task | tok in/out | iters (p50) | wall (p50) | timeout | 429s |"
+  );
+  lines.push("|---|---:|---:|---:|---:|---:|---:|---:|---:|");
+  for (const c of sorted) {
+    lines.push(renderRow(c));
+  }
+  lines.push("");
+  lines.push(
+    `_Fingerprint hash: \`${fingerprint.hash}\` \xB7 tools: ${fingerprint.toolNames.length} \xB7 subset: \`${fingerprint.subsetId}\`_`
+  );
+  lines.push("");
+  return lines.join("\n");
+}
+function renderRow(c) {
+  const passCell = c.gradedCount === 0 ? "\u2014" : c.gradedCount < c.taskCount ? `${pct(c.passRate)} (${c.gradedCount}/${c.taskCount})` : pct(c.passRate);
+  return [
+    "",
+    c.cell.label,
+    passCell,
+    pct(c.editApplyRate),
+    `$${c.avgCostUsd.toFixed(3)}`,
+    `${fmtK(c.avgTokensIn)}/${fmtK(c.avgTokensOut)}`,
+    String(Math.round(c.p50Iterations)),
+    fmtMs(c.p50ElapsedMs),
+    pct(c.timeoutRate),
+    String(c.totalRateLimitRetries),
+    ""
+  ].join(" | ").trim();
+}
+function pct(x) {
+  return `${(x * 100).toFixed(1)}%`;
+}
+function fmtK(n) {
+  if (n >= 1e3) return `${(n / 1e3).toFixed(1)}k`;
+  return String(Math.round(n));
+}
+function fmtMs(ms) {
+  if (ms >= 1e3) return `${(ms / 1e3).toFixed(1)}s`;
+  return `${Math.round(ms)}ms`;
+}
+function reportHeaderLine(fp) {
+  return fingerprintLabel(fp);
+}
+var LANGUAGE_RUNNERS = {
+  python: {
+    dir: "python",
+    test: (tests) => ({ command: "python", args: ["-m", "pytest", "-q", ...tests] })
+  },
+  javascript: {
+    dir: "javascript",
+    setup: { command: "npm", args: ["install", "--no-audit", "--no-fund"] },
+    test: () => ({ command: "npm", args: ["test"] })
+  },
+  go: {
+    dir: "go",
+    test: () => ({ command: "go", args: ["test", "./..."] })
+  },
+  rust: {
+    dir: "rust",
+    test: () => ({ command: "cargo", args: ["test", "--", "--include-ignored"] })
+  },
+  cpp: {
+    dir: "cpp",
+    test: () => ({ command: "cmake", args: ["--build", "build", "--target", "test"] })
+  },
+  java: {
+    dir: "java",
+    test: () => ({ command: "./gradlew", args: ["test"] })
+  }
+};
+function createPolyglotSuite(opts) {
+  return {
+    id: "polyglot",
+    async loadTasks({ limit }) {
+      const tasks = [];
+      const langs = opts.languages ?? Object.keys(LANGUAGE_RUNNERS);
+      for (const lang of langs) {
+        const runner = LANGUAGE_RUNNERS[lang];
+        if (!runner) continue;
+        const practiceDir = path.join(opts.polyglotDir, runner.dir, "exercises", "practice");
+        let slugs;
+        try {
+          slugs = (await fs3.readdir(practiceDir, { withFileTypes: true })).filter((d) => d.isDirectory()).map((d) => d.name).sort((a, b) => a.localeCompare(b));
+        } catch {
+          continue;
+        }
+        for (const slug3 of slugs) {
+          const exerciseDir = path.join(practiceDir, slug3);
+          const task = await loadExercise(exerciseDir, lang, runner, slug3);
+          if (task) tasks.push(task);
+          if (limit !== void 0 && tasks.length >= limit) return tasks;
+        }
+      }
+      return tasks;
+    },
+    subsetId(tasks) {
+      const ids = tasks.map((t) => t.id).sort((a, b) => a.localeCompare(b));
+      return `polyglot:${createHash("sha256").update(ids.join("\n")).digest("hex").slice(0, 12)}`;
+    }
+  };
+}
+async function loadExercise(exerciseDir, language, runner, slug3) {
+  let manifest;
+  try {
+    const raw = await fs3.readFile(path.join(exerciseDir, ".meta", "config.json"), "utf8");
+    manifest = JSON.parse(raw);
+  } catch {
+    return void 0;
+  }
+  const solutionFiles = manifest.files?.solution ?? [];
+  const testFiles = manifest.files?.test ?? [];
+  if (solutionFiles.length === 0) return void 0;
+  const instructions = await readInstructions(exerciseDir);
+  const meta = {
+    language,
+    solutionFiles,
+    testFiles,
+    testCommand: runner.test(testFiles),
+    setupCommand: runner.setup
+  };
+  return {
+    id: `polyglot/${language}/${slug3}`,
+    suite: "polyglot",
+    prompt: buildPrompt(instructions, solutionFiles, testFiles),
+    templateDir: exerciseDir,
+    // Never copy the reference solution into the agent's workdir.
+    templateExclude: [".meta"],
+    meta
+  };
+}
+async function readInstructions(exerciseDir) {
+  const docs = path.join(exerciseDir, ".docs");
+  const parts = [];
+  for (const name of ["introduction.md", "instructions.md", "instructions.append.md"]) {
+    try {
+      parts.push((await fs3.readFile(path.join(docs, name), "utf8")).trim());
+    } catch {
+    }
+  }
+  return parts.filter(Boolean).join("\n\n");
+}
+function buildPrompt(instructions, solutionFiles, testFiles) {
+  return [
+    instructions,
+    "",
+    "---",
+    "",
+    `Implement the solution by editing **only** these file(s): ${solutionFiles.join(", ")}.`,
+    testFiles.length > 0 ? `The test suite (${testFiles.join(", ")}) is already present and will be run to grade your work \u2014 do not modify the tests.` : "A hidden test suite will be run to grade your work.",
+    "Make the tests pass. Use the available file tools to read and edit the files. When the implementation is complete, stop."
+  ].join("\n");
+}
+var SUBSET_FILE = "swe-bench-verified-50.json";
+async function resolveDefaultSubset() {
+  const candidates = [
+    fileURLToPath(new URL(`../subsets/${SUBSET_FILE}`, import.meta.url)),
+    fileURLToPath(new URL(`../../subsets/${SUBSET_FILE}`, import.meta.url))
+  ];
+  for (const c of candidates) {
+    try {
+      await fs3.access(c);
+      return c;
+    } catch {
+    }
+  }
+  return candidates[0];
+}
+async function loadSubset(subsetFile) {
+  const file = subsetFile ?? await resolveDefaultSubset();
+  const raw = await fs3.readFile(file, "utf8");
+  const parsed = JSON.parse(raw);
+  if (!Array.isArray(parsed.instances)) {
+    throw new Error(`subset file ${file} is missing an "instances" array`);
+  }
+  return parsed.instances.filter((x) => typeof x === "string");
+}
+function createSwebenchSuite(opts = {}) {
+  return {
+    id: "swebench",
+    async loadTasks({ limit }) {
+      const instanceIds = await loadSubset(opts.subsetFile);
+      if (!opts.datasetDir) {
+        throw new Error(
+          `SWE-bench requires a prepared dataset directory (materialized repos).
+Run with \`--dataset-dir <path>\`; add \`--docker\` to grade inline, or omit it to export predictions.jsonl for the official harness.
+The pinned subset (${instanceIds.length} instances) is committed in packages/bench/subsets/swe-bench-verified-50.json. See packages/bench/README.md for the dataset-preparation steps.`
+        );
+      }
+      const tasks = [];
+      for (const id of instanceIds) {
+        const instanceDir = path.join(opts.datasetDir, id);
+        let meta;
+        try {
+          meta = JSON.parse(await fs3.readFile(path.join(instanceDir, "instance.json"), "utf8"));
+        } catch {
+          continue;
+        }
+        const swMeta = {
+          instanceId: id,
+          instanceDir,
+          image: meta["image"],
+          failToPass: meta["FAIL_TO_PASS"] ?? [],
+          passToPass: meta["PASS_TO_PASS"] ?? [],
+          testPatch: meta["test_patch"]
+        };
+        tasks.push({
+          id: `swebench/${id}`,
+          suite: "swebench",
+          prompt: buildPrompt2(
+            typeof meta.problem_statement === "string" ? meta.problem_statement : ""
+          ),
+          templateDir: path.join(instanceDir, "repo"),
+          meta: swMeta
+        });
+        if (limit !== void 0 && tasks.length >= limit) break;
+      }
+      return tasks;
+    },
+    subsetId(tasks) {
+      const ids = tasks.map((t) => t.id).sort((a, b) => a.localeCompare(b));
+      return `swebench:${createHash("sha256").update(ids.join("\n")).digest("hex").slice(0, 12)}`;
+    }
+  };
+}
+function buildPrompt2(problemStatement) {
+  return [
+    problemStatement,
+    "",
+    "---",
+    "",
+    "You are working in a checkout of the repository. Resolve the issue above by editing the",
+    "source. Do not edit test files. Use the available tools to explore and modify the code.",
+    "When the fix is complete, stop."
+  ].join("\n");
+}
+
+export { LANGUAGE_RUNNERS, aggregateAll, aggregateCell, cleanupSandbox, collectCellPredictions, computeHarnessFingerprint, createPolyglotSuite, createSandbox, createSwebenchSuite, execCommand, extractModelPatch, extractPatchPaths, filterPatchExcludingPaths, filterPatchSections, fingerprintLabel, gradePolyglot, gradeSwebench, loadBenchConfig, loadSubset, mapWithConcurrency, median, parseBenchConfig, parseResolvedIds, prepareWorkdir, readSummary, readToolMetrics, renderMarkdownReport, reportHeaderLine, runBenchmark, runWstack, writeInstancePrediction, writeJsonArtifacts, writePredictionsJsonl };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map