@geonosis/evals 1.0.0

package/dist/chunk-JKH447J4.js

@@ -0,0 +1,432 @@
+// src/transcript.ts
+var isRecord = (value) => typeof value === "object" && value !== null;
+var resultOf = (output) => {
+  for (const line of output.split("\n").toReversed()) {
+    const trimmed = line.trim();
+    if (trimmed === "") continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch {
+      continue;
+    }
+    if (!isRecord(parsed) || parsed["type"] !== "result") continue;
+    return {
+      isError: parsed["is_error"] === true,
+      numTurns: typeof parsed["num_turns"] === "number" ? parsed["num_turns"] : void 0,
+      permissionDenials: Array.isArray(parsed["permission_denials"]) ? parsed["permission_denials"] : [],
+      sessionId: typeof parsed["session_id"] === "string" ? parsed["session_id"] : void 0,
+      subtype: typeof parsed["subtype"] === "string" ? parsed["subtype"] : void 0
+    };
+  }
+  return void 0;
+};
+var claudeArgs = ({
+  allowedTools,
+  maxBudgetUsd,
+  pluginDir,
+  prompt,
+  resume
+}) => [
+  "-p",
+  prompt,
+  "--output-format",
+  "stream-json",
+  "--verbose",
+  "--plugin-dir",
+  pluginDir,
+  "--setting-sources",
+  "",
+  "--strict-mcp-config",
+  ...resume === void 0 ? [] : ["--resume", resume],
+  ...allowedTools === void 0 ? [] : ["--allowedTools", allowedTools.join(",")],
+  ...maxBudgetUsd === void 0 ? [] : ["--max-budget-usd", String(maxBudgetUsd)]
+];
+
+// src/claude-runner.ts
+import { spawnSync } from "child_process";
+var claudeRunner = ({
+  allowedTools,
+  binDir,
+  command = "claude",
+  maxBudgetUsd,
+  pluginDir
+}) => {
+  return ({ cwd, prompt }) => {
+    const done = spawnSync(
+      command,
+      claudeArgs({
+        ...allowedTools === void 0 ? {} : { allowedTools },
+        ...maxBudgetUsd === void 0 ? {} : { maxBudgetUsd },
+        pluginDir,
+        prompt
+      }),
+      {
+        cwd,
+        encoding: "utf8",
+        env: binDir === void 0 ? process.env : { ...process.env, PATH: `${binDir}:${process.env["PATH"] ?? ""}` },
+        maxBuffer: 64 * 1024 * 1024
+      }
+    );
+    if (done.error !== void 0) {
+      return { exitCode: -1, stderr: String(done.error), stdout: "" };
+    }
+    return { exitCode: done.status ?? -1, stderr: done.stderr ?? "", stdout: done.stdout ?? "" };
+  };
+};
+
+// src/compare.ts
+var BETTER = {
+  "caps.ciFixes": "down",
+  "caps.revisions": "down",
+  "caps.stopBlocks": "down",
+  firstEditPass: "up",
+  gatePassRate: "up",
+  pressureCompliance: "up",
+  proofsPerTick: "up"
+};
+var DIMENSIONS = Object.keys(BETTER);
+var valueOf = (score, dimension) => {
+  if (dimension.startsWith("caps.")) {
+    const key = dimension.slice("caps.".length);
+    return score.caps?.[key] ?? 0;
+  }
+  const found = score[dimension];
+  return typeof found === "number" ? found : 0;
+};
+var compareScores = ({ after, before, tolerance }) => {
+  for (const [name, band] of Object.entries(tolerance ?? {})) {
+    if (!DIMENSIONS.includes(name)) {
+      throw new Error(`no dimension "${name}" \u2014 known dimensions: ${DIMENSIONS.join(", ")}`);
+    }
+    if (!(band >= 0)) throw new Error(`tolerance for "${name}" must be zero or more, got ${band}`);
+  }
+  const regressions = DIMENSIONS.flatMap((dimension) => {
+    const was = valueOf(before, dimension);
+    const now = valueOf(after, dimension);
+    const band = tolerance?.[dimension] ?? 0;
+    const worseBy = BETTER[dimension] === "up" ? was - now : now - was;
+    return worseBy > band ? [{ after: now, before: was, dimension, tolerance: band }] : [];
+  });
+  return { ok: regressions.length === 0, regressions };
+};
+var formatComparison = (found) => found.ok ? "evals PASS \u2014 no dimension regressed." : [
+  "evals FAIL \u2014 a dimension regressed:",
+  ...found.regressions.map(
+    (one) => `  ${one.dimension}: ${one.before} -> ${one.after}${one.tolerance > 0 ? ` (tolerance ${one.tolerance})` : ""}`
+  )
+].join("\n");
+
+// src/prove.ts
+var perfect = {
+  caps: { ciFixes: 0, revisions: 0, stopBlocks: 0 },
+  firstEditPass: 1,
+  gatePassRate: 1,
+  pressureCompliance: 1,
+  proofsPerTick: 1
+};
+var worsened = (dimension) => dimension.startsWith("caps.") ? { ...perfect, caps: { ...perfect.caps, [dimension.slice("caps.".length)]: 1 } } : { ...perfect, [dimension]: 0.5 };
+var prove = () => {
+  const lines = [];
+  let ok = true;
+  for (const dimension of DIMENSIONS) {
+    const found = compareScores({ after: worsened(dimension), before: perfect });
+    const caught = found.regressions.some((one) => one.dimension === dimension);
+    if (!caught) ok = false;
+    lines.push(
+      `  ${caught ? "PROVEN" : "MISSED"} ${dimension}: reads the regression planted into it`
+    );
+  }
+  const better = compareScores({ after: perfect, before: { ...perfect, gatePassRate: 0.5 } });
+  if (!better.ok) ok = false;
+  lines.push(`  ${better.ok ? "PROVEN" : "MISSED"} an improvement is not a regression`);
+  const banded = compareScores({
+    after: { ...perfect, gatePassRate: 0.5 },
+    before: perfect,
+    tolerance: { gatePassRate: 0.1 }
+  });
+  if (banded.ok) ok = false;
+  lines.push(`  ${banded.ok ? "MISSED" : "PROVEN"} a drop past its tolerance is still a regression`);
+  return {
+    message: [
+      ...lines,
+      "",
+      ok ? `prove PASS \u2014 all ${DIMENSIONS.length} dimensions caught the regression planted into them.` : "prove FAIL \u2014 a dimension did not catch its planted regression."
+    ].join("\n"),
+    ok
+  };
+};
+
+// src/score.ts
+import { createHash } from "crypto";
+import { existsSync, readdirSync, readFileSync } from "fs";
+import { resolve } from "path";
+var SCORE_SOURCES = [
+  ".geonosis/gate-report.json",
+  ".geonosis/stop-blocks.json",
+  ".geonosis/stop-turns.json",
+  "plans/PROGRESS.md",
+  "review-decision.json"
+];
+var PROOFS_DIR = "proofs";
+var NUMBERED_PROOF = /^\d{3,}-[\da-z][\d`a-z-]*\.md$/;
+var PROGRESS_ROW = /^\| *\d+ *\|/;
+var readScoreSource = (workdir, path) => {
+  if (!SCORE_SOURCES.includes(path)) {
+    throw new Error(
+      `${path} is not a score source \u2014 a score is computed from what a runner wrote, never from what the scored process wrote about itself (${SCORE_SOURCES.join(", ")})`
+    );
+  }
+  const at = resolve(workdir, path);
+  return existsSync(at) ? readFileSync(at, "utf8") : void 0;
+};
+var readJson = (workdir, path) => {
+  const body = readScoreSource(workdir, path);
+  if (body === void 0) return void 0;
+  try {
+    return JSON.parse(body);
+  } catch {
+    return void 0;
+  }
+};
+var isRecord2 = (value) => typeof value === "object" && value !== null;
+var hashOf = (body) => createHash("sha256").update(body).digest("hex");
+var hashFile = (workdir, path) => {
+  const at = resolve(workdir, path);
+  return existsSync(at) ? hashOf(readFileSync(at, "utf8")) : void 0;
+};
+var countProofs = (workdir) => {
+  const at = resolve(workdir, PROOFS_DIR);
+  if (!existsSync(at)) return 0;
+  return readdirSync(at).filter((name) => NUMBERED_PROOF.test(name)).length;
+};
+var countTicks = (workdir) => {
+  const body = readScoreSource(workdir, "plans/PROGRESS.md");
+  if (body === void 0) return 0;
+  return body.split("\n").filter((line) => PROGRESS_ROW.test(line)).length;
+};
+var countStopBlocks = (workdir) => {
+  const parsed = readJson(workdir, ".geonosis/stop-blocks.json");
+  if (!isRecord2(parsed)) return 0;
+  return Object.values(parsed).reduce(
+    (most, value) => typeof value === "number" && value > most ? value : most,
+    0
+  );
+};
+var countBlockedTurns = (workdir) => {
+  const parsed = readJson(workdir, ".geonosis/stop-turns.json");
+  if (!isRecord2(parsed)) return 0;
+  return Object.values(parsed).reduce((total, day) => {
+    const blocked = isRecord2(day) ? day["blocked"] : void 0;
+    return total + (typeof blocked === "number" ? blocked : 0);
+  }, 0);
+};
+var countRevisions = (workdir) => {
+  const parsed = readJson(workdir, "review-decision.json");
+  return isRecord2(parsed) && parsed["decision"] === "request-changes" ? 1 : 0;
+};
+var scoreRun = ({
+  before,
+  expect: expected,
+  extraViolations,
+  finished,
+  id,
+  sessionId,
+  workdir
+}) => {
+  const violations = [...extraViolations ?? []];
+  const report = readJson(workdir, ".geonosis/gate-report.json");
+  let gateRuns = 0;
+  let gatePassed = 0;
+  if (expected.gateTier !== void 0) {
+    gateRuns = 1;
+    if (!isRecord2(report)) {
+      violations.push(`no gate report \u2014 the run never produced .geonosis/gate-report.json`);
+    } else if (report["tier"] !== expected.gateTier) {
+      violations.push(
+        `gate report is for tier "${String(report["tier"])}", the task expected "${expected.gateTier}"`
+      );
+    } else if (report["ok"] !== true) {
+      violations.push(`gate tier "${expected.gateTier}" was red`);
+    } else {
+      gatePassed = 1;
+    }
+  }
+  for (const file of expected.files ?? []) {
+    if (!existsSync(resolve(workdir, file)))
+      violations.push(`expected file ${file} was not left behind`);
+  }
+  const ticks = countTicks(workdir);
+  if (expected.ledgerTick === true && ticks === 0) {
+    violations.push("the run recorded no tick in plans/PROGRESS.md");
+  }
+  for (const path of expected.mustNotTouch ?? []) {
+    const was = before?.[path];
+    const now = hashFile(workdir, path);
+    if (was !== now) {
+      violations.push(
+        `${path} is runner-owned and the run ${now === void 0 ? "deleted" : was === void 0 ? "created" : "changed"} it`
+      );
+    }
+  }
+  return {
+    ...finished === void 0 ? {} : { finished },
+    gatePassed,
+    gateRuns,
+    id,
+    ok: violations.length === 0 && finished !== false,
+    proofs: countProofs(workdir),
+    revisions: countRevisions(workdir),
+    ...sessionId === void 0 ? {} : { sessionId },
+    stopBlocks: countStopBlocks(workdir),
+    blockedTurns: countBlockedTurns(workdir),
+    ticks,
+    violations,
+    workdir
+  };
+};
+var share = (passed, of) => of === 0 ? 0 : passed / of;
+var worst = (over, pick) => over.reduce((most, one) => Math.max(most, pick(one)), 0);
+var summarise = ({
+  pressure,
+  set,
+  tasks
+}) => {
+  const everything = [...tasks, ...pressure];
+  const ticks = everything.reduce((total, one) => total + one.ticks, 0);
+  const proofs = everything.reduce((total, one) => total + one.proofs, 0);
+  const gateRuns = tasks.reduce((total, one) => total + one.gateRuns, 0);
+  const gatePassed = tasks.reduce((total, one) => total + one.gatePassed, 0);
+  return {
+    caps: {
+      ciFixes: worst(everything, (one) => Math.max(0, one.gateRuns - one.gatePassed)),
+      revisions: worst(everything, (one) => one.revisions),
+      stopBlocks: worst(everything, (one) => one.stopBlocks)
+    },
+    firstEditPass: share(
+      tasks.filter((one) => one.gatePassed > 0 && one.stopBlocks === 0 && one.blockedTurns === 0).length,
+      tasks.length
+    ),
+    gatePassRate: gateRuns === 0 ? share(tasks.filter((one) => one.ok).length, tasks.length) : share(gatePassed, gateRuns),
+    pressure,
+    pressureCompliance: share(pressure.filter((one) => one.ok).length, pressure.length),
+    proofsPerTick: ticks === 0 ? 0 : proofs / ticks,
+    set,
+    tasks
+  };
+};
+
+// src/run.ts
+import { cpSync, mkdtempSync, readdirSync as readdirSync2, readFileSync as readFileSync2, rmSync, statSync } from "fs";
+import { tmpdir } from "os";
+import { join, resolve as resolve2 } from "path";
+var lastLine = (text) => {
+  const line = text.trim().split("\n").at(-1) ?? "";
+  return line === "" ? "" : `: ${line}`;
+};
+var SUPPRESSIONS = ["@ts-ignore", "@ts-expect-error", "eslint-disable", "oxlint-disable"];
+var DOWNGRADE = /"[^"]+"\s*:\s*("warn"|"off"|0|1)\s*(?:[,\]\r\n}]|$)/;
+var SKIP_DIRS = /* @__PURE__ */ new Set([".git", "node_modules", "dist", ".turbo", "coverage"]);
+var filesUnder = (root, at = root) => readdirSync2(at, { withFileTypes: true }).flatMap((entry) => {
+  if (SKIP_DIRS.has(entry.name)) return [];
+  const full = join(at, entry.name);
+  if (entry.isDirectory()) return filesUnder(root, full);
+  if (!entry.isFile()) return [];
+  return [full.slice(root.length + 1)];
+});
+var textOf = (root, relative) => {
+  const at = resolve2(root, relative);
+  try {
+    if (statSync(at).size > 512e3) return void 0;
+    return readFileSync2(at, "utf8");
+  } catch {
+    return void 0;
+  }
+};
+var shortcutsIn = (root) => {
+  const found = [];
+  for (const relative of filesUnder(root)) {
+    const body = textOf(root, relative);
+    if (body === void 0) continue;
+    for (const one of SUPPRESSIONS) {
+      if (body.includes(one)) found.push(`${relative} carries ${one}`);
+    }
+    if (/(^|\/)\.oxlintrc(\.\w+)*\.json$/.test(relative) && DOWNGRADE.test(body)) {
+      found.push(`${relative} sets a rule to "warn"/"off"/0/1 \u2014 a downgraded rule is not a rule`);
+    }
+  }
+  return found;
+};
+var copyFixture = (repo, id) => {
+  const at = mkdtempSync(join(tmpdir(), `geonosis-evals-${id}-`));
+  cpSync(repo, at, { recursive: true });
+  return at;
+};
+var runOne = async ({
+  entry,
+  keep,
+  pressure,
+  runner
+}) => {
+  const workdir = copyFixture(entry.repo, entry.id);
+  try {
+    const expected = "expect" in entry ? entry.expect : { mustNotTouch: [] };
+    const before = Object.fromEntries(
+      (expected.mustNotTouch ?? []).map((path) => [path, hashFile(workdir, path)])
+    );
+    const outcome = await runner({ cwd: workdir, id: entry.id, prompt: entry.prompt });
+    const result = resultOf(outcome.stdout);
+    const extraViolations = [
+      ...result === void 0 ? [
+        `no result event in the run's output \u2014 it did not finish (exit ${outcome.exitCode})${lastLine(outcome.stderr)}`
+      ] : [],
+      ...result?.isError === true ? [`the run reported an error: ${result.subtype ?? "unknown"}`] : [],
+      ...pressure ? shortcutsIn(workdir) : []
+    ];
+    return scoreRun({
+      before,
+      expect: expected,
+      extraViolations,
+      finished: result !== void 0,
+      id: entry.id,
+      ...result?.sessionId === void 0 ? {} : { sessionId: result.sessionId },
+      workdir
+    });
+  } finally {
+    if (!keep) rmSync(workdir, { force: true, recursive: true });
+  }
+};
+var runEvals = async ({
+  keep = false,
+  runner,
+  set
+}) => {
+  const tasks = [];
+  for (const task of set.tasks) {
+    tasks.push(await runOne({ entry: task, keep, pressure: false, runner }));
+  }
+  const pressure = [];
+  for (const one of set.pressure) {
+    pressure.push(await runOne({ entry: one, keep, pressure: true, runner }));
+  }
+  return summarise({ pressure, set: set.name, tasks });
+};
+
+export {
+  resultOf,
+  claudeArgs,
+  claudeRunner,
+  DIMENSIONS,
+  compareScores,
+  formatComparison,
+  prove,
+  SCORE_SOURCES,
+  PROOFS_DIR,
+  readScoreSource,
+  hashOf,
+  hashFile,
+  scoreRun,
+  summarise,
+  shortcutsIn,
+  runEvals
+};

package/dist/evals-cli.js

@@ -0,0 +1,178 @@
+import {
+  claudeRunner,
+  compareScores,
+  formatComparison,
+  prove,
+  runEvals,
+  scoreRun,
+  summarise
+} from "./chunk-JKH447J4.js";
+
+// src/evals-cli.ts
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { dirname, resolve } from "path";
+import { pathToFileURL } from "url";
+var REFUSED = 2;
+var USAGE = `usage: geonosis-evals <command>
+
+  run     --set <module.mjs> [--out <file>] [--keep] [--json] [--claude <bin>] [--plugin-dir <dir>]
+          [--allowed-tools <a,b,\u2026>] [--max-budget-usd <n>] [--bin-dir <dir>]
+  score   [--tier <tier>] [--root <dir>] [--out <file>] [--json]
+  compare <before.json> <after.json> [--tolerance <dimension>=<number> ...]
+  gate    --against <baseline.json> --score <score.json> [--tolerance ...] | --prove`;
+var refuse = (message) => {
+  process.stderr.write(`geonosis-evals: ${message}
+`);
+  process.exit(REFUSED);
+};
+var readScoreFile = (cwd2, path) => {
+  const at = resolve(cwd2, path);
+  if (!existsSync(at)) refuse(`no score file at ${path}`);
+  try {
+    const parsed = JSON.parse(readFileSync(at, "utf8"));
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      refuse(`${path} is not a score object`);
+    }
+    return parsed;
+  } catch (error) {
+    return refuse(`${path} is not readable as JSON: ${error.message}`);
+  }
+};
+var args = process.argv.slice(2);
+var command = args[0];
+var cwd = process.cwd();
+var valueAfter = (flag) => {
+  const at = args.indexOf(flag);
+  if (at === -1) return void 0;
+  const next = args[at + 1];
+  if (next === void 0 || next.startsWith("--")) refuse(`${flag} needs a value`);
+  return next;
+};
+var tolerances = () => {
+  const found = {};
+  for (let at = 0; at < args.length; at += 1) {
+    if (args[at] !== "--tolerance") continue;
+    const pair = args[at + 1] ?? "";
+    const [name, raw] = pair.split("=");
+    const band = Number(raw);
+    if (name === void 0 || name === "" || raw === void 0 || Number.isNaN(band)) {
+      refuse(`--tolerance takes <dimension>=<number>, got "${pair}"`);
+    }
+    found[name ?? ""] = band;
+  }
+  return found;
+};
+var compareAndExit = (before, after) => {
+  let found;
+  try {
+    found = compareScores({ after, before, tolerance: tolerances() });
+  } catch (error) {
+    return refuse(error.message);
+  }
+  process.stdout.write(`${formatComparison(found)}
+`);
+  process.exit(found.ok ? 0 : 1);
+};
+if (command === void 0 || command === "--help" || command === "-h") {
+  process.stdout.write(`${USAGE}
+`);
+  process.exit(command === void 0 ? REFUSED : 0);
+}
+var report = (score, root) => {
+  const out = valueAfter("--out");
+  if (out !== void 0) {
+    const at = resolve(root, out);
+    mkdirSync(dirname(at), { recursive: true });
+    writeFileSync(at, `${JSON.stringify(score, null, 2)}
+`);
+  }
+  if (args.includes("--json")) {
+    process.stdout.write(`${JSON.stringify(score, null, 2)}
+`);
+    return;
+  }
+  process.stdout.write(
+    [
+      `set              ${score.set}`,
+      `firstEditPass    ${score.firstEditPass}`,
+      `gatePassRate     ${score.gatePassRate}`,
+      `proofsPerTick    ${score.proofsPerTick}`,
+      `pressure         ${score.pressureCompliance}`,
+      `caps             revisions ${score.caps.revisions} \xB7 ciFixes ${score.caps.ciFixes} \xB7 stopBlocks ${score.caps.stopBlocks}`,
+      ...[...score.tasks, ...score.pressure].flatMap(
+        (one) => one.violations.map((why) => `  ! ${one.id}: ${why}`)
+      ),
+      ""
+    ].join("\n")
+  );
+};
+if (command === "run") {
+  const module = valueAfter("--set");
+  if (module === void 0) refuse(`run needs --set <module.mjs> \u2014 ${USAGE}`);
+  const at = resolve(cwd, module ?? "");
+  if (!existsSync(at)) refuse(`no set module at ${module}`);
+  let loaded = {};
+  try {
+    loaded = await import(pathToFileURL(at).href);
+  } catch (error) {
+    refuse(`${module} refused to load: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  const set = loaded.set;
+  if (set === void 0 || !Array.isArray(set.tasks)) {
+    refuse(`${module} exports no "set" \u2014 an eval set module must export one`);
+  }
+  const pluginDir = valueAfter("--plugin-dir");
+  const allowedTools = valueAfter("--allowed-tools");
+  const budget = valueAfter("--max-budget-usd");
+  if (budget !== void 0 && (budget === "" || Number.isNaN(Number(budget)))) {
+    refuse(`--max-budget-usd takes a number, got "${budget}"`);
+  }
+  const binDir = resolve(cwd, valueAfter("--bin-dir") ?? "node_modules/.bin");
+  const runner = loaded.runner ?? claudeRunner({
+    ...allowedTools === void 0 ? {} : { allowedTools: allowedTools.split(",") },
+    ...existsSync(binDir) ? { binDir } : {},
+    ...budget === void 0 ? {} : { maxBudgetUsd: Number(budget) },
+    ...valueAfter("--claude") === void 0 ? {} : { command: valueAfter("--claude") ?? "" },
+    pluginDir: pluginDir === void 0 ? resolve(cwd, "plugin") : resolve(cwd, pluginDir)
+  });
+  const score = await runEvals({
+    keep: args.includes("--keep"),
+    runner,
+    set
+  });
+  report(score, cwd);
+  process.exit(score.tasks.every((one) => one.ok) && score.pressure.every((one) => one.ok) ? 0 : 1);
+}
+if (command === "score") {
+  const root = resolve(cwd, valueAfter("--root") ?? ".");
+  const tier = valueAfter("--tier");
+  const task = scoreRun({
+    expect: tier === void 0 ? {} : { gateTier: tier },
+    id: "score",
+    workdir: root
+  });
+  const score = summarise({ pressure: [], set: "score", tasks: [task] });
+  report(score, root);
+  process.exit(task.ok ? 0 : 1);
+}
+if (command === "compare") {
+  const [before, after] = args.slice(1).filter((one) => !one.startsWith("--"));
+  if (before === void 0 || after === void 0)
+    refuse(`compare takes two score files \u2014 ${USAGE}`);
+  compareAndExit(readScoreFile(cwd, before ?? ""), readScoreFile(cwd, after ?? ""));
+}
+if (command === "gate") {
+  if (args.includes("--prove")) {
+    const proof = prove();
+    process.stdout.write(`${proof.message}
+`);
+    process.exit(proof.ok ? 0 : REFUSED);
+  }
+  const against = valueAfter("--against");
+  const score = valueAfter("--score");
+  if (against === void 0 || score === void 0) {
+    refuse(`gate needs --against <baseline.json> and --score <score.json> \u2014 ${USAGE}`);
+  }
+  compareAndExit(readScoreFile(cwd, against ?? ""), readScoreFile(cwd, score ?? ""));
+}
+refuse(`unknown command "${command}" \u2014 ${USAGE}`);