@namzu/cli 34.1.1 → 36.0.0

package/dist/chunks/eval-BMJQXDKA.js

@@ -0,0 +1,187 @@
+import { createRequire as __namzuCreateRequire } from 'node:module';
+const require = __namzuCreateRequire(import.meta.url);
+import {
+  Command,
+  CommanderError
+} from "./chunk-MZP6SH3M.js";
+import {
+  formatReport
+} from "./chunk-K5HARZ7G.js";
+import {
+  __name
+} from "./chunk-76FLAFCG.js";
+
+// src/commands/eval.ts
+import { mkdir, readdir, writeFile } from "node:fs/promises";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+var EVAL_EXIT = {
+  /** Every case passed. */
+  ok: 0,
+  /** At least one case failed. A real regression signal. */
+  failed: 1,
+  /** At least one case was inconclusive — the harness could not judge. */
+  inconclusive: 2,
+  /** No suite was found, or one could not be loaded. */
+  usage: 3
+};
+var SUITE_SUFFIX = ".eval.js";
+var DEFAULT_SUITE_TIMEOUT_MS = 3e5;
+var TIMED_OUT = /* @__PURE__ */ Symbol("eval.suite.timed-out");
+async function withDeadline(work, ms) {
+  let timer;
+  try {
+    return await Promise.race([
+      work,
+      new Promise((resolve2) => {
+        timer = setTimeout(() => resolve2(TIMED_OUT), ms);
+      })
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+__name(withDeadline, "withDeadline");
+async function discoverSuites(root) {
+  const found = [];
+  async function walk(dir) {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true, encoding: "utf-8" });
+    } catch {
+      return;
+    }
+    for (const entry of entries.sort((a, b) => a.name < b.name ? -1 : 1)) {
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (entry.name === "node_modules" || entry.name.startsWith(".")) continue;
+        await walk(full);
+        continue;
+      }
+      if (!entry.name.endsWith(SUITE_SUFFIX)) continue;
+      found.push({ id: suiteId(root, full), file: full });
+    }
+  }
+  __name(walk, "walk");
+  await walk(root);
+  const seen = /* @__PURE__ */ new Map();
+  for (const suite of found) {
+    const previous = seen.get(suite.id);
+    if (previous !== void 0) {
+      throw new Error(
+        `Two eval suites resolve to the id "${suite.id}": ${previous} and ${suite.file}. Ids are path-derived so artifacts from two commits can be diffed; with a duplicate, a report describes whichever ran last.`
+      );
+    }
+    seen.set(suite.id, suite.file);
+  }
+  return found;
+}
+__name(discoverSuites, "discoverSuites");
+function suiteId(root, file) {
+  return relative(root, file).slice(0, -SUITE_SUFFIX.length).split(sep).join("/");
+}
+__name(suiteId, "suiteId");
+async function loadSuite(file) {
+  const mod = await import(pathToFileURL(file).href);
+  if (typeof mod.default !== "function") {
+    throw new Error(
+      `Eval suite "${file}" must default-export a function returning an ExperimentReport (call \`runExperiment\` inside it).`
+    );
+  }
+  return mod;
+}
+__name(loadSuite, "loadSuite");
+async function runEval(ctx, options) {
+  const root = isAbsolute(options.dir) ? options.dir : resolve(process.cwd(), options.dir);
+  let suites;
+  try {
+    suites = await discoverSuites(root);
+  } catch (err) {
+    ctx.formatter.error({ message: err instanceof Error ? err.message : String(err) });
+    return EVAL_EXIT.usage;
+  }
+  if (suites.length === 0) {
+    ctx.formatter.error({
+      message: `No eval suites found under ${root}. A suite is a file named *${SUITE_SUFFIX} that default-exports a function returning an ExperimentReport.`
+    });
+    return EVAL_EXIT.usage;
+  }
+  const reports = [];
+  let skipped = 0;
+  for (const suite of suites) {
+    let mod;
+    try {
+      mod = await loadSuite(suite.file);
+    } catch (err) {
+      ctx.formatter.error({ message: err instanceof Error ? err.message : String(err) });
+      return EVAL_EXIT.usage;
+    }
+    if (options.tag && !(mod.tags ?? []).includes(options.tag)) {
+      skipped++;
+      continue;
+    }
+    const report = await withDeadline(
+      mod.default(),
+      options.timeoutMs ?? DEFAULT_SUITE_TIMEOUT_MS
+    );
+    if (report === TIMED_OUT) {
+      ctx.formatter.error({
+        message: `Eval suite "${suite.id}" did not finish within ${options.timeoutMs ?? DEFAULT_SUITE_TIMEOUT_MS}ms. A suite that hangs judges nothing; raise --timeout-ms if it is genuinely slow, and look for a turn parked on an approval nobody is there to give if it is not.`
+      });
+      return EVAL_EXIT.inconclusive;
+    }
+    reports.push({ suite: suite.id, report });
+    ctx.formatter.print({ suite: suite.id, report, text: formatReport(report) });
+  }
+  if (skipped > 0) {
+    ctx.formatter.info(`${skipped} suite(s) skipped by --tag ${options.tag}`);
+  }
+  if (reports.length === 0) {
+    ctx.formatter.error({ message: `No suite matched --tag ${options.tag}.` });
+    return EVAL_EXIT.usage;
+  }
+  if (options.out) {
+    await writeArtifact(options.out, reports);
+    ctx.formatter.info(`Wrote ${options.out}`);
+  }
+  const failed = reports.reduce((sum, r) => sum + r.report.failed, 0);
+  const inconclusive = reports.reduce((sum, r) => sum + r.report.inconclusive, 0);
+  if (inconclusive > 0) return EVAL_EXIT.inconclusive;
+  if (failed > 0) return EVAL_EXIT.failed;
+  return EVAL_EXIT.ok;
+}
+__name(runEval, "runEval");
+async function writeArtifact(out, reports) {
+  const path = isAbsolute(out) ? out : resolve(process.cwd(), out);
+  await mkdir(join(path, ".."), { recursive: true });
+  await writeFile(path, `${JSON.stringify({ suites: reports }, null, 2)}
+`, "utf-8");
+}
+__name(writeArtifact, "writeArtifact");
+var evalCommand = {
+  name: "eval",
+  description: "Run eval suites and set an exit code",
+  passThrough: true,
+  handler: /* @__PURE__ */ __name(async ({ ctx, rawArgs }) => {
+    const program = new Command("eval").exitOverride().description("Run eval suites and set an exit code").option("-d, --dir <path>", "Directory to discover *.eval.js suites in", "packages/evals").option("-t, --tag <tag>", "Only run suites declaring this tag").option("-o, --out <file>", "Write the full JSON report to this path").option(
+      "--timeout-ms <ms>",
+      "How long one suite may run before it is called inconclusive",
+      (v) => Number.parseInt(v, 10)
+    );
+    try {
+      program.parse(rawArgs, { from: "user" });
+    } catch (err) {
+      if (err instanceof CommanderError) {
+        return err.code === "commander.helpDisplayed" ? EVAL_EXIT.ok : EVAL_EXIT.usage;
+      }
+      throw err;
+    }
+    return runEval(ctx, program.opts());
+  }, "handler")
+};
+export {
+  EVAL_EXIT,
+  discoverSuites,
+  evalCommand
+};
+//# sourceMappingURL=eval-BMJQXDKA.js.map

package/dist/chunks/eval-BMJQXDKA.js.map

@@ -0,0 +1,6 @@
+{
+  "version": 3,
+  "sources": ["../../src/commands/eval.ts"],
+  "mappings": ";;;;;;;;;;;;;;AAgBA,SAAS,OAAO,SAAS,iBAAiB;AAC1C,SAAS,YAAY,MAAM,UAAU,SAAS,WAAW;AACzD,SAAS,qBAAqB;AAiBvB,IAAM,YAAY;AAAA;AAAA,EAExB,IAAI;AAAA;AAAA,EAEJ,QAAQ;AAAA;AAAA,EAER,cAAc;AAAA;AAAA,EAEd,OAAO;AACR;AAEA,IAAM,eAAe;AAWrB,IAAM,2BAA2B;AAEjC,IAAM,YAAY,uBAAO,sBAAsB;AAS/C,eAAe,aAAgB,MAAkB,IAA2C;AAC3F,MAAI;AACJ,MAAI;AACH,WAAO,MAAM,QAAQ,KAAK;AAAA,MACzB;AAAA,MACA,IAAI,QAA0B,CAACA,aAAY;AAC1C,gBAAQ,WAAW,MAAMA,SAAQ,SAAS,GAAG,EAAE;AAAA,MAChD,CAAC;AAAA,IACF,CAAC;AAAA,EACF,UAAE;AACD,QAAI,MAAO,cAAa,KAAK;AAAA,EAC9B;AACD;AAZe;AAgCf,eAAsB,eAAe,MAA0C;AAC9E,QAAM,QAA2B,CAAC;AAElC,iBAAe,KAAK,KAA4B;AAC/C,QAAI;AACJ,QAAI;AACH,gBAAU,MAAM,QAAQ,KAAK,EAAE,eAAe,MAAM,UAAU,QAAQ,CAAC;AAAA,IACxE,QAAQ;AAIP;AAAA,IACD;AACA,eAAW,SAAS,QAAQ,KAAK,CAAC,GAAG,MAAO,EAAE,OAAO,EAAE,OAAO,KAAK,CAAE,GAAG;AACvE,YAAM,OAAO,KAAK,KAAK,MAAM,IAAI;AACjC,UAAI,MAAM,YAAY,GAAG;AACxB,YAAI,MAAM,SAAS,kBAAkB,MAAM,KAAK,WAAW,GAAG,EAAG;AACjE,cAAM,KAAK,IAAI;AACf;AAAA,MACD;AACA,UAAI,CAAC,MAAM,KAAK,SAAS,YAAY,EAAG;AACxC,YAAM,KAAK,EAAE,IAAI,QAAQ,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC;AAAA,IACnD;AAAA,EACD;AApBe;AAsBf,QAAM,KAAK,IAAI;AAEf,QAAM,OAAO,oBAAI,IAAoB;AACrC,aAAW,SAAS,OAAO;AAC1B,UAAM,WAAW,KAAK,IAAI,MAAM,EAAE;AAClC,QAAI,aAAa,QAAW;AAC3B,YAAM,IAAI;AAAA,QACT,sCAAsC,MAAM,EAAE,MAAM,QAAQ,QAAQ,MAAM,IAAI;AAAA,MAC/E;AAAA,IACD;AACA,SAAK,IAAI,MAAM,IAAI,MAAM,IAAI;AAAA,EAC9B;AAEA,SAAO;AACR;AAvCsB;AA0CtB,SAAS,QAAQ,MAAc,MAAsB;AACpD,SAAO,SAAS,MAAM,IAAI,EAAE,MAAM,GAAG,CAAC,aAAa,MAAM,EAAE,MAAM,GAAG,EAAE,KAAK,GAAG;AAC/E;AAFS;AAkBT,eAAe,UAAU,MAAoC;AAC5D,QAAM,MAAO,MAAM,OAAO,cAAc,IAAI,EAAE;AAC9C,MAAI,OAAO,IAAI,YAAY,YAAY;AACtC,UAAM,IAAI;AAAA,MACT,eAAe,IAAI;AAAA,IACpB;AAAA,EACD;AACA,SAAO;AACR;AARe;AAiBf,eAAe,QAAQ,KAAqB,SAAuC;AAClF,QAAM,OAAO,WAAW,QAAQ,GAAG,IAAI,QAAQ,MAAM,QAAQ,QAAQ,IAAI,GAAG,QAAQ,GAAG;AAEvF,MAAI;AACJ,MAAI;AACH,aAAS,MAAM,eAAe,IAAI;AAAA,EACnC,SAAS,KAAK;AACb,QAAI,UAAU,MAAM,EAAE,SAAS,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;AACjF,WAAO,UAAU;AAAA,EAClB;AAEA,MAAI,OAAO,WAAW,GAAG;AACxB,QAAI,UAAU,MAAM;AAAA,MACnB,SAAS,8BAA8B,IAAI,8BAA8B,YAAY;AAAA,IACtF,CAAC;AACD,WAAO,UAAU;AAAA,EAClB;AAEA,QAAM,UAA8D,CAAC;AACrE,MAAI,UAAU;AAEd,aAAW,SAAS,QAAQ;AAC3B,QAAI;AACJ,QAAI;AACH,YAAM,MAAM,UAAU,MAAM,IAAI;AAAA,IACjC,SAAS,KAAK;AACb,UAAI,UAAU,MAAM,EAAE,SAAS,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;AACjF,aAAO,UAAU;AAAA,IAClB;AAEA,QAAI,QAAQ,OAAO,EAAE,IAAI,QAAQ,CAAC,GAAG,SAAS,QAAQ,GAAG,GAAG;AAC3D;AACA;AAAA,IACD;AAEA,UAAM,SAAS,MAAM;AAAA,MACnB,IAAI,QAA4C;AAAA,MACjD,QAAQ,aAAa;AAAA,IACtB;AAYA,QAAI,WAAW,WAAW;AACzB,UAAI,UAAU,MAAM;AAAA,QACnB,SAAS,eAAe,MAAM,EAAE,2BAA2B,QAAQ,aAAa,wBAAwB;AAAA,MACzG,CAAC;AACD,aAAO,UAAU;AAAA,IAClB;AAEA,YAAQ,KAAK,EAAE,OAAO,MAAM,IAAI,OAAO,CAAC;AACxC,QAAI,UAAU,MAAM,EAAE,OAAO,MAAM,IAAI,QAAQ,MAAM,aAAa,MAAM,EAAE,CAAC;AAAA,EAC5E;AAKA,MAAI,UAAU,GAAG;AAChB,QAAI,UAAU,KAAK,GAAG,OAAO,8BAA8B,QAAQ,GAAG,EAAE;AAAA,EACzE;AACA,MAAI,QAAQ,WAAW,GAAG;AACzB,QAAI,UAAU,MAAM,EAAE,SAAS,0BAA0B,QAAQ,GAAG,IAAI,CAAC;AACzE,WAAO,UAAU;AAAA,EAClB;AAEA,MAAI,QAAQ,KAAK;AAChB,UAAM,cAAc,QAAQ,KAAK,OAAO;AACxC,QAAI,UAAU,KAAK,SAAS,QAAQ,GAAG,EAAE;AAAA,EAC1C;AAEA,QAAM,SAAS,QAAQ,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,OAAO,QAAQ,CAAC;AAClE,QAAM,eAAe,QAAQ,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,OAAO,cAAc,CAAC;AAM9E,MAAI,eAAe,EAAG,QAAO,UAAU;AACvC,MAAI,SAAS,EAAG,QAAO,UAAU;AACjC,SAAO,UAAU;AAClB;AAvFe;AAyFf,eAAe,cACd,KACA,SACgB;AAChB,QAAM,OAAO,WAAW,GAAG,IAAI,MAAM,QAAQ,QAAQ,IAAI,GAAG,GAAG;AAC/D,QAAM,MAAM,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AAGjD,QAAM,UAAU,MAAM,GAAG,KAAK,UAAU,EAAE,QAAQ,QAAQ,GAAG,MAAM,CAAC,CAAC;AAAA,GAAM,OAAO;AACnF;AATe;AAWR,IAAM,cAA0B;AAAA,EACtC,MAAM;AAAA,EACN,aAAa;AAAA,EACb,aAAa;AAAA,EACb,SAAS,8BAAO,EAAE,KAAK,QAAQ,MAAuB;AACrD,UAAM,UAAU,IAAI,QAAQ,MAAM,EAChC,aAAa,EACb,YAAY,sCAAsC,EAClD,OAAO,oBAAoB,6CAA6C,gBAAgB,EACxF,OAAO,mBAAmB,oCAAoC,EAC9D,OAAO,oBAAoB,yCAAyC,EACpE;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAC,MAAM,OAAO,SAAS,GAAG,EAAE;AAAA,IAC7B;AAED,QAAI;AACH,cAAQ,MAAM,SAAS,EAAE,MAAM,OAAO,CAAC;AAAA,IACxC,SAAS,KAAK;AACb,UAAI,eAAe,gBAAgB;AAGlC,eAAO,IAAI,SAAS,4BAA4B,UAAU,KAAK,UAAU;AAAA,MAC1E;AACA,YAAM;AAAA,IACP;AAEA,WAAO,QAAQ,KAAK,QAAQ,KAAkB,CAAC;AAAA,EAChD,GAzBS;AA0BV;",
+  "names": ["resolve"]
+}