@wrongstack/cli 0.319.0 → 1.0.0

package/dist/{bench-XYG56AMO.js → bench-YBU6BVQP.js}

@@ -1,24 +1,27 @@
 import {
   CLI_VERSION
 } from "./chunk-XJXDOF63.js";
-import "./chunk-7OCVIDC7.js";
 
 // src/subcommands/handlers/bench.ts
 import * as fs from "node:fs/promises";
 import { createRequire } from "node:module";
-import * as path from "node:path";
+import * as path2 from "node:path";
 import {
   collectCellPredictions,
+  compareReports,
   computeToolManifestHash,
+  createCoreSuite,
   createLocalManifestSuite,
   createPolyglotSuite,
+  createSmokeSuite,
   createSwebenchSuite,
   gradeLocalManifest,
   gradePolyglot,
   gradeSwebench,
-  loadBenchConfig,
+  loadBenchConfig as loadBenchConfig2,
   mineTranscript,
-  readSummary,
+  readRunDir,
+  renderComparisonMarkdown,
   renderMarkdownReport,
   reportHeaderLine,
   runBenchmark,
@@ -26,6 +29,68 @@ import {
   writePredictionsJsonl
 } from "@wrongstack/bench";
 import { color, toErrorMessage } from "@wrongstack/core/utils";
+
+// src/subcommands/handlers/bench-run-config.ts
+import * as path from "node:path";
+import {
+  configFromCells,
+  loadBenchConfig,
+  parseCellList,
+  CORE_CONFIG_DEFAULTS,
+  SMOKE_CONFIG_DEFAULTS
+} from "@wrongstack/bench";
+var DEFAULT_BENCH_MODELS_FILE = "bench.config.json";
+async function resolveBenchRunConfig(input) {
+  const suiteDefaults = input.suiteId === "core" ? CORE_CONFIG_DEFAULTS : input.suiteId === "smoke" ? SMOKE_CONFIG_DEFAULTS : void 0;
+  const cellSpec = stringFlag(input.flags, "cell") ?? stringFlag(input.flags, "cells");
+  if (cellSpec) {
+    return configFromCells(parseCellList(cellSpec), suiteDefaults);
+  }
+  const providerFlag = stringFlag(input.flags, "provider");
+  const modelFlag = stringFlag(input.flags, "model");
+  if (providerFlag && modelFlag) {
+    return configFromCells(
+      [{ label: `${providerFlag}/${modelFlag}`, provider: providerFlag, model: modelFlag }],
+      suiteDefaults
+    );
+  }
+  const modelsFlag = stringFlag(input.flags, "models");
+  const modelsPath = modelsFlag ?? DEFAULT_BENCH_MODELS_FILE;
+  const resolvedModels = path.resolve(input.cwd, modelsPath);
+  const customModelsFile = modelsFlag !== void 0 && modelsFlag !== DEFAULT_BENCH_MODELS_FILE;
+  try {
+    return await loadBenchConfig(resolvedModels);
+  } catch (err) {
+    if (customModelsFile || !isMissingConfigError(err)) throw err;
+  }
+  const savedProvider = input.savedProvider;
+  const savedModel = input.savedModel;
+  if (savedProvider && savedModel) {
+    return configFromCells(
+      [
+        {
+          label: `${savedProvider}/${savedModel}`,
+          provider: savedProvider,
+          model: savedModel
+        }
+      ],
+      suiteDefaults
+    );
+  }
+  throw new Error(
+    "No model cells. Pass --cell provider/model[,provider/model], --provider + --model, or --models <config.json>"
+  );
+}
+function stringFlag(flags, name) {
+  const value = flags?.[name];
+  return typeof value === "string" && value.length > 0 ? value : void 0;
+}
+function isMissingConfigError(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  return /ENOENT|no such file or directory/i.test(message);
+}
+
+// src/subcommands/handlers/bench.ts
 var benchCmd = async (args, deps) => {
   const sub = args[0];
   const rest = args.slice(1);
@@ -34,6 +99,8 @@ var benchCmd = async (args, deps) => {
       return benchRun(rest, deps);
     case "report":
       return benchReport(rest, deps);
+    case "compare":
+      return benchCompare(rest, deps);
     case "list":
       return benchList(rest, deps);
     case "mine":
@@ -48,16 +115,20 @@ function printUsage(deps) {
     [
       color.bold("wstack bench") + " \u2014 model-independent agentic benchmarks",
       "",
-      "  run     Run a suite across a model matrix and write a report",
-      "  mine    Copy a real session transcript and draft trace-eval cases",
-      "  report  Re-render report.md from a finished run directory",
-      "  list    Show available suites and configured model cells",
+      "  run      Run a suite across a model matrix and write a report",
+      "  compare  Diff two finished run directories (fingerprint-aware)",
+      "  mine     Copy a real session transcript and draft trace-eval cases",
+      "  report   Re-render report.md from a finished run directory",
+      "  list     Show available suites and configured model cells",
       "",
       color.dim("Examples:"),
+      color.dim("  wstack bench run --cell anthropic/claude-sonnet-4-6,openai/gpt-5.4"),
+      color.dim("  wstack bench run --suite smoke --cell zai-coding-plan/glm-5.3-flash"),
       color.dim(
         "  wstack bench run --suite polyglot --polyglot-dir ./polyglot --models bench.config.json --limit 5"
       ),
       color.dim("  wstack bench run --suite local --suite-dir ./evals --models bench.config.json"),
+      color.dim("  wstack bench compare ./bench-results/<baseline> ./bench-results/<candidate>"),
       color.dim("  wstack bench mine --transcript ./session.jsonl --out ./evals"),
       color.dim("  wstack bench report ./bench-results/2026-06-14T10-00-00"),
       ""
@@ -76,7 +147,7 @@ async function resolveWstackEntry() {
   try {
     const req = createRequire(import.meta.url);
     const pkgPath = req.resolve("@wrongstack/cli/package.json");
-    const entry = path.join(path.dirname(pkgPath), "dist", "index.js");
+    const entry = path2.join(path2.dirname(pkgPath), "dist", "index.js");
     await fs.access(entry);
     return entry;
   } catch {
@@ -84,14 +155,19 @@ async function resolveWstackEntry() {
   }
 }
 async function benchRun(_args, deps) {
-  const suiteId = flagStr(deps, "suite") ?? "polyglot";
-  const modelsPath = flagStr(deps, "models") ?? "bench.config.json";
+  const suiteId = flagStr(deps, "suite") ?? "core";
   const limitRaw = flagStr(deps, "limit");
   const limit = limitRaw ? Math.max(1, Number.parseInt(limitRaw, 10)) : void 0;
   const outBase = flagStr(deps, "out") ?? "bench-results";
   let config;
   try {
-    config = await loadBenchConfig(path.resolve(deps.cwd, modelsPath));
+    config = await resolveBenchRunConfig({
+      suiteId,
+      cwd: deps.cwd,
+      flags: deps.flags,
+      savedProvider: deps.config?.provider,
+      savedModel: deps.config?.model
+    });
   } catch (err) {
     deps.renderer.writeError(toErrorMessage(err));
     return 1;
@@ -101,13 +177,28 @@ async function benchRun(_args, deps) {
     const c = Number.parseInt(concurrencyRaw, 10);
     if (c > 0) config.concurrency = c;
   }
+  const repeatsRaw = flagStr(deps, "repeats");
+  if (repeatsRaw) {
+    const r = Number.parseInt(repeatsRaw, 10);
+    if (!Number.isFinite(r) || r <= 0) {
+      deps.renderer.writeError("--repeats must be a positive integer.");
+      return 1;
+    }
+    config.repeats = r;
+  }
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-  const outDir = path.resolve(deps.cwd, outBase, stamp);
-  const predictionsDir = path.join(outDir, "predictions");
+  const outDir = path2.resolve(deps.cwd, outBase, stamp);
+  const predictionsDir = path2.join(outDir, "predictions");
   let suite;
   let grade;
   let isSwebench = false;
-  if (suiteId === "polyglot") {
+  if (suiteId === "core") {
+    suite = createCoreSuite();
+    grade = (a) => gradeLocalManifest(a);
+  } else if (suiteId === "smoke") {
+    suite = createSmokeSuite();
+    grade = (a) => gradeLocalManifest(a);
+  } else if (suiteId === "polyglot") {
     const polyglotDir = flagStr(deps, "polyglot-dir");
     if (!polyglotDir) {
       deps.renderer.writeError("--polyglot-dir <path> is required for the polyglot suite.");
@@ -115,14 +206,14 @@ async function benchRun(_args, deps) {
     }
     const languagesRaw = flagStr(deps, "languages");
     const languages = languagesRaw ? languagesRaw.split(",").map((s) => s.trim()).filter(Boolean) : void 0;
-    suite = createPolyglotSuite({ polyglotDir: path.resolve(deps.cwd, polyglotDir), languages });
+    suite = createPolyglotSuite({ polyglotDir: path2.resolve(deps.cwd, polyglotDir), languages });
     grade = (a) => gradePolyglot(a);
   } else if (suiteId === "swebench") {
     isSwebench = true;
     const datasetDir = flagStr(deps, "dataset-dir");
     const docker = flagBool(deps, "docker");
     suite = createSwebenchSuite({
-      datasetDir: datasetDir ? path.resolve(deps.cwd, datasetDir) : void 0,
+      datasetDir: datasetDir ? path2.resolve(deps.cwd, datasetDir) : void 0,
       docker
     });
     grade = (a) => gradeSwebench({ ...a, predictionsDir });
@@ -135,12 +226,14 @@ async function benchRun(_args, deps) {
       );
       return 1;
     }
-    const manifestFile = manifestRaw ? path.resolve(deps.cwd, manifestRaw) : void 0;
-    const suiteDir = suiteDirRaw ? path.resolve(deps.cwd, suiteDirRaw) : path.dirname(manifestFile);
+    const manifestFile = manifestRaw ? path2.resolve(deps.cwd, manifestRaw) : void 0;
+    const suiteDir = suiteDirRaw ? path2.resolve(deps.cwd, suiteDirRaw) : path2.dirname(manifestFile);
     suite = createLocalManifestSuite({ suiteDir, manifestFile });
     grade = (a) => gradeLocalManifest(a);
   } else {
-    deps.renderer.writeError(`unknown suite "${suiteId}" (expected: polyglot | swebench | local)`);
+    deps.renderer.writeError(
+      `unknown suite "${suiteId}" (expected: core | smoke | local | polyglot | swebench)`
+    );
     return 1;
   }
   const tools = deps.toolRegistry?.list() ?? [];
@@ -148,6 +241,9 @@ async function benchRun(_args, deps) {
   const toolManifestHash = computeToolManifestHash(tools);
   const wstackEntry = await resolveWstackEntry();
   deps.renderer.writeInfo(`Running ${suiteId} across ${config.cells.length} model(s)\u2026`);
+  await fs.mkdir(outDir, { recursive: true });
+  const partialPath = path2.join(outDir, "results.jsonl");
+  await fs.writeFile(partialPath, "", "utf8");
   let report;
   try {
     report = await runBenchmark({
@@ -160,7 +256,10 @@ async function benchRun(_args, deps) {
       nodeBin: process.execPath,
       wstackEntry,
       limit,
-      onProgress: (msg) => deps.renderer.write(color.dim(msg) + "\n")
+      hostHomeDir: deps.paths?.globalRoot,
+      keepSandbox: flagBool(deps, "keep-sandbox"),
+      onProgress: (msg) => deps.renderer.write(color.dim(msg) + "\n"),
+      onResult: (result) => fs.appendFile(partialPath, JSON.stringify(result) + "\n", "utf8")
     });
   } catch (err) {
     deps.renderer.writeError(toErrorMessage(err));
@@ -168,7 +267,7 @@ async function benchRun(_args, deps) {
   }
   await writeJsonArtifacts(outDir, report);
   const md = renderMarkdownReport(report);
-  await fs.writeFile(path.join(outDir, "report.md"), md, "utf8");
+  await fs.writeFile(path2.join(outDir, "report.md"), md, "utf8");
   deps.renderer.write("\n" + md + "\n");
   if (isSwebench) {
     for (const cell of config.cells) {
@@ -181,7 +280,14 @@ async function benchRun(_args, deps) {
       "Grade with the official SWE-bench harness: python -m swebench.harness.run_evaluation --predictions_path <file> --run_id <id>"
     );
   }
-  deps.renderer.writeInfo(`Report written to ${path.join(outDir, "report.md")}`);
+  deps.renderer.writeInfo(`Report written to ${path2.join(outDir, "report.md")}`);
+  const spawned = report.results.filter((row) => row.run.status !== "crashed").length;
+  if (report.results.length > 0 && spawned === 0) {
+    deps.renderer.writeError(
+      "Every run crashed before producing a result \u2014 check the provider/model ids and credentials. See the Failures section of the report."
+    );
+    return 1;
+  }
   return 0;
 }
 async function benchReport(args, deps) {
@@ -190,19 +296,47 @@ async function benchReport(args, deps) {
     deps.renderer.writeError("Usage: wstack bench report <run-directory>");
     return 1;
   }
-  const outDir = path.resolve(deps.cwd, dir);
-  let summary;
+  const outDir = path2.resolve(deps.cwd, dir);
+  let report;
   try {
-    summary = await readSummary(outDir);
+    report = await readRunDir(outDir);
   } catch (err) {
-    deps.renderer.writeError(`cannot read summary.json in ${outDir}: ${toErrorMessage(err)}`);
+    deps.renderer.writeError(`cannot read run artifacts in ${outDir}: ${toErrorMessage(err)}`);
     return 1;
   }
-  const md = renderMarkdownReport(summary);
-  await fs.writeFile(path.join(outDir, "report.md"), md, "utf8");
+  const md = renderMarkdownReport(report);
+  await fs.writeFile(path2.join(outDir, "report.md"), md, "utf8");
   deps.renderer.write("\n" + md + "\n");
   return 0;
 }
+async function benchCompare(args, deps) {
+  const dirs = args.filter((a) => !a.startsWith("-"));
+  const baselineRaw = dirs[0];
+  const candidateRaw = dirs[1];
+  if (!baselineRaw || !candidateRaw) {
+    deps.renderer.writeError("Usage: wstack bench compare <baseline-dir> <candidate-dir>");
+    return 1;
+  }
+  const baselineDir = path2.resolve(deps.cwd, baselineRaw);
+  const candidateDir = path2.resolve(deps.cwd, candidateRaw);
+  let comparisonMd;
+  try {
+    const baseline = await readRunDir(baselineDir);
+    const candidate = await readRunDir(candidateDir);
+    const comparison = compareReports(baseline, candidate);
+    comparisonMd = renderComparisonMarkdown(comparison);
+  } catch (err) {
+    deps.renderer.writeError(`cannot compare runs: ${toErrorMessage(err)}`);
+    return 1;
+  }
+  const outRaw = flagStr(deps, "out");
+  const outFile = outRaw ? path2.resolve(deps.cwd, outRaw) : path2.join(candidateDir, "compare.md");
+  await fs.mkdir(path2.dirname(outFile), { recursive: true });
+  await fs.writeFile(outFile, comparisonMd, "utf8");
+  deps.renderer.write("\n" + comparisonMd + "\n");
+  deps.renderer.writeInfo(`Comparison written to ${outFile}`);
+  return 0;
+}
 async function benchMine(args, deps) {
   const transcriptRaw = flagStr(deps, "transcript") ?? args.find((arg) => !arg.startsWith("-"));
   if (!transcriptRaw) {
@@ -214,8 +348,8 @@ async function benchMine(args, deps) {
   const outRaw = flagStr(deps, "out") ?? "evals";
   try {
     const mined = await mineTranscript({
-      transcriptPath: path.resolve(deps.cwd, transcriptRaw),
-      outDir: path.resolve(deps.cwd, outRaw)
+      transcriptPath: path2.resolve(deps.cwd, transcriptRaw),
+      outDir: path2.resolve(deps.cwd, outRaw)
     });
     deps.renderer.writeInfo(
       `Mined ${mined.candidates.length} edit attempt(s) from ${mined.sessionId}.`
@@ -234,18 +368,24 @@ async function benchMine(args, deps) {
 async function benchList(_args, deps) {
   deps.renderer.write(color.bold("Suites\n"));
   deps.renderer.write(
-    "  polyglot  " + color.dim("Aider polyglot (edit accuracy) \u2014 Phase 1, Docker-free\n")
+    "  core      " + color.dim("Bundled 6-task agent-edit eval (hidden tests) \u2014 default for `bench run`\n")
   );
   deps.renderer.write(
-    "  swebench  " + color.dim("SWE-bench Verified (end-to-end) \u2014 Phase 2, Docker-gated\n")
+    "  smoke     " + color.dim("3 trivial file edits \u2014 harness wiring only, not a quality score\n")
   );
   deps.renderer.write(
     "  local     " + color.dim("Project-defined manifest tasks with command/file graders\n")
   );
+  deps.renderer.write(
+    "  polyglot  " + color.dim("Aider polyglot (edit accuracy) \u2014 Docker-free, needs --polyglot-dir\n")
+  );
+  deps.renderer.write(
+    "  swebench  " + color.dim("SWE-bench Verified (end-to-end) \u2014 Docker-gated\n")
+  );
   const modelsPath = flagStr(deps, "models");
   if (modelsPath) {
     try {
-      const config = await loadBenchConfig(path.resolve(deps.cwd, modelsPath));
+      const config = await loadBenchConfig2(path2.resolve(deps.cwd, modelsPath));
       deps.renderer.write("\n" + color.bold("Model cells\n"));
       for (const cell of config.cells) {
         deps.renderer.write(
@@ -274,4 +414,4 @@ async function benchList(_args, deps) {
 export {
   benchCmd
 };
-//# sourceMappingURL=bench-XYG56AMO.js.map
+//# sourceMappingURL=bench-YBU6BVQP.js.map

package/dist/boot/brain-decision-log.d.ts

@@ -12,16 +12,29 @@
  * The host wires this up once at boot and retains a reference to `brainLog`
  * (and indirectly `pushBrainLog`) for the duration of the session.
  */
+import { type BrainTierStats } from '@wrongstack/core/coordination';
 type BrainDecisionKind = 'answered' | 'ask_human' | 'denied' | 'intervention' | 'council_warn';
 export interface BrainDecisionEntry {
     at: number;
     kind: BrainDecisionKind;
     question: string;
     outcome: string;
+    /**
+     * Which tier resolved the decision, as carried on the event.
+     *
+     * `/brain stats` groups the ring buffer by this field. It used to read a
+     * `tier` that nothing here ever wrote, so every decision counted as
+     * `unattributed` and the deterministic/model-backed split was permanently
+     * 0/0 — the whole provenance chain (`markDecisionTier` → the WeakMap →
+     * the `tier` field on `brain.decision_*`) terminated one step short of its
+     * only surface. Typed as a plain string: the CLI does not need to import
+     * the core union to bucket a label.
+     */
+    tier?: string | undefined;
 }
 /**
- * Subscribe to the four brain.* decision events and maintain a rolling
- * 20-entry ring buffer. Caller retains the returned `brainLog` array; the
+ * Subscribe to the brain.* decision events and maintain a rolling 20-entry
+ * ring buffer PLUS a session-lifetime per-tier tally. Caller retains the returned `brainLog` array; the
  * returned `pushBrainLog` lets external code (e.g. slash-command handlers)
  * append additional entries that should appear in /brain status.
  *
@@ -35,6 +48,13 @@ export interface BrainDecisionEntry {
 export declare function subscribeBrainDecisionLog(events: any): {
     brainLog: BrainDecisionEntry[];
     pushBrainLog: (entry: BrainDecisionEntry) => void;
+    /**
+     * Per-tier tally over the WHOLE session, not just the 20 entries the ring
+     * still holds. `/brain stats` is a "how often does the Brain burn a
+     * provider call" question, and a 20-decision window answers it for the
+     * last minute of a session that has been running for hours.
+     */
+    getTierStats: () => BrainTierStats;
     dispose: () => void;
 };
 export {};

package/dist/{boot-auto-thin-BZ675EVT.js → boot-auto-thin-MANHOYHE.js}

@@ -4,7 +4,6 @@ import {
 import {
   activeProfileConfigPath
 } from "./chunk-YMXXOOFN.js";
-import "./chunk-7OCVIDC7.js";
 
 // src/boot/boot-auto-thin.ts
 import {
@@ -99,4 +98,4 @@ async function runBootAutoThin(deps) {
 export {
   runBootAutoThin
 };
-//# sourceMappingURL=boot-auto-thin-BZ675EVT.js.map
+//# sourceMappingURL=boot-auto-thin-MANHOYHE.js.map

package/dist/{chronicle-6EPPMUC5.js → chronicle-UTFETHP6.js}

@@ -1,4 +1,3 @@
-import "./chunk-7OCVIDC7.js";
 
 // src/subcommands/handlers/chronicle.ts
 import { statSync } from "node:fs";
@@ -274,4 +273,4 @@ function tryStat(file) {
 export {
   chronicleCmd
 };
-//# sourceMappingURL=chronicle-6EPPMUC5.js.map
+//# sourceMappingURL=chronicle-UTFETHP6.js.map

package/dist/{chunk-VWCZL4OH.js → chunk-4HV5RKZH.js}

@@ -1,10 +1,10 @@
 import {
   helpCmd,
   versionCmd
-} from "./chunk-QAC263FZ.js";
+} from "./chunk-FPIWTJ2G.js";
 import {
   parseArgs
-} from "./chunk-GGGUZSAU.js";
+} from "./chunk-Z7A5MXKJ.js";
 
 // src/boot/short-circuit-flags.ts
 async function handleHelpVersionShortCircuit(argv) {
@@ -22,11 +22,11 @@ async function handleHelpVersionShortCircuit(argv) {
   }
   if (earlyFlags["help"] === true) {
     if (positional.length > 0) {
-      const { subcommandNames } = await import("./subcommands-J4JMWC5W.js");
+      const { subcommandNames } = await import("./subcommands-HZQKA4G2.js");
       if (subcommandNames.includes(positional[0])) {
         const first = positional[0];
         const deep = positional[1];
-        const { renderDeepHelp, renderFocusedHelp } = await import("./per-subcommand-help-6VCIKDUW.js");
+        const { renderDeepHelp, renderFocusedHelp } = await import("./per-subcommand-help-BWBHYVJO.js");
         if (deep && renderDeepHelp(`${first}:${deep}`, stubRenderer)) {
           return 0;
         }
@@ -43,4 +43,4 @@ async function handleHelpVersionShortCircuit(argv) {
 export {
   handleHelpVersionShortCircuit
 };
-//# sourceMappingURL=chunk-VWCZL4OH.js.map
+//# sourceMappingURL=chunk-4HV5RKZH.js.map

package/dist/{chunk-42JD25KU.js → chunk-5IA5337I.js}

@@ -1,10 +1,10 @@
 import {
   patchConfig
-} from "./chunk-TYO2OVAD.js";
+} from "./chunk-EWKESL4L.js";
 import {
   PLUGIN_AUDIT_ENTRIES,
   loadExternalPlugins
-} from "./chunk-H7GABTGS.js";
+} from "./chunk-TEBMCYZN.js";
 
 // src/wiring/plugins.ts
 import { join } from "node:path";
@@ -261,4 +261,4 @@ export {
   BUILTIN_PLUGIN_FACTORIES,
   setupPlugins
 };
-//# sourceMappingURL=chunk-42JD25KU.js.map
+//# sourceMappingURL=chunk-5IA5337I.js.map