nomarmy 0.1.0-alpha.14 → 0.1.0-alpha.15

package/lib/stats.mjs

@@ -118,12 +118,13 @@ export function computeStats(records, { repo = null, sinceMs = null, untilMs = n
 
   const workerMinutes = implement.map((r) => r.metrics?.worker_elapsed).filter(Number.isFinite).map((ms) => ms / 60000);
   const jobMinutes = implement.map((r) => r.metrics?.total_elapsed).filter(Number.isFinite).map((ms) => ms / 60000);
-  const tokens = { total: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
+  const tokens = { total: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, untracked: 0 };
   const spend = {};
   for (const r of jobs) {
     const m = r.metrics ?? {};
     tokens.total += m.worker_tokens_total ?? 0; tokens.input += m.worker_tokens_in ?? 0; tokens.output += m.worker_tokens_out ?? 0;
     tokens.cacheRead += m.worker_tokens_cache_read ?? 0; tokens.cacheWrite += m.worker_tokens_cache_write ?? 0;
+    if (r.mode !== "verify" && !(m.worker_tokens_total > 0)) tokens.untracked++;
     if (Number.isFinite(m.worker_cost_usd) && m.worker_cost_usd > 0) spend[jobModel(r) ?? "unknown"] = (spend[jobModel(r) ?? "unknown"] ?? 0) + m.worker_cost_usd;
   }
 
@@ -131,6 +132,8 @@ export function computeStats(records, { repo = null, sinceMs = null, untilMs = n
   const claimedDone = implement.filter((r) => r.reportValidation?.status === "done" && r.reportValidation?.tests === "pass");
   const verificationFailed = claimedDone.filter((r) => r.independentVerification?.status === "fail");
   const revertStillPassed = claimedDone.filter((r) => r.independentVerification?.status === "pass" && r.regressionCheck?.status === "fail");
+  // Claimed success with an empty diff: there was nothing to verify.
+  const changedNothing = claimedDone.filter((r) => r.independentVerification?.status === "not_run");
   const passedBoth = claimedDone.filter((r) => r.independentVerification?.status === "pass" && r.regressionCheck?.status !== "fail");
   const flaggedAfterPassing = passedBoth.filter((r) => (r.issues ?? []).some((i) => /^(MUTANTS SURVIVED|REPORT MAY NOT MATCH|JUDGE \(|VERIFICATION INPUT CHANGED)/.test(i)));
 
@@ -176,6 +179,7 @@ export function computeStats(records, { repo = null, sinceMs = null, untilMs = n
       verificationFailed: verificationFailed.length,
       revertStillPassed: revertStillPassed.length,
       passedBoth: passedBoth.length,
+      changedNothing: changedNothing.length,
       flaggedAfterPassing: flaggedAfterPassing.length,
     },
     notCompleted: sortDesc(count(jobs.filter((r) => !/^(WORKER_DONE|RECOVERED_SUCCESS|VERIFIED|SCOUT_DONE|DECOMPOSE_DONE|SCOUT_NOT_FOUND)$/.test(r.outcome ?? "")), (r) => r.outcome)),
@@ -212,7 +216,7 @@ export function formatStats(s) {
     `  Committed     ${s.code.committedJobs} job(s) · +${s.code.linesAdded} / -${s.code.linesRemoved} lines · ${s.code.files} files · ${s.code.newTestFiles} new test files`,
     `  Worker time   median ${mins(s.workerMinutes.median)} per implement job, p90 ${mins(s.workerMinutes.p90)}, total ${Math.round(s.workerMinutes.total)} min`,
     `  Job time      median ${mins(s.jobMinutes.median)}, p90 ${mins(s.jobMinutes.p90)}, total ${Math.round(s.jobMinutes.total)} min (with verification and checks)`,
-    `  Tokens        ${big(s.tokens.total)} total (${big(s.tokens.input)} in, ${big(s.tokens.output)} out, ${big(s.tokens.cacheRead)} cache read)`,
+    `  Tokens        ${big(s.tokens.total)} total (${big(s.tokens.input)} in, ${big(s.tokens.output)} out, ${big(s.tokens.cacheRead)} cache read)${s.tokens.untracked ? `; ${s.tokens.untracked} job(s) recorded no token counts` : ""}`,
     `  API spend     $${s.spendUsd.total.toFixed(2)}${Object.keys(s.spendUsd.byModel).length ? ` (${Object.entries(s.spendUsd.byModel).map(([k, v]) => `${k} $${v.toFixed(2)}`).join(", ")})` : ""}; subscriptions aren't billed per call`,
     "",
     `CLAIM VS EVIDENCE (implement jobs that reported "done, tests pass": ${c.claimedDone})`,
@@ -220,6 +224,7 @@ export function formatStats(s) {
     `  Passed, but reverting still passed    ${c.revertStillPassed}${pct(c.revertStillPassed, c.claimedDone)}`,
     `  Passed both                           ${c.passedBoth}${pct(c.passedBoth, c.claimedDone)}`,
     `    of those, flagged by another check  ${c.flaggedAfterPassing} (mutants, Jev, judge, rewritten checks)`,
+    ...(c.changedNothing ? [`  Changed nothing, nothing to verify    ${c.changedNothing}${pct(c.changedNothing, c.claimedDone)}`] : []),
     `  High-stakes jobs                      ${s.highStakes?.jobs ?? 0}, ${s.highStakes?.reviewed ?? 0} with an independent review`,
     "  Defects the General found at integration aren't in the records; count them in your own review.",
     "",

package/lib/suggestions.mjs

@@ -11,7 +11,12 @@ const OK = /^(WORKER_DONE|RECOVERED_SUCCESS|SCOUT_DONE|SCOUT_NOT_FOUND|DECOMPOSE
 
 const provider = (r) => r.metrics?.worker_provider ?? r.worker?.provider ?? null;
 const model = (r) => r.metrics?.worker_model ?? r.worker?.model ?? null;
-const agent = (r) => r.labels?.agent ?? null;
+// The local model is the built-in "local" agent, whatever model is loaded.
+const agent = (r) => r.labels?.agent ?? (provider(r) === "llama-cpp" ? "local" : null);
+// New tokens only: cache reads are most of an api job's total and cost a fraction.
+const freshTokens = (r) => (r.metrics?.worker_tokens_in ?? 0) + (r.metrics?.worker_tokens_out ?? 0);
+/** The runner exited before a report: an OpenClaw, sandbox or provider failure, not the model's work. */
+export const runnerFailed = (r) => r.outcome === "WORKER_FAILED" && [...(r.issues ?? []), ...(r.reasons ?? [])].some((x) => /^(worker|scout|decomposer) process failed/.test(x));
 const pct = (n, of) => Math.round((100 * n) / of);
 
 /** Whether a high-stakes job has had an independent review: a scout, or a judge, on another vendor. */
@@ -36,11 +41,13 @@ export function computeSuggestions(records, { minJobs = MIN_JOBS, agentFor = ()
   const groups = new Map();
   for (const r of work) {
     const key = `${jobRole(r) ?? ""}|${model(r) ?? ""}|${r.mode}`;
-    const g = groups.get(key) ?? { role: jobRole(r), model: model(r), mode: r.mode, agent: agent(r), provider: provider(r), jobs: 0, ok: 0, timeout: 0, unsupported: 0, tokens: 0 };
-    g.jobs++; if (OK.test(r.outcome ?? "")) g.ok++;
+    const g = groups.get(key) ?? { role: jobRole(r), model: model(r), mode: r.mode, agent: agent(r), provider: provider(r), jobs: 0, rated: 0, ok: 0, runner: 0, timeout: 0, unsupported: 0, tokens: 0, tokenJobs: 0 };
+    g.jobs++;
+    if (runnerFailed(r)) g.runner++; else g.rated++;
+    if (OK.test(r.outcome ?? "")) g.ok++;
     if (r.outcome === "WORKER_TIMEOUT") g.timeout++;
     if (r.outcome === "SCOUT_UNSUPPORTED") g.unsupported++;
-    g.tokens += r.metrics?.worker_tokens_total ?? 0;
+    if (freshTokens(r) > 0) { g.tokens += freshTokens(r); g.tokenJobs++; }
     g.agent = g.agent ?? agent(r) ?? agentFor(provider(r));
     groups.set(key, g);
   }
@@ -50,39 +57,47 @@ export function computeSuggestions(records, { minJobs = MIN_JOBS, agentFor = ()
     if (!g.model) continue;
     const kind = g.mode === "scout" ? "scouts" : "implement jobs";
     const who = g.role ? `${g.role} on ${g.model}` : `${kind} with no role on ${g.model}`;
+    // The runner failing isn't the model doing poor work: say so apart, and leave those out of its rate.
+    if (g.runner >= 3 && g.runner / g.jobs >= 0.4) {
+      out.push({ level: "warn", key: `runner-failed:${g.role}:${g.model}:${g.mode}`, title: `${who}: the runner failed on ${g.runner} of ${g.jobs} ${kind} before any report`,
+        evidence: "OpenClaw, the sandbox or the provider exited early, so these say nothing about the model's work. Check `nomarmy health` and one job's log (`nomarmy jobs <id>`); a model its vendor refuses fails this way too.", command: null });
+    }
     // Scouts that come back empty.
-    if (g.mode === "scout" && g.jobs >= 3 && g.unsupported / g.jobs >= 0.4) {
-      out.push({ level: "warn", key: `scout-unsupported:${g.role}:${g.model}`, title: `${who}: ${g.unsupported} of ${g.jobs} scouts came back unsupported`,
+    if (g.mode === "scout" && g.rated >= 3 && g.unsupported / g.rated >= 0.4) {
+      out.push({ level: "warn", key: `scout-unsupported:${g.role}:${g.model}`, title: `${who}: ${g.unsupported} of ${g.rated} scouts came back unsupported`,
         evidence: "Their findings couldn't be tied to cited lines. A different agent, or report: full, usually fixes it.", command: assign(g) });
       continue;
     }
-    if (g.jobs < minJobs) continue;
+    if (g.rated < minJobs) continue;
+    const runnerNote = g.runner ? ` (plus ${g.runner} the runner failed on, not counted)` : "";
     // A pairing that rarely finishes.
-    if (g.ok / g.jobs < 0.5) {
-      const better = [...groups.values()].filter((o) => o !== g && o.mode === g.mode && o.jobs >= minJobs && o.model && o.ok / o.jobs >= g.ok / g.jobs + 0.2)
-        .sort((a, b) => b.ok / b.jobs - a.ok / a.jobs)[0];
-      out.push({ level: "warn", key: `low-success:${g.role}:${g.model}:${g.mode}`, title: `${who} finished ${g.ok} of ${g.jobs} ${g.role ? kind : ""}`.trim() + ` (${pct(g.ok, g.jobs)}%)`,
-        evidence: (better ? `${better.model} finished ${pct(better.ok, better.jobs)}% of its ${better.jobs} ${kind} here${better.role ? ` (as ${better.role})` : ""}.` : "No other model has enough jobs here to compare.") + (g.role ? "" : " These ran with no army role: send this kind of work to a role on a stronger agent instead."),
+    if (g.ok / g.rated < 0.5) {
+      const better = [...groups.values()].filter((o) => o !== g && o.mode === g.mode && o.rated >= minJobs && o.model && o.ok / o.rated >= g.ok / g.rated + 0.2)
+        .sort((a, b) => b.ok / b.rated - a.ok / a.rated)[0];
+      out.push({ level: "warn", key: `low-success:${g.role}:${g.model}:${g.mode}`, title: `${who} finished ${g.ok} of ${g.rated} ${g.role ? kind : ""}`.trim() + ` (${pct(g.ok, g.rated)}%)${runnerNote}`,
+        evidence: (better ? `${better.model} finished ${pct(better.ok, better.rated)}% of its ${better.rated} ${kind} here${better.role ? ` (as ${better.role})` : ""}.` : "No other model has enough jobs here to compare.") + (g.role ? "" : " These ran with no army role: send this kind of work to a role on a stronger agent instead."),
         command: g.role ? assign(g, better?.model ?? "<another model>") : null });
     }
     // Timeouts.
-    if (g.timeout >= 3 && g.timeout / g.jobs >= 0.25) {
-      out.push({ level: "info", key: `timeouts:${g.role}:${g.model}`, title: `${who} timed out on ${g.timeout} of ${g.jobs} jobs`,
+    if (g.timeout >= 3 && g.timeout / g.rated >= 0.25) {
+      out.push({ level: "info", key: `timeouts:${g.role}:${g.model}`, title: `${who} timed out on ${g.timeout} of ${g.rated} jobs`,
         evidence: "Smaller briefs (one outcome each), a longer timeout_seconds, or a faster model would help.", command: null });
     }
   }
 
-  // A lighter model doing as well on implement work, for much less.
-  const impl = [...groups.values()].filter((g) => g.mode === "implement" && g.model && g.jobs >= minJobs && g.tokens > 0);
+  // A lighter model doing as well on the same role's implement work, for much less. Only
+  // within one role: different roles do different work, so across roles the numbers don't compare.
+  const impl = [...groups.values()].filter((g) => g.mode === "implement" && g.role && g.model && g.rated >= minJobs && g.tokenJobs >= minJobs);
   for (const heavy of impl) {
     for (const light of impl) {
-      if (light === heavy || light.model === heavy.model) continue;
-      const lightRate = light.ok / light.jobs, heavyRate = heavy.ok / heavy.jobs;
-      if (lightRate >= heavyRate - 0.05 && light.tokens / light.jobs <= 0.5 * (heavy.tokens / heavy.jobs)) {
+      if (light === heavy || light.role !== heavy.role || light.model === heavy.model) continue;
+      const lightRate = light.ok / light.rated, heavyRate = heavy.ok / heavy.rated;
+      const lightPerJob = light.tokens / light.tokenJobs, heavyPerJob = heavy.tokens / heavy.tokenJobs;
+      if (lightRate >= heavyRate - 0.05 && lightPerJob <= 0.5 * heavyPerJob) {
         out.push({ level: "info", key: `lighter:${heavy.role}:${heavy.model}:${light.model}`,
-          title: `${light.model} finished ${pct(light.ok, light.jobs)}% of its jobs${light.role ? ` (as ${light.role})` : ""} on ${Math.round((light.tokens / light.jobs) / 1000)}k tokens a job; ${heavy.model}${heavy.role ? ` (as ${heavy.role})` : ""} finished ${pct(heavy.ok, heavy.jobs)}% on ${Math.round((heavy.tokens / heavy.jobs) / 1000)}k`,
-          evidence: "They did different work, so try it rather than switch outright: put the role on auto so the General picks per job, or send its routine pieces to the lighter model.",
-          command: heavy.role ? `nomarmy army assign ${heavy.role} ${heavy.agent ?? "<agent>"} auto` : null });
+          title: `${heavy.role}: ${light.model} finished ${pct(light.ok, light.rated)}% of its jobs on ${Math.round(lightPerJob / 1000)}k new tokens a job; ${heavy.model} finished ${pct(heavy.ok, heavy.rated)}% on ${Math.round(heavyPerJob / 1000)}k`,
+          evidence: `Same role, so similar work. Moving ${heavy.role} to ${light.model} would cost less; keep ${heavy.model} for the harder pieces with model on the job.${light.agent === "local" ? ` The local agent runs whichever model is loaded; these ran on ${light.model}.` : ""}`,
+          command: light.agent ? `nomarmy army assign ${heavy.role} ${light.agent}${light.agent === "local" ? "" : ` ${light.model}`}` : null });
       }
     }
   }

package/package.json

@@ -3,7 +3,7 @@
   "description": "Every byte verified: a harness for AI coding workers whose claims are never trusted. Your coding assistant stays in charge while workers implement and test in sandboxes, and nomArmy checks every change before it is committed.",
   "author": "Rayson Technologies",
   "license": "Apache-2.0",
-  "version": "0.1.0-alpha.14",
+  "version": "0.1.0-alpha.15",
   "private": false,
   "type": "module",
   "engines": {