nomarmy 0.1.0-alpha.11 → 0.1.0-alpha.12

package/bin/nomarmy.mjs

@@ -20,6 +20,9 @@ import { readGGUFMetadata, resolveModelPath, totalSplitBytes } from "../lib/gguf
 import { recommend, customRecommendation, evaluateConfig, bytesPerKvElementForCacheTypes, MIN_CONTEXT_PER_NOM } from "../lib/sizing.mjs";
 import { connectClaude, connectCodex, connectCursor, cursorAlreadyConnected, deriveWorkerModelEnv, defaultInstallDir, installMcpCopy, SCOPES, claudeUserScoped, portableServerLaunch } from "../lib/connect.mjs";
 import { compareVersions, readPackageVersion, readInstallVersions, copyIsStale } from "../lib/install-freshness.mjs";
+import { loadJobRecords, computeStats, formatStats, parseSince, resolveRepo } from "../lib/stats.mjs";
+import { loadValidators, saveJevKey, removeJev, jevSettings, askJev, validatorsPath, JEV_CHECKS, saveJudge, removeJudge, judgeSettings } from "../lib/validators.mjs";
+import { probeModel } from "../lib/model-probe.mjs";
 import { ID_RE, AUTH_ENV_NAME_RE, OPENCLAW_PROVIDER_ID_RE, openclawProviderId, isNativeProviderType } from "../lib/dispatch-schema.mjs";
 import { loadAgents, readAgentsFile, writeAgentsFile, agentsConfigPath, apiAgentAsPoolEntry, describeAgent as describeAgentLabel, agentRunsToolsOnHost, agentProviderId, AGENT_KINDS, API_PROVIDER_TYPES, RESERVED_AGENT_NAMES, BUILTIN_LOCAL_AGENT } from "../lib/agents.mjs";
 import { loadArmy, mergeArmy, describeArmy, readArmyFile, updateArmyInFile, assignRoleInFile, parseTargetSpec, armyLayerPath, globalConfigDir, DEFAULT_ARMY, ARMY_PHASES, LOCAL_CONFIG_FILENAME } from "../lib/army.mjs";
@@ -222,6 +225,24 @@ Usage: nomarmy <command> [options]
                   project for this repository, committed for the team
                   (.mcp.json or .cursor/mcp.json, running \`nomarmy mcp\`).
                   Codex has only the user scope.
+  stats [--since 7d|<date>] [--until <date>] [--role <role>] [--model <model>]
+        [--repo <path|name>] [--all-repos] [--json]
+                  What nomArmy's job records show for this repository (or
+                  all): volume by role and model, code committed, time,
+                  tokens and spend, how often a "done" report failed
+                  independent verification, what didn't finish, reviewers,
+                  and review flags. From verified records, never reports.
+  validators <list|add jev|test jev|remove jev|add judge|test judge|remove judge>
+                  Optional semantic checks from a model you configure with
+                  your own key. Today: Jev (TypeSafe). \`add jev\` asks for the
+                  key without echoing it (or reads --key-stdin), saves it
+                  where only you can read it, and makes one test call. Its
+                  answers only add review flags, and it sends excerpts of
+                  your code to TypeSafe. \`add judge --agent <name> --model
+                  <model>\` makes one of your agents a model judge: does the
+                  diff meet each acceptance criterion, match the report, keep
+                  its tests as strong? An agent whose tools run on this
+                  machine needs --host-tools.
   mcp             Start nomArmy's MCP server on stdio with this machine's
                   settings. What a --scope project registration runs.
   sandbox         The Podman VM every sandbox shares (macOS, Windows): its
@@ -2646,6 +2667,135 @@ async function cmdStatusline() {
   process.stdout.write(`${statusLineText({ session })}\n`);
 }
 
+function cmdStats() {
+  const stateRoot = process.env.NOMARMY_AGENT_STATE || path.join(os.homedir(), ".local", "share", "nomarmy-local-agents");
+  const records = loadJobRecords(path.join(stateRoot, "jobs"));
+  let repo = null;
+  if (value("repo")) repo = resolveRepo(records, value("repo"));
+  else if (!flag("all-repos")) {
+    try { repo = execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd: repoDir, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); }
+    catch { throw new Error(`${repoDir} isn't inside a git repository; run nomarmy stats from one, or pass --repo <name> or --all-repos`); }
+  }
+  const stats = computeStats(records, { repo, sinceMs: parseSince(value("since")), untilMs: parseSince(value("until")), role: value("role"), model: value("model") });
+  if (json) return out(stats);
+  console.log(formatStats(stats));
+}
+
+// Read one line without echoing it: stty -echo around the read, restored
+// even if the read fails. Windows has no stty, so it says the input shows.
+async function readHiddenLine(prompt) {
+  const hide = process.stdin.isTTY && process.platform !== "win32";
+  if (!hide) console.log(c.yellow("(your input will be visible as you type)"));
+  const rl = createInterface({ input, output });
+  try {
+    if (hide) spawnSync("stty", ["-echo"], { stdio: ["inherit", "ignore", "ignore"] });
+    return (await rl.question(prompt)).trim();
+  } finally {
+    if (hide) { spawnSync("stty", ["echo"], { stdio: ["inherit", "ignore", "ignore"] }); process.stdout.write("\n"); }
+    rl.close();
+  }
+}
+
+async function readStdin() {
+  const chunks = [];
+  for await (const chunk of process.stdin) chunks.push(chunk);
+  return Buffer.concat(chunks).toString("utf8").trim();
+}
+
+// One tiny System One request, to prove the key and the route work.
+async function testJev(settings) {
+  const { answers } = await askJev({ key: settings.key, model: settings.model, state: { text: "The build finished and all 12 tests passed." },
+    questions: { passed: { type: "noul", instructions: "Does the text say the tests passed?", criteria: { true: "It says the tests passed", false: "It doesn't" } } } });
+  return typeof answers.passed?.noul === "number";
+}
+
+async function cmdValidators() {
+  const [sub = "list", name] = argv.slice(1).filter((a) => !a.startsWith("--"));
+  if (sub === "list") {
+    let config = {};
+    try { config = loadValidators(); } catch (error) { if (json) return out({ error: error.message }); console.log(c.red(error.message)); process.exitCode = 1; return; }
+    const judge = config.judge ? { enabled: config.judge.enabled, agent: config.judge.agent, model: config.judge.model, checks: config.judge.checks, hostTools: config.judge.host_tools } : null;
+    const jev = config.jev ? { enabled: config.jev.enabled, checks: config.jev.checks, model: config.jev.model, key: config.jev.key_env ? `env ${config.jev.key_env}` : config.jev.key_file, keyReadable: Boolean(jevSettings()) } : null;
+    if (json) return out({ path: validatorsPath(), jev, judge });
+    if (!jev && !judge) { console.log("No validators configured. Add one with: nomarmy validators add jev, or nomarmy validators add judge --agent <name> --model <model>"); return; }
+    if (jev) console.log(`Jev: ${jev.enabled ? c.green("on") : "off"} (${jev.model}); checks: ${jev.checks.join(", ")}; key: ${jev.key}${jev.keyReadable ? "" : c.red(" (not readable)")}`);
+    if (judge) console.log(`Judge: ${judge.enabled ? c.green("on") : "off"} (${judge.agent}/${judge.model}); checks: ${judge.checks.join(", ")}${judge.hostTools ? c.yellow("; host tools allowed") : ""}`);
+    return;
+  }
+  if (name === "judge") return cmdValidatorsJudge(sub);
+  if (name !== "jev") throw new Error("Usage: nomarmy validators <list|add jev|test jev|remove jev|add judge|test judge|remove judge>");
+  if (sub === "add") {
+    if (!json) {
+      console.log(c.bold("🍪 Jev (TypeSafe) for nomArmy's semantic checks\n"));
+      console.log("It checks that a scout's cited lines support its finding, and that a worker's report matches its diff.");
+      console.log("Its answers only add review flags; they never pass a check or allow a commit.");
+      console.log(c.yellow("It sends excerpts of your code (findings, cited lines, diffs, worker reports) to TypeSafe.\n"));
+    }
+    const key = flag("key-stdin") ? await readStdin() : await readHiddenLine("TypeSafe API key (not shown): ");
+    const saved = saveJevKey(key);
+    let ok = false, why = null;
+    try { ok = await testJev(jevSettings()); } catch (error) { why = error.message; }
+    if (json) return out({ saved: true, keyFile: saved.keyFile, configPath: saved.configPath, test: ok ? "pass" : "fail", reason: why });
+    console.log(c.green(`✓ Saved the key to ${saved.keyFile} (readable only by you) and turned Jev on in ${saved.configPath}.`));
+    console.log(ok ? c.green("✓ Test call answered. New jobs use it; restart open coordinator sessions to pick it up.") : c.red(`✗ Test call failed: ${why ?? "no answer"}. Check the key, then: nomarmy validators test jev`));
+    if (!ok) process.exitCode = 1;
+    return;
+  }
+  if (sub === "test") {
+    const settings = jevSettings();
+    if (!settings) throw new Error("Jev isn't configured, or its key isn't readable. Add it with: nomarmy validators add jev");
+    let ok = false, why = null;
+    try { ok = await testJev(settings); } catch (error) { why = error.message; }
+    if (json) return out({ test: ok ? "pass" : "fail", reason: why });
+    console.log(ok ? c.green("✓ Jev answered.") : c.red(`✗ Jev test call failed: ${why ?? "no answer"}`));
+    if (!ok) process.exitCode = 1;
+    return;
+  }
+  if (sub === "remove") {
+    const result = removeJev();
+    if (json) return out(result);
+    console.log(c.green(`✓ Jev is off${result.removedKey ? ", and its saved key is deleted" : ""}.`));
+    return;
+  }
+  throw new Error("Usage: nomarmy validators <list|add jev|test jev|remove jev>");
+}
+
+async function cmdValidatorsJudge(sub) {
+  const agents = loadAgents(globalConfigDir()).agents;
+  const resolve = () => judgeSettings({ agents, providerOf: agentProviderId, runsOnHost: agentRunsToolsOnHost });
+  const probe = async (settings) => probeModel({ provider: settings.provider, model: settings.model, stateRoot: process.env.NOMARMY_AGENT_STATE || path.join(os.homedir(), ".local", "share", "nomarmy-local-agents") });
+  if (sub === "add") {
+    const agent = value("agent"), model = value("model");
+    if (!agent || !model) throw new Error("Usage: nomarmy validators add judge --agent <name> --model <model> [--host-tools]");
+    if (!agents[agent]) throw new Error(`"${agent}" isn't an agent in agents.yml. Agents: ${Object.keys(agents).join(", ") || "(none)"}`);
+    if (agentRunsToolsOnHost(agents[agent]) && !flag("host-tools")) throw new Error(`agent "${agent}" runs its tools on this machine, and a judge reads text the worker wrote. Pass --host-tools to accept that, or pick a sandboxed agent (an api key, Codex, Muse).`);
+    const saved = saveJudge({ agent, model, hostTools: flag("host-tools") });
+    const settings = resolve();
+    if (settings?.problem) throw new Error(settings.problem);
+    const test = await probe(settings);
+    if (json) return out({ saved: true, configPath: saved.configPath, test: test.ok ? "pass" : test.refused ? "refused" : "inconclusive", reason: test.reason });
+    console.log(c.green(`✓ The judge is ${agent}/${model}, in ${saved.configPath}.`));
+    console.log(test.ok ? c.green("✓ Test call answered. New implement jobs use it; restart open coordinator sessions to pick it up.") : c.red(`✗ Test call ${test.refused ? "refused" : "didn't answer"}: ${test.reason ?? "no answer"}`));
+    if (!test.ok) process.exitCode = 1;
+    return;
+  }
+  if (sub === "test") {
+    const settings = resolve();
+    if (!settings) throw new Error("No judge configured. Add one with: nomarmy validators add judge --agent <name> --model <model>");
+    if (settings.problem) throw new Error(settings.problem);
+    const test = await probe(settings);
+    if (json) return out({ test: test.ok ? "pass" : "fail", reason: test.reason });
+    console.log(test.ok ? c.green(`✓ ${settings.agent}/${settings.model} answered.`) : c.red(`✗ ${test.reason ?? "no answer"}`));
+    if (!test.ok) process.exitCode = 1;
+    return;
+  }
+  if (sub === "remove") {
+    removeJudge();
+    return json ? out({ removed: true }) : console.log(c.green("✓ The judge is off."));
+  }
+  throw new Error("Usage: nomarmy validators <add judge --agent <name> --model <model> [--host-tools]|test judge|remove judge>");
+}
+
 // `nomarmy mcp`: what a --scope project registration runs. Nothing goes to
 // stdout but the server's own protocol.
 function cmdMcp() {
@@ -2655,7 +2805,7 @@ function cmdMcp() {
   child.on("exit", (code, signal) => { if (signal) process.kill(process.pid, signal); else process.exit(code ?? 1); });
 }
 
-const commands = { mcp: cmdMcp, scan: cmdScan, validate: cmdValidate, sizing: cmdSizing, init: cmdInit, setup: cmdSetup, install: cmdInstall, model: cmdModel, agents: cmdAgents, army: cmdArmy, jobs: cmdJobs, statusline: cmdStatusline, health: cmdHealth, config: cmdConfig, update: cmdUpdate, connect: cmdConnect, sandbox: cmdSandbox, start: cmdStart, stop: cmdStop, uninstall: cmdUninstall, help: () => usage(0) };
+const commands = { stats: cmdStats, validators: cmdValidators, mcp: cmdMcp, scan: cmdScan, validate: cmdValidate, sizing: cmdSizing, init: cmdInit, setup: cmdSetup, install: cmdInstall, model: cmdModel, agents: cmdAgents, army: cmdArmy, jobs: cmdJobs, statusline: cmdStatusline, health: cmdHealth, config: cmdConfig, update: cmdUpdate, connect: cmdConnect, sandbox: cmdSandbox, start: cmdStart, stop: cmdStop, uninstall: cmdUninstall, help: () => usage(0) };
 // doctor command
 async function cmdDoctor() {
   // Import lazily to avoid circular dependencies

package/lib/admission.mjs

@@ -126,6 +126,19 @@ export function createJobRuntime(deps) {
       try { return await fn(); } finally { slot.release(); }
     })();
   }
+  // Who ran the job and for whom, onto its finished record, so `nomarmy
+  // stats` can count by role and agent, and by repo for verify runs too
+  // (their records didn't carry the repo). Best-effort; never fails the job.
+  function stampJobRecord(entry, result) {
+    const file = path.join(result?.jobDir ?? path.join(jobsRoot, entry.jobId), "metadata.json");
+    try {
+      const record = JSON.parse(fs.readFileSync(file, "utf8"));
+      record.labels = { role: entry.role ?? null, agent: entry.agent ?? null, label: entry.label ?? null, runId: entry.runId ?? null };
+      record.projectDir = record.projectDir ?? projectDir;
+      fs.writeFileSync(`${file}.tmp`, JSON.stringify(record, null, 2));
+      fs.renameSync(`${file}.tmp`, file);
+    } catch { /* no record (a refused or crashed job): nothing to label */ }
+  }
   function track(jobId, meta, promise) {
     const entry = { ...meta, jobId, startedAt: new Date().toISOString(), settled: false, result: null, error: null, promise: null };
     // A machine-wide lease for as long as the job runs, so every session's
@@ -134,7 +147,7 @@ export function createJobRuntime(deps) {
     if (meta.lane) writeLease(leasesRoot, jobId, { lane: meta.lane, agent: meta.agent ?? null, runId: meta.runId ?? null, role: meta.role ?? null, model: meta.model ?? null, repo: projectDir });
     const release = () => removeLease(leasesRoot, jobId);
     entry.promise = promise.then(
-      r => { entry.settled = true; entry.result = r; release(); notifyJobFinished(entry, r, null); return r; },
+      r => { entry.settled = true; entry.result = r; release(); stampJobRecord(entry, r); notifyJobFinished(entry, r, null); return r; },
       e => { entry.settled = true; entry.error = e; release(); notifyJobFinished(entry, null, e); throw e; });
     entry.promise.catch(() => {});
     activeJobs.set(jobId, entry);

package/lib/execute.mjs

@@ -9,6 +9,9 @@ import { parseScoutReport, verifyCitations, resolveScoutOutcome, renderScoutRepo
 import { parseDecomposeReport, buildDecomposeFindings, resolveDecomposeOutcome, checkDecompositionOverlap, renderDecomposeReport } from "./decompose.mjs";
 import { deriveTimeBudget } from "./budget.mjs";
 import { continuationProblem, continuationBase, snapshotRetainedWork, applyRetainedWork, continuationNote } from "./continue-from.mjs";
+import { checkScoutCitations, checkReportClaims } from "./jev-checks.mjs";
+import { runJudge } from "./judge.mjs";
+import { pickMutants, runMutants, describeSurvivors } from "./mutation.mjs";
 import { estimateDisplacement } from "./transcript.mjs";
 import { outlineFile, findReferences } from "./repo-query.mjs";
 import { loadConfig } from "./config.mjs";
@@ -18,7 +21,7 @@ import { describeRecoveryChanges, reportRecoveryPrompt } from "./worker-prompt.m
 import { parseWorkerReport } from "./report.mjs";
 import { OUTCOMES, COORDINATOR_STATUS_BY_OUTCOME } from "./outcomes.mjs";
 import { resolveOutcome, finalText, workerMetadata, applyRefactorContract, applyVerificationPolicy } from "./outcome.mjs";
-import { isTestPath, isDocumentationPath, detectScopedTestSelectionRisk, detectUnwiredNewDefinitions, detectMislabeledTestNames, detectPossibleSecrets, detectVerificationInputChanges } from "./diff-checks.mjs";
+import { parseAddedLineNumbers, isTestPath, isDocumentationPath, detectScopedTestSelectionRisk, detectUnwiredNewDefinitions, detectMislabeledTestNames, detectPossibleSecrets, detectVerificationInputChanges } from "./diff-checks.mjs";
 
 // ---------------------------------------------------------------------------
 // Job status for polling. `status.json` is written at every phase transition
@@ -36,6 +39,8 @@ export function createExecutor(deps) {
     resolveReasoningApplied, recordedBudgets, runOpenClaw, sweepStaleSandboxContainers,
     verificationFlow, normalizeVerification, runIndependentVerification,
     runRegressionCheck, repoPolicy } = deps;
+  // Optional Jev checks (lib/validators.mjs): the server wires the settings; without them (tests), none run.
+  const jevSettingsFor = (check) => { try { const s = deps.jevSettings?.(); return s?.checks?.includes(check) ? s : null; } catch { return null; } };
 
   async function executeJob({ task, acceptance, verification, mode = "implement", baseRef, timeoutSeconds = 600, profile = "coder", reasoning = "high", pool = null, subscriptionWorker = null, onBehalfOf = null, model = null, reportSize = null, workerId, evidence = null, verifyRegression = false, commitSubject = null, refactor = false, continueFrom = null, jobId: presetJobId = null }) {
     await assertRepo();
@@ -300,6 +305,36 @@ export function createExecutor(deps) {
         if (regressionCheck.status === "restore_failed") regressionCheckFatal = true;
       }
 
+      // Mutation testing (lib/mutation.mjs), when this repo opts in: small
+      // mistakes planted one at a time in the changed lines must each fail
+      // the same profile. Survivors raise review; a failed restore is fatal.
+      let mutation = null, mutationElapsedMs = null;
+      const mutationConfig = (() => { try { return loadConfig(projectDir)?.config?.mutation ?? null; } catch { return null; } })();
+      if (mutationConfig && verification && independentVerification.status === "pass" && codeFilesChanged.length > 0 && !regressionCheckFatal) {
+        const mutationStartedMs = Date.now();
+        try {
+          const untracked = new Set((preCommit.nameStatus ?? []).filter((e) => e.untracked).map((e) => e.path));
+          const deleted = new Set((preCommit.nameStatus ?? []).filter((e) => /^D/.test(e.status)).map((e) => e.path));
+          const files = [];
+          for (const file of codeFilesChanged.filter((f) => !deleted.has(f))) {
+            const full = path.join(cwd, file);
+            let lines;
+            if (untracked.has(file)) { try { lines = fs.readFileSync(full, "utf8").split("\n").map((_, i) => i + 1); } catch { continue; } }
+            else lines = parseAddedLineNumbers(await gitRaw(["diff", "-U0", base.sha, "--", file], cwd));
+            if (lines.length) files.push({ path: file, full, lines });
+          }
+          const mutants = pickMutants(files, mutationConfig.mutants);
+          let n = 0;
+          mutation = await runMutants({ mutants, deadlineMs: mutationStartedMs + mutationConfig.max_seconds * 1000,
+            verify: () => runIndependentVerification({ profile: verification, cwd, jobId: `${jobId}-mutant-${++n}`, baseSha: base.sha, branch, mode, record: preCommit }) });
+          mutation.planned = mutants.length;
+        } catch (error) {
+          mutation = { status: "not_run", killed: 0, survived: [], inconclusive: 0, tried: 0, reason: `mutation testing failed to run: ${error.message}` };
+        }
+        mutationElapsedMs = Date.now() - mutationStartedMs;
+        fs.appendFileSync(path.join(jobDir, "coordinator.log"), `${new Date().toISOString()} mutation testing: ${mutation.killed ?? 0} killed, ${mutation.survived?.length ?? 0} survived, ${mutation.inconclusive ?? 0} inconclusive of ${mutation.tried ?? 0} tried (${Math.round(mutationElapsedMs / 1000)}s)\n`);
+      }
+
       // resolveOutcome's own contract only ever sees pass/fail/not_run for
       // regressionCheck -- a restore_failed status is substituted to not_run
       // here so resolveOutcome never needs a fourth value; the hard override
@@ -427,21 +462,67 @@ export function createExecutor(deps) {
             commitBlockedReason: `possible secret detected: ${possibleSecrets.reason}`,
             reasons: [...afterHostInstalls.reasons, `POSSIBLE SECRET DETECTED: ${possibleSecrets.reason}`] }
         : afterHostInstalls;
+      // Jev: does the worker's report match its diff? Found live: a note said
+      // "restored check.js to base commit" while the diff rewrote check.js.
+      // Only raises review; it never blocks or allows a commit.
+      let jevClaims = null, judged = null;
+      const jevImplement = jevSettingsFor("report-claims");
+      const judge = (() => { try { return deps.judgeSettings?.() ?? null; } catch { return null; } })();
+      // The whole change, new files included, for whichever validators run.
+      const jobDiff = async () => {
+        let diff = await gitRaw(["diff", base.sha, "--"], cwd);
+        for (const entry of (preCommit.nameStatus ?? []).filter((e) => e.untracked)) {
+          let text = "";
+          try { text = fs.readFileSync(path.join(cwd, entry.path), "utf8"); } catch { continue; }
+          diff += `\ndiff --git a/${entry.path} b/${entry.path}\nnew file\n--- /dev/null\n+++ b/${entry.path}\n${text.split("\n").map((l) => `+${l}`).join("\n")}\n`;
+        }
+        return diff;
+      };
+      let diffText = null;
+      if (mode === "implement" && reportValidation?.valid && repositoryChanged && (jevImplement || (judge && !judge.problem))) {
+        try { diffText = await jobDiff(); } catch { diffText = null; }
+      }
+      if (jevImplement && diffText != null) {
+        try { jevClaims = await checkReportClaims({ report: reportValidation, diff: diffText, settings: jevImplement }); }
+        catch (error) { jevClaims = { flag: null, verdict: null, error: error.message, usage: 0 }; }
+      }
+      // The model judge (lib/judge.mjs): acceptance criteria, the report and
+      // changed tests. Only raises review, never blocks or allows a commit.
+      if (judge?.problem) judged = { flags: [], answer: null, error: `judge not run: ${judge.problem}`, skipped: true };
+      else if (judge && diffText != null) {
+        const modifiedTests = preCommit.testChanges?.existing_tests_modified ?? [];
+        let testDiff = "";
+        if (modifiedTests.length) { try { testDiff = await gitRaw(["diff", base.sha, "--", ...modifiedTests], cwd); } catch { testDiff = ""; } }
+        judged = await runJudge({ settings: judge, task, acceptance: acceptance ?? [], report: reportValidation, diff: diffText, testDiff, stateRoot: path.join(jobsRoot, "..") });
+      }
+      const afterJevOnly = jevClaims?.flag
+        ? { ...afterSecrets, reviewRequired: true, reasons: [...afterSecrets.reasons, `REPORT MAY NOT MATCH THE DIFF (Jev, ${jevClaims.flag.probability.toFixed(2)}): the report says "${String(reportValidation.note ?? "").slice(0, 200)}", and the diff may show otherwise. Read the diff before accepting.`] }
+        : afterSecrets;
+      const afterJev = judged?.flags?.length
+        ? { ...afterJevOnly, reviewRequired: true, reasons: [...afterJevOnly.reasons, `JUDGE (${judge.agent}/${judge.model}): ${judged.flags.join("; ")}. Read the diff before accepting.`] }
+        : afterJevOnly;
+
       // A worker must not be judged by a check it rewrote: a changed script,
       // Makefile or package.json script that a verification command runs
       // blocks the commit; changed test-runner config only asks for review.
       const blockedInputs = verificationInputs?.blocked ?? [], flaggedInputs = verificationInputs?.flagged ?? [];
       const inputLine = blockedInputs.map((b) => `${b.file} (${b.why})`).join("; ");
       const afterInputs = blockedInputs.length
-        ? { ...afterSecrets, outcome: afterSecrets.commitAllowed || afterSecrets.outcome === OUTCOMES.WORKER_DONE ? OUTCOMES.NEEDS_REVIEW : afterSecrets.outcome,
+        ? { ...afterJev, outcome: afterJev.commitAllowed || afterJev.outcome === OUTCOMES.WORKER_DONE ? OUTCOMES.NEEDS_REVIEW : afterJev.outcome,
             reviewRequired: true, commitAllowed: false,
-            commitBlockedReason: afterSecrets.commitAllowed ? `the diff changes what verification runs: ${inputLine}` : afterSecrets.commitBlockedReason,
-            reasons: [...afterSecrets.reasons, `VERIFICATION INPUT CHANGED: the diff changes what profile '${verification}' runs, so its result can't be trusted: ${inputLine}`] }
-        : afterSecrets;
+            commitBlockedReason: afterJev.commitAllowed ? `the diff changes what verification runs: ${inputLine}` : afterJev.commitBlockedReason,
+            reasons: [...afterJev.reasons, `VERIFICATION INPUT CHANGED: the diff changes what profile '${verification}' runs, so its result can't be trusted: ${inputLine}`] }
+        : afterJev;
       const afterConfig = flaggedInputs.length
         ? { ...afterInputs, reviewRequired: true, reasons: [...afterInputs.reasons, `TEST CONFIG CHANGED: ${flaggedInputs.map((c) => `${c.file} (${c.why})`).join("; ")}`] }
         : afterInputs;
-      const finalOutcome = applyRefactorContract(applyVerificationPolicy(afterConfig, independentVerification.status, repoPolicy()),
+      const afterMutation = mutation?.status === "restore_failed"
+        ? { ...afterConfig, outcome: OUTCOMES.NEEDS_REVIEW, reviewRequired: true, commitAllowed: false,
+            commitBlockedReason: `mutation testing could not restore the worker's file: ${mutation.reason}`, reasons: [...afterConfig.reasons, `MUTATION RESTORE FAILED: ${mutation.reason}`] }
+        : mutation?.status === "survivors"
+          ? { ...afterConfig, reviewRequired: true, reasons: [...afterConfig.reasons, describeSurvivors(mutation, verification)] }
+          : afterConfig;
+      const finalOutcome = applyRefactorContract(applyVerificationPolicy(afterMutation, independentVerification.status, repoPolicy()),
         { refactor, verificationStatus: independentVerification.status, testChanges: preCommit.testChanges });
 
       progress("commit");
@@ -454,6 +535,8 @@ export function createExecutor(deps) {
       const issues = [...finalOutcome.reasons, ...(preCommit.issues ?? [])];
       if (workerError) issues.push(`worker error: ${String(workerError).split("\n")[0]}`);
       if ((result ?? attempted)?.salvaged) issues.push(`runner cleanup failed after the run (${(result ?? attempted).salvagedFrom}); the worker's report was recovered from the run's transcript`);
+      if (jevClaims?.error) issues.push(`Jev report check skipped (${jevClaims.error}); this job's result doesn't depend on it`);
+      if (judged?.error) issues.push(`Judge ${judged.skipped ? "skipped" : "didn't answer"} (${judged.error}); this job's result doesn't depend on it`);
       if (workerFailed || workerTimedOut) { const restarted = vmRestartIssue(vmStartedBefore, deps.podmanVmStartedAt?.() ?? null); if (restarted) issues.unshift(restarted); }
       if (repositoryChanged && !commit.created) {
         if (coordinatorStatus === "complete") coordinatorStatus = "incomplete";
@@ -483,6 +566,8 @@ export function createExecutor(deps) {
       const metrics = buildMetrics({ result: result ?? attempted, record, reportValidation, outcome: finalOutcome, workerElapsedMs, totalElapsedMs: Date.now() - jobStartedMs, regressionCheckElapsedMs, transientAbortRetried });
       const manifest = { version: VERSION, jobId, workerId: workerId || jobId, mode, projectDir, worktree, branch, startedAt, finishedAt,
         objective: task, acceptance: acceptance ?? [], verificationProfile: verification ?? null, ...(continuedFrom ? { continuedFrom } : {}),
+        ...(mutation ? { mutation: { ...mutation, elapsedSeconds: Math.round((mutationElapsedMs ?? 0) / 1000) } } : {}),
+        ...(jevClaims || judged ? { validators: { ...(jevClaims ? { jev: { check: "report-claims", verdict: jevClaims.verdict, flagged: Boolean(jevClaims.flag), error: jevClaims.error, truncated: Boolean(jevClaims.truncated), inputTokens: jevClaims.usage } } : {}), ...(judged ? { judge: { agent: judge?.agent ?? null, model: judge?.model ?? null, answer: judged.answer, flags: judged.flags, error: judged.error } } : {}) } } : {}),
         outcome: finalOutcome.outcome, recovered: finalOutcome.recovered, recoveryAttempted: finalOutcome.recoveryAttempted,
         reportRecoveryAttempted, reportRecovered,
         reviewRequired: finalOutcome.reviewRequired || record.testChanges.reviewRequired,
@@ -609,7 +694,16 @@ export function createExecutor(deps) {
       const dirty = record.repoStatusFiles.length > 0;
       const readFile = async p => { try { return await gitRaw(["show", `${base.sha}:${p}`], projectDir); } catch { return null; } };
       const verified = await verifyCitations(report.findings, { readFile, limits: used.scout });
+      // Jev: do the cited lines support each finding? Only adds flags.
+      let jevCitations = null;
+      const jevScout = jevSettingsFor("scout-citations");
+      if (jevScout && verified?.findings?.length) {
+        try { jevCitations = await checkScoutCitations({ findings: verified.findings, settings: jevScout, readFile }); }
+        catch (error) { jevCitations = { flags: [], checked: 0, errors: [error.message], usage: 0, verdicts: [] }; }
+        for (const f of jevCitations.flags) verified.findings[f.index].jev = { verdict: f.verdict, probability: f.probability };
+      }
       const outcome = resolveScoutOutcome({ report, verified, workerFailed, workerTimedOut, dirty });
+      if (jevCitations?.flags.length) outcome.reviewRequired = true;
 
       progress("record");
       if (outcome.retainWorktree) worktreeRetained = true;
@@ -621,6 +715,8 @@ export function createExecutor(deps) {
       if ((result ?? attempted)?.salvaged) issues.push(`runner cleanup failed after the run (${(result ?? attempted).salvagedFrom}); the scout's report was recovered from the run's transcript`);
       const failures = worker.toolSummary?.failures ?? 0; if (failures > 0) issues.push(`scout recorded ${failures} tool failure(s)`);
       if (dirty) issues.push(`snapshot changed: ${record.repoStatusFiles.join(", ")}`);
+      if (jevCitations?.flags.length) issues.push(`CITATIONS MAY NOT SUPPORT FINDINGS (Jev): ${jevCitations.flags.map((f) => `"${String(verified.findings[f.index].text).slice(0, 80)}${String(verified.findings[f.index].text).length > 80 ? "..." : ""}" (${f.verdict}, ${f.probability.toFixed(2)})`).join("; ")}. Read those cited lines before relying on them; they're marked [JEV] in the report.`);
+      if (jevCitations?.errors.length) issues.push(`Jev citation check skipped or incomplete (${jevCitations.errors.join("; ")}); this job's result doesn't depend on it`);
       if (reportRecoveryAttempted) {
         issues.push(reportRecovered
           ? "scout report recovered via a follow-up call after the first reply was cut off"
@@ -658,6 +754,7 @@ export function createExecutor(deps) {
       };
       const manifest = { version: VERSION, jobId, workerId: workerId || jobId, mode, projectDir, worktree: worktreeRetained ? worktree : null, branch: null, baseSha: base.sha, startedAt, finishedAt,
         objective: task, mustCover: acceptance ?? [],
+        ...(jevCitations ? { validators: { jev: { check: "scout-citations", checked: jevCitations.checked, flags: jevCitations.flags, errors: jevCitations.errors, inputTokens: jevCitations.usage } } } : {}),
         outcome: outcome.outcome, coordinatorStatus: outcome.coordinatorStatus, reviewRequired: outcome.reviewRequired, issues,
         scout: { question: report.question, confidence: report.confidence, notFound: report.notFound,
           findings: verified.findings, supported: verified.supported, unsupported: verified.unsupported, weak: verified.weak,

package/lib/jev-checks.mjs

@@ -0,0 +1,110 @@
+// nomArmy's Jev checks (see lib/validators.mjs): narrow judgments on evidence
+// nomArmy already has, each able only to add a review flag.
+//
+//   scout-citations  nomArmy verifies a scout's [path:line] exists and attaches
+//                    the lines; this asks whether those lines support the
+//                    finding (the shape of TypeSafe's citation-check cookbook).
+//   report-claims    whether a worker's report matches its diff. Found live: a
+//                    worker's note said "restored check.js to base commit"
+//                    while its diff rewrote check.js.
+
+import { askJev, jevBreaker, tripJevBreaker } from "./validators.mjs";
+
+// A flag needs the model to lean clearly; the rest is left to the General.
+export const FLAG_AT = 0.7;
+const DIFF_CHARS = 60000;           // well inside the 32k-token state budget
+const MAX_FINDINGS = 24;
+// The report's excerpts are capped at 12 lines for the General's context; Jev
+// judges the whole cited range (to this cap), or a long range looks
+// unrelated when the supporting line is past the excerpt.
+const CITED_LINES = 80;
+const CONCURRENCY = 4;
+// However many findings, a job never waits longer than this on Jev.
+const JOB_BUDGET_MS = 45000;
+
+// The answer, or null with the breaker tripped: any failure means TypeSafe
+// isn't answering well right now, so stop asking (lib/validators.mjs).
+async function guarded(ask, request, errors) {
+  if (jevBreaker().open) { errors.push(`skipped: Jev failed recently (${jevBreaker().reason}); retrying after ${jevBreaker().retryAt}`); return null; }
+  try { return await ask(request); }
+  catch (error) { const why = error?.name === "AbortError" ? "timed out" : error.message; tripJevBreaker(why); errors.push(why); return null; }
+}
+
+async function mapLimit(items, limit, fn) {
+  const out = new Array(items.length);
+  let next = 0;
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
+    while (next < items.length) { const i = next++; out[i] = await fn(items[i], i); }
+  }));
+  return out;
+}
+
+const SUPPORT_QUESTION = {
+  type: "choice",
+  instructions: "A code researcher made the claim in `finding` and cited the source lines in `cited` as evidence. Judge only what the cited lines show, not whether the claim might be true elsewhere in the codebase. How do the cited lines relate to the claim?",
+  criteria: {
+    supports: "The cited lines directly show what the claim says.",
+    contradicts: "The cited lines show something different from, or opposite to, the claim.",
+    unrelated: "The cited lines don't address the claim: they're about something else, or too little of the claim is visible in them.",
+  },
+};
+
+/**
+ * For each finding with verified cited lines: does the evidence support it?
+ * @returns {Promise<{ flags: {index: number, verdict: string, probability: number}[], checked: number, errors: string[], usage: number }>}
+ */
+export async function checkScoutCitations({ findings = [], settings, ask = askJev, readFile = null }) {
+  const fileCache = new Map();
+  const fullRange = async (c) => {
+    if (!readFile) return null;
+    if (!fileCache.has(c.path)) fileCache.set(c.path, readFile(c.path).then((t) => (typeof t === "string" ? t.split("\n") : null)).catch(() => null));
+    const lines = await fileCache.get(c.path);
+    if (!lines) return null;
+    const end = Math.min(c.end, c.start + CITED_LINES - 1, lines.length);
+    return Array.from({ length: Math.max(0, end - c.start + 1) }, (_, k) => `${c.start + k}: ${lines[c.start + k - 1]}`).join("\n");
+  };
+  const candidates = findings.map((f, index) => ({ f, index }))
+    .filter(({ f }) => (f.citations ?? []).some((c) => c.status === "ok" && c.excerpt?.length)).slice(0, MAX_FINDINGS);
+  const errors = [];
+  let usage = 0;
+  const deadline = Date.now() + JOB_BUDGET_MS;
+  const verdicts = await mapLimit(candidates, CONCURRENCY, async ({ f, index }) => {
+    if (Date.now() > deadline) { errors.push("skipped the rest: over the job's time budget for Jev"); return null; }
+    const cited = await Promise.all(f.citations.filter((c) => c.status === "ok" && c.excerpt?.length)
+      .map(async (c) => ({ file: c.path, lines: `${c.start}-${c.end}`, text: (await fullRange(c)) ?? c.excerpt.map((l) => `${l.line}: ${l.text}`).join("\n") })));
+    const r = await guarded(ask, { key: settings.key, model: settings.model, state: { finding: f.text, cited }, questions: { support: SUPPORT_QUESTION } }, errors);
+    if (!r) return null;
+    usage += r.usage?.input_tokens ?? 0;
+    const a = r.answers?.support;
+    return a ? { index, verdict: a.choice, probability: a.probabilities?.[a.choice] ?? 0 } : null;
+  });
+  const flags = verdicts.filter((v) => v && v.verdict !== "supports" && v.probability >= FLAG_AT);
+  return { flags, checked: verdicts.filter(Boolean).length, errors: [...new Set(errors)].slice(0, 3), usage, verdicts: verdicts.filter(Boolean) };
+}
+
+const CLAIMS_QUESTION = {
+  type: "choice",
+  instructions: "A coding worker wrote the report in `report` about the change it made. `diff` shows every change from the base commit it started from: a file that isn't in the diff is exactly as it was at the base commit, and a file restored to the base commit wouldn't appear. Judge only the concrete things the report says were done or changed. Does the diff match them?",
+  criteria: {
+    consistent: "The concrete things the report says were done are what the diff shows.",
+    contradicts: "The diff shows a concrete claim in the report is false: something said to be done, restored, removed or left alone wasn't, or was done differently.",
+    unclear: "The report makes no concrete claim the diff can confirm or refute, or the diff shown is too partial to tell.",
+  },
+};
+
+/**
+ * Does the worker's report match its diff?
+ * @returns {Promise<{ flag: {verdict: string, probability: number}|null, verdict: object|null, error: string|null, usage: number, truncated: boolean }>}
+ */
+export async function checkReportClaims({ report, diff, settings, ask = askJev }) {
+  const note = [report?.note, report?.notDone && report.notDone !== "none" ? `Not done: ${report.notDone}` : null].filter(Boolean).join("\n");
+  if (!note.trim() || !String(diff ?? "").trim()) return { flag: null, verdict: null, error: null, usage: 0, truncated: false };
+  const truncated = diff.length > DIFF_CHARS;
+  const state = { report: { status: report.status ?? null, tests: report.tests ?? null, note }, diff: truncated ? `${diff.slice(0, DIFF_CHARS)}\n[diff truncated]` : diff };
+  const errors = [];
+  const r = await guarded(ask, { key: settings.key, model: settings.model, state, questions: { claims: CLAIMS_QUESTION } }, errors);
+  if (!r) return { flag: null, verdict: null, error: errors[0] ?? "no answer", usage: 0, truncated };
+  const a = r.answers?.claims;
+  const verdict = a ? { verdict: a.choice, probability: a.probabilities?.[a.choice] ?? 0 } : null;
+  return { flag: verdict && verdict.verdict === "contradicts" && verdict.probability >= FLAG_AT ? verdict : null, verdict, error: null, usage: r.usage?.input_tokens ?? 0, truncated };
+}