nomarmy 0.1.0-alpha.2 → 0.1.0-alpha.21

package/lib/execute.mjs

@@ -1,13 +1,18 @@
-import { writeStatus, shouldRetryTransientAbort, shouldAttemptScoutRecovery } from "./openclaw-run.mjs";
+import { writeStatus, shouldRetryTransientAbort, shouldAttemptScoutRecovery, readStopRequest } from "./openclaw-run.mjs";
 import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
+import { vmRestartIssue } from "./podman-health.mjs";
 import { measureReads } from "./job-budgets.mjs";
 import { coordinatorCommitMessage, worktreePointerState } from "./git-record.mjs";
 import { parseScoutReport, verifyCitations, resolveScoutOutcome, renderScoutReport, isScoutReportUnusable, scoutReportRecoveryPrompt } from "./scout.mjs";
 import { parseDecomposeReport, buildDecomposeFindings, resolveDecomposeOutcome, checkDecompositionOverlap, renderDecomposeReport } from "./decompose.mjs";
 import { deriveTimeBudget } from "./budget.mjs";
-import { estimateDisplacement } from "./transcript.mjs";
+import { continuationProblem, continuationBase, snapshotRetainedWork, applyRetainedWork, continuationNote } from "./continue-from.mjs";
+import { checkScoutCitations, checkReportClaims } from "./jev-checks.mjs";
+import { runJudge } from "./judge.mjs";
+import { pickMutants, runMutants, describeSurvivors } from "./mutation.mjs";
+import { estimateDisplacement, readOpenClawTranscript } from "./transcript.mjs";
 import { outlineFile, findReferences } from "./repo-query.mjs";
 import { loadConfig } from "./config.mjs";
 import { linkNodePackages, nodeModulesState, repairHostInstalls } from "./sandbox-images.mjs";
@@ -15,8 +20,8 @@ import { detectTestSabotage, addedLinesOf, loadDependencyNames } from "./sabotag
 import { describeRecoveryChanges, reportRecoveryPrompt } from "./worker-prompt.mjs";
 import { parseWorkerReport } from "./report.mjs";
 import { OUTCOMES, COORDINATOR_STATUS_BY_OUTCOME } from "./outcomes.mjs";
-import { resolveOutcome, finalText, workerMetadata, applyRefactorContract, applyVerificationPolicy } from "./outcome.mjs";
-import { isTestPath, isDocumentationPath, detectScopedTestSelectionRisk, detectUnwiredNewDefinitions, detectMislabeledTestNames, detectPossibleSecrets } from "./diff-checks.mjs";
+import { resolveOutcome, finalText, workerMetadata, applyRefactorContract, applyVerificationPolicy, HIGH_STAKES_NOTE } from "./outcome.mjs";
+import { parseAddedLineNumbers, isTestPath, planRegressionProductionFiles, detectScopedTestSelectionRisk, detectUnwiredNewDefinitions, detectMislabeledTestNames, detectPossibleSecrets, detectVerificationInputChanges } from "./diff-checks.mjs";
 
 // ---------------------------------------------------------------------------
 // Job status for polling. `status.json` is written at every phase transition
@@ -34,13 +39,23 @@ export function createExecutor(deps) {
     resolveReasoningApplied, recordedBudgets, runOpenClaw, sweepStaleSandboxContainers,
     verificationFlow, normalizeVerification, runIndependentVerification,
     runRegressionCheck, repoPolicy } = deps;
+  // Optional Jev checks (lib/validators.mjs): the server wires the settings; without them (tests), none run.
+  const jevSettingsFor = (check) => { try { const s = deps.jevSettings?.(); return s?.checks?.includes(check) ? s : null; } catch { return null; } };
 
-  async function executeJob({ task, acceptance, verification, mode = "implement", baseRef, timeoutSeconds = 600, profile = "coder", reasoning = "high", pool = null, subscriptionWorker = null, onBehalfOf = null, model = null, reportSize = null, workerId, evidence = null, verifyRegression = false, commitSubject = null, refactor = false, jobId: presetJobId = null }) {
+  async function executeJob({ task, acceptance, verification, mode = "implement", baseRef, timeoutSeconds = 600, profile = "coder", reasoning = "high", pool = null, subscriptionWorker = null, onBehalfOf = null, model = null, reportSize = null, workerId, evidence = null, verifyRegression = false, commitSubject = null, refactor = false, continueFrom = null, stakes = null, reviews = null, jobId: presetJobId = null }) {
     await assertRepo();
+    // A continuation starts from the retained job's own base commit.
+    let continuation = null;
+    if (continueFrom) {
+      const checked = continuationProblem({ continueFrom, mode, baseRef, jobsRoot, projectDir });
+      if (checked.problem) throw new Error(checked.problem);
+      continuation = { jobId: continueFrom, record: checked.record };
+      baseRef = continuationBase(checked.record);
+    }
     ensureJobsRoot();
     // Fire-and-forget: sweeps whatever this or any other nomArmy install left
     // behind, without adding container-CLI round-trip latency to this job's own start.
-    sweepStaleSandboxContainers().catch(() => {});
+    if (mode !== "verify") sweepStaleSandboxContainers().catch(() => {});
     const jobStartedMs = Date.now();
     const base = await resolveBase(baseRef), jobId = presetJobId || slug(workerId || (mode === "scout" ? "scout" : mode === "decompose" ? "decompose" : "worker")), jobDir = path.join(jobsRoot, jobId), runtimeDir = path.join(jobDir, "runtime");
     fs.mkdirSync(runtimeDir, { recursive: true });
@@ -48,20 +63,65 @@ export function createExecutor(deps) {
       jobId, workerId: workerId || jobId, mode, phase, state: phase === "finished" ? "finished" : "running",
       serverPid: process.pid, baseSha: base.sha, timeoutSeconds, ...extra
     });
-    progress("starting", { startedAt: new Date().toISOString(), agent: pool ?? subscriptionWorker ?? "local", model: model ?? null });
+    progress("starting", { startedAt: new Date().toISOString(), agent: mode === "verify" ? null : pool ?? subscriptionWorker ?? "local", model: model ?? null });
     const common = { task, acceptance, base, jobId, jobDir, runtimeDir, timeoutSeconds, profile, reasoning, pool, subscriptionWorker, onBehalfOf, model, reportSize, workerId, progress, jobStartedMs };
-    if (mode === "scout") return executeScout(common);
+    if (mode === "verify") return executeVerify({ ...common, verification });
+    if (mode === "scout") return executeScout({ ...common, reviews });
     if (mode === "decompose") return executeDecompose(common);
-    return executeImplement({ ...common, verification, evidence, verifyRegression, commitSubject, refactor });
+    return executeImplement({ ...common, verification, evidence, verifyRegression, commitSubject, refactor, continuation, stakes });
+  }
+
+  async function executeVerify({ task, verification: profile, base, jobId, jobDir, workerId, progress, jobStartedMs }) {
+    const worktree = path.join(jobDir, "worktree");
+    let verification, record = null, error = null;
+    try {
+      progress("worktree");
+      await run("git", ["worktree", "add", "--detach", worktree, base.sha], { cwd: projectDir });
+      record = await collectGitRecord({ cwd: worktree, baseSha: base.sha, baseRef: base.ref, branch: null, jobId });
+      progress("verification");
+      verification = await runIndependentVerification({ profile, cwd: worktree, jobId, baseSha: base.sha, branch: null, mode: "verify", record, logFile: path.join(jobDir, "verification.log") });
+    } catch (err) {
+      error = err.message;
+      verification = normalizeVerification({ status: "not_run", reason: error }, profile);
+    } finally {
+      try { await run("git", ["worktree", "remove", "--force", worktree], { cwd: projectDir }); }
+      catch (err) { error = [error, err.message].filter(Boolean).join("\n"); }
+    }
+    const retained = fs.existsSync(worktree);
+    const outcome = retained ? OUTCOMES.VERIFICATION_NOT_RUN : {
+      pass: OUTCOMES.VERIFIED, fail: OUTCOMES.VERIFICATION_FAILED, not_run: OUTCOMES.VERIFICATION_NOT_RUN
+    }[verification.status];
+    const manifest = { version: VERSION, jobId, workerId: workerId || jobId, mode: "verify", task,
+      baseRef: base.ref, baseSha: base.sha, branch: null, worktree: retained ? worktree : null, worktreeRetained: retained,
+      startedAt: new Date(jobStartedMs).toISOString(), finishedAt: new Date().toISOString(),
+      outcome, coordinatorStatus: COORDINATOR_STATUS_BY_OUTCOME[outcome], verification, git: record, error,
+      // Network access (step 9) and skipped registry credentials (step 8) both surface as issues.
+      issues: [...new Set([...(verification.network ? verification.issues ?? [] : []), ...(record?.issues ?? [])])],
+      metrics: { total_elapsed: Date.now() - jobStartedMs, worker_cost_usd: 0, worker_tokens_total: 0, model_calls: 0 } };
+    fs.writeFileSync(path.join(jobDir, "metadata.json"), JSON.stringify(manifest, null, 2));
+    progress("finished", { outcome, coordinatorStatus: manifest.coordinatorStatus });
+    return { ok: outcome === OUTCOMES.VERIFIED, manifest, jobDir, report: "" };
   }
 
-  async function executeImplement({ task, acceptance, verification, base, jobId, jobDir, runtimeDir, timeoutSeconds, profile, reasoning, pool = null, subscriptionWorker = null, onBehalfOf = null, model = null, reportSize = null, workerId, evidence, verifyRegression = false, commitSubject = null, refactor = false, progress, jobStartedMs }) {
+  async function executeImplement({ task, acceptance, verification, base, jobId, jobDir, runtimeDir, timeoutSeconds, profile, reasoning, pool = null, subscriptionWorker = null, onBehalfOf = null, model = null, reportSize = null, workerId, evidence, verifyRegression = false, commitSubject = null, refactor = false, continuation = null, stakes = null, progress, jobStartedMs }) {
     const mode = "implement";
     let branch = `agent/${jobId}`, worktree = path.join(jobDir, "worktree");
     try {
       progress("worktree");
       await run("git", ["worktree", "add", "-b", branch, worktree, base.sha], { cwd: projectDir });
       const cwd = worktree;
+      // continue_from: lay the retained job's unfinished work into this
+      // worktree as uncommitted changes, so this job's diff (and so its
+      // verification and revert check) covers that work too.
+      let continuedFrom = null;
+      if (continuation) {
+        const git = async (args, opts = {}) => (await run("git", args, { trim: false, ...opts })).stdout;
+        const snapshot = await snapshotRetainedWork({ worktree: continuation.record.worktree, baseSha: base.sha, jobId: continuation.jobId, git });
+        await applyRetainedWork({ worktree, baseSha: base.sha, commit: snapshot.commit, git });
+        continuedFrom = { jobId: continuation.jobId, snapshot: snapshot.commit, files: snapshot.files };
+        evidence = [continuationNote({ continueFrom: continuation.jobId, record: continuation.record, files: snapshot.files }), evidence].filter(Boolean).join("\n\n");
+        fs.appendFileSync(path.join(jobDir, "coordinator.log"), `${new Date().toISOString()} continuing job ${continuation.jobId}: ${snapshot.files.length} file(s) of its unfinished work carried into this worktree (snapshot ${snapshot.commit})\n`);
+      }
       // Each npm package below the root reaches its install in the sandbox image.
       const nodeConfig = (() => { try { return loadConfig(projectDir)?.config ?? null; } catch { return null; } })();
       try { linkNodePackages(worktree, nodeConfig); } catch { /* verification reports what's missing */ }
@@ -79,6 +139,7 @@ export function createExecutor(deps) {
       const timeBudget = deriveTimeBudget({ timeoutSeconds });
       let result = null, attempted = null, workerFailed = false, workerTimedOut = false, workerStopReason = null, workerError = null;
       const workerStartedMs = Date.now();
+      const vmStartedBefore = deps.podmanVmStartedAt?.() ?? null;
       progress("worker");
       try {
         result = await runOpenClaw({
@@ -177,7 +238,7 @@ export function createExecutor(deps) {
           const recoveryResult = await runOpenClaw({
             task, acceptance, verification, mode, cwd, baseRef: base.ref, baseSha: base.sha,
             timeoutSeconds: timeBudget.reportReserveSeconds, runtimeDir, profile, reasoning, pool, subscriptionWorker, onBehalfOf, model, reportSize, jobDir, workerId: workerId || jobId,
-            overridePrompt: reportRecoveryPrompt({ report: budgetState.budgets.report.implement, changes }), logSuffix: "-recovery",
+            overridePrompt: reportRecoveryPrompt({ report: budgetState.budgets.report.implement, changes, task }), logSuffix: "-recovery",
           });
           const recoveryText = finalText(recoveryResult);
           const recoveryValidation = parseWorkerReport(recoveryText);
@@ -214,23 +275,26 @@ export function createExecutor(deps) {
         // failed job that changed nothing was recorded "pass" (a Senti run),
         // which reads as evidence about work that never happened.
         independentVerification = normalizeVerification({ status: "not_run", basis: "not-applicable", reason: "the worker changed nothing, so there was none of its work to verify" }, verification ?? null);
+      } else if (workerStopReason === "stopped") {
+        // Stopped on request: end now, without spending time on tests.
+        independentVerification = normalizeVerification({ status: "not_run", basis: "not-applicable", reason: "the job was stopped on request" }, verification ?? null);
       } else if (verificationFlow.verificationRunner || !reportValidation.valid) {
-        independentVerification = await runIndependentVerification({ profile: verification ?? null, cwd, jobId, baseSha: base.sha, branch, mode, record: preCommit });
+        independentVerification = await runIndependentVerification({ profile: verification ?? null, cwd, jobId, baseSha: base.sha, branch, mode, record: preCommit, logFile: path.join(jobDir, "verification.log") });
       }
 
       // verify_regression: on by default whenever there's a verification
       // profile (resolveVerifyRegression). It doubles verification wall-clock,
       // so it runs only when there's something to re-check: a passing
       // first-pass verification on a diff that touched production code.
-      // Documentation isn't code a test can prove, so it's neither reverted
-      // nor a reason to run the check (isDocumentationPath).
-      const codeFilesChanged = preCommit.testChanges.production_files_changed.filter((f) => !isDocumentationPath(f));
+      // Documentation and CI configuration cannot be proven by local tests,
+      // so neither is reverted or used as a reason to run this check.
+      const codeFilesChanged = planRegressionProductionFiles(preCommit.testChanges.production_files_changed);
       let regressionCheck = null, regressionCheckFatal = false, regressionCheckElapsedMs = null;
       if (verifyRegression && independentVerification.status === "pass" && codeFilesChanged.length > 0) {
         const regressionStartedMs = Date.now();
         try {
           regressionCheck = await runRegressionCheck({
-            cwd, jobId, productionFiles: codeFilesChanged,
+            cwd, jobId, productionFiles: codeFilesChanged, verificationResult: independentVerification,
             nameStatus: preCommit.nameStatus, profile: verification, baseSha: base.sha, branch, mode,
           });
         } catch (error) {
@@ -244,6 +308,36 @@ export function createExecutor(deps) {
         if (regressionCheck.status === "restore_failed") regressionCheckFatal = true;
       }
 
+      // Mutation testing (lib/mutation.mjs), when this repo opts in: small
+      // mistakes planted one at a time in the changed lines must each fail
+      // the same profile. Survivors raise review; a failed restore is fatal.
+      let mutation = null, mutationElapsedMs = null;
+      const mutationConfig = (() => { try { return loadConfig(projectDir)?.config?.mutation ?? null; } catch { return null; } })();
+      if (mutationConfig && verification && independentVerification.status === "pass" && codeFilesChanged.length > 0 && !regressionCheckFatal) {
+        const mutationStartedMs = Date.now();
+        try {
+          const untracked = new Set((preCommit.nameStatus ?? []).filter((e) => e.untracked).map((e) => e.path));
+          const deleted = new Set((preCommit.nameStatus ?? []).filter((e) => /^D/.test(e.status)).map((e) => e.path));
+          const files = [];
+          for (const file of codeFilesChanged.filter((f) => !deleted.has(f))) {
+            const full = path.join(cwd, file);
+            let lines;
+            if (untracked.has(file)) { try { lines = fs.readFileSync(full, "utf8").split("\n").map((_, i) => i + 1); } catch { continue; } }
+            else lines = parseAddedLineNumbers(await gitRaw(["diff", "-U0", base.sha, "--", file], cwd));
+            if (lines.length) files.push({ path: file, full, lines });
+          }
+          const mutants = pickMutants(files, mutationConfig.mutants);
+          let n = 0;
+          mutation = await runMutants({ mutants, deadlineMs: mutationStartedMs + mutationConfig.max_seconds * 1000,
+            verify: () => runIndependentVerification({ profile: verification, cwd, jobId: `${jobId}-mutant-${++n}`, baseSha: base.sha, branch, mode, record: preCommit }) });
+          mutation.planned = mutants.length;
+        } catch (error) {
+          mutation = { status: "not_run", killed: 0, survived: [], inconclusive: 0, tried: 0, reason: `mutation testing failed to run: ${error.message}` };
+        }
+        mutationElapsedMs = Date.now() - mutationStartedMs;
+        fs.appendFileSync(path.join(jobDir, "coordinator.log"), `${new Date().toISOString()} mutation testing: ${mutation.killed ?? 0} killed, ${mutation.survived?.length ?? 0} survived, ${mutation.inconclusive ?? 0} inconclusive of ${mutation.tried ?? 0} tried (${Math.round(mutationElapsedMs / 1000)}s)\n`);
+      }
+
       // resolveOutcome's own contract only ever sees pass/fail/not_run for
       // regressionCheck -- a restore_failed status is substituted to not_run
       // here so resolveOutcome never needs a fourth value; the hard override
@@ -264,12 +358,19 @@ export function createExecutor(deps) {
       // Cheap, always-on, additive: never changes commitAllowed/commitBlockedReason
       // on its own (unlike the regression-check override above), only flags for
       // review -- see detectScopedTestSelectionRisk's own doc comment for why.
-      let selectionRisk = null;
+      let selectionRisk = null, verificationInputs = null;
       if (mode === "implement" && verification) {
         try {
           const loaded = loadConfig(projectDir); // the operator's contract; see registerVerificationRunner's call
           const profileCommands = loaded.found ? (loaded.config?.verification?.[verification]?.commands ?? []) : [];
           selectionRisk = detectScopedTestSelectionRisk({ commands: profileCommands, testChanges: preCommit.testChanges });
+          // A diff that changes what those commands run (see
+          // detectVerificationInputChanges): applied below, after the others.
+          verificationInputs = await detectVerificationInputChanges({
+            commands: profileCommands, changedFiles: preCommit.changedFiles ?? [],
+            readBase: (file) => gitRaw(["show", `${base.sha}:${file}`], cwd).catch(() => null),
+            readHead: (file) => { try { return fs.readFileSync(path.join(cwd, file), "utf8"); } catch { return null; } },
+          });
         } catch { /* a config load failure here is the verification runner's own problem to report, not this check's */ }
       }
       const afterSelectionRisk = selectionRisk
@@ -364,18 +465,90 @@ export function createExecutor(deps) {
             commitBlockedReason: `possible secret detected: ${possibleSecrets.reason}`,
             reasons: [...afterHostInstalls.reasons, `POSSIBLE SECRET DETECTED: ${possibleSecrets.reason}`] }
         : afterHostInstalls;
-      const finalOutcome = applyRefactorContract(applyVerificationPolicy(afterSecrets, independentVerification.status, repoPolicy()),
+      // Jev: does the worker's report match its diff? Found live: a note said
+      // "restored check.js to base commit" while the diff rewrote check.js.
+      // Only raises review; it never blocks or allows a commit.
+      let jevClaims = null, judged = null;
+      const jevImplement = jevSettingsFor("report-claims");
+      const judge = (() => { try { return deps.judgeSettings?.() ?? null; } catch { return null; } })();
+      // The whole change, new files included, for whichever validators run.
+      const jobDiff = async () => {
+        let diff = await gitRaw(["diff", base.sha, "--"], cwd);
+        for (const entry of (preCommit.nameStatus ?? []).filter((e) => e.untracked)) {
+          let text = "";
+          try { text = fs.readFileSync(path.join(cwd, entry.path), "utf8"); } catch { continue; }
+          diff += `\ndiff --git a/${entry.path} b/${entry.path}\nnew file\n--- /dev/null\n+++ b/${entry.path}\n${text.split("\n").map((l) => `+${l}`).join("\n")}\n`;
+        }
+        return diff;
+      };
+      let diffText = null;
+      if (mode === "implement" && reportValidation?.valid && repositoryChanged && (jevImplement || (judge && !judge.problem))) {
+        try { diffText = await jobDiff(); } catch { diffText = null; }
+      }
+      if (jevImplement && diffText != null) {
+        try { jevClaims = await checkReportClaims({ report: reportValidation, diff: diffText, settings: jevImplement }); }
+        catch (error) { jevClaims = { flag: null, verdict: null, error: error.message, usage: 0 }; }
+      }
+      // The model judge (lib/judge.mjs): acceptance criteria, the report and
+      // changed tests. Only raises review, never blocks or allows a commit.
+      if (judge?.problem) judged = { flags: [], answer: null, error: `judge not run: ${judge.problem}`, skipped: true };
+      else if (judge && diffText != null) {
+        const modifiedTests = preCommit.testChanges?.existing_tests_modified ?? [];
+        let testDiff = "";
+        if (modifiedTests.length) { try { testDiff = await gitRaw(["diff", base.sha, "--", ...modifiedTests], cwd); } catch { testDiff = ""; } }
+        judged = await runJudge({ settings: judge, task, acceptance: acceptance ?? [], report: reportValidation, diff: diffText, testDiff, stateRoot: path.join(jobsRoot, "..") });
+      }
+      const afterJevOnly = jevClaims?.flag
+        ? { ...afterSecrets, reviewRequired: true, reasons: [...afterSecrets.reasons, `REPORT MAY NOT MATCH THE DIFF (Jev, ${jevClaims.flag.probability.toFixed(2)}): the report says "${String(reportValidation.note ?? "").slice(0, 200)}", and the diff may show otherwise. Read the diff before accepting.`] }
+        : afterSecrets;
+      const afterJev = judged?.flags?.length
+        ? { ...afterJevOnly, reviewRequired: true, reasons: [...afterJevOnly.reasons, `JUDGE (${judge.agent}/${judge.model}): ${judged.flags.join("; ")}. Read the diff before accepting.`] }
+        : afterJevOnly;
+
+      // A worker must not be judged by a check it rewrote: a changed script,
+      // Makefile or package.json script that a verification command runs
+      // blocks the commit; changed test-runner config only asks for review.
+      const blockedInputs = verificationInputs?.blocked ?? [], flaggedInputs = verificationInputs?.flagged ?? [];
+      const inputLine = blockedInputs.map((b) => `${b.file} (${b.why})`).join("; ");
+      const afterInputs = blockedInputs.length
+        ? { ...afterJev, outcome: afterJev.commitAllowed || afterJev.outcome === OUTCOMES.WORKER_DONE ? OUTCOMES.NEEDS_REVIEW : afterJev.outcome,
+            reviewRequired: true, commitAllowed: false,
+            commitBlockedReason: afterJev.commitAllowed ? `the diff changes what verification runs: ${inputLine}` : afterJev.commitBlockedReason,
+            reasons: [...afterJev.reasons, `VERIFICATION INPUT CHANGED: the diff changes what profile '${verification}' runs, so its result can't be trusted: ${inputLine}`] }
+        : afterJev;
+      const afterConfig = flaggedInputs.length
+        ? { ...afterInputs, reviewRequired: true, reasons: [...afterInputs.reasons, `TEST CONFIG CHANGED: ${flaggedInputs.map((c) => `${c.file} (${c.why})`).join("; ")}`] }
+        : afterInputs;
+      const afterMutation = mutation?.status === "restore_failed"
+        ? { ...afterConfig, outcome: OUTCOMES.NEEDS_REVIEW, reviewRequired: true, commitAllowed: false,
+            commitBlockedReason: `mutation testing could not restore the worker's file: ${mutation.reason}`, reasons: [...afterConfig.reasons, `MUTATION RESTORE FAILED: ${mutation.reason}`] }
+        : mutation?.status === "survivors"
+          ? { ...afterConfig, reviewRequired: true, reasons: [...afterConfig.reasons, describeSurvivors(mutation, verification)] }
+          : afterConfig;
+      const afterStakes = stakes === "high" ? { ...afterMutation, reviewRequired: true, reasons: [...afterMutation.reasons, HIGH_STAKES_NOTE] } : afterMutation;
+      const finalOutcome = applyRefactorContract(applyVerificationPolicy(afterStakes, independentVerification.status, repoPolicy()),
         { refactor, verificationStatus: independentVerification.status, testChanges: preCommit.testChanges });
 
       progress("commit");
       const commit = await createCoordinatorCommit({ cwd, jobId, outcome: finalOutcome,
-        message: coordinatorCommitMessage({ task, subject: commitSubject, note: reportValidation?.note ?? null, jobId, workerId, recovered: Boolean(finalOutcome.recovered), provider: (result ?? attempted)?.provider ?? null, model: (result ?? attempted)?.model ?? null }) });
+        message: coordinatorCommitMessage({ task: continuation?.record?.objective ?? task, subject: commitSubject, note: reportValidation?.note ?? null, jobId, workerId, recovered: Boolean(finalOutcome.recovered), provider: (result ?? attempted)?.provider ?? null, model: (result ?? attempted)?.model ?? null, continuedFrom: continuedFrom?.jobId ?? null }) });
       progress("record");
       const record = await collectGitRecord({ cwd, baseSha: base.sha, branch, baseRef: base.ref, jobId }), worker = workerMetadata(result ?? attempted);
 
       let coordinatorStatus = COORDINATOR_STATUS_BY_OUTCOME[finalOutcome.outcome] ?? "incomplete";
-      const issues = [...finalOutcome.reasons];
-      if (workerError) issues.push(`worker error: ${String(workerError).split("\n")[0]}`);
+      const issues = [...finalOutcome.reasons, ...(preCommit.issues ?? [])];
+      if (workerStopReason === "stopped") {
+        const request = readStopRequest(jobDir);
+        issues.push(`stopped on request${request?.reason ? `: ${request.reason}` : ""}; the worktree is kept, so continue_from can pick the work up (on another model too)`);
+      } else if (workerError) issues.push(`worker error: ${String(workerError).split("\n")[0]}`);
+      // OpenClaw's own cleanup failing after a finished run is benign once the report
+      // is recovered, and it happens on most Codex jobs: kept on the record for
+      // stats, out of the issues the General reviews.
+      const runnerNotes = [];
+      if ((result ?? attempted)?.salvaged) runnerNotes.push(`runner cleanup failed after the run (${(result ?? attempted).salvagedFrom}); the worker's report was recovered from the run's transcript`);
+      if (jevClaims?.error) issues.push(`Jev report check skipped (${jevClaims.error}); this job's result doesn't depend on it`);
+      if (judged?.error) issues.push(`Judge ${judged.skipped ? "skipped" : "didn't answer"} (${judged.error}); this job's result doesn't depend on it`);
+      if (workerFailed || workerTimedOut) { const restarted = vmRestartIssue(vmStartedBefore, deps.podmanVmStartedAt?.() ?? null); if (restarted) issues.unshift(restarted); }
       if (repositoryChanged && !commit.created) {
         if (coordinatorStatus === "complete") coordinatorStatus = "incomplete";
         // A timed-out or crashed worker can still leave real, salvageable work
@@ -385,7 +558,7 @@ export function createExecutor(deps) {
         // the diffstat right in the issue a caller actually reads -- not just
         // buried in the full manifest's git record -- is what makes "go look at
         // the worktree" worth doing instead of discarding the job.
-        issues.push(`repository changes remain uncommitted (${record.filesChanged} file(s), +${record.additions}/-${record.deletions}): ${commit.reason}`);
+        if (record.filesChanged > 0) issues.push(`repository changes remain uncommitted (${record.filesChanged} file(s), +${record.additions}/-${record.deletions}): ${commit.reason}`);
       }
       const failures = worker.toolSummary?.failures ?? 0; if (failures > 0) issues.push(`worker recorded ${failures} tool failure(s)`);
       if (record.ignoredRuntimeJunk.length) issues.push(`runtime junk ignored: ${record.ignoredRuntimeJunk.join(", ")}`);
@@ -403,11 +576,13 @@ export function createExecutor(deps) {
 
       const metrics = buildMetrics({ result: result ?? attempted, record, reportValidation, outcome: finalOutcome, workerElapsedMs, totalElapsedMs: Date.now() - jobStartedMs, regressionCheckElapsedMs, transientAbortRetried });
       const manifest = { version: VERSION, jobId, workerId: workerId || jobId, mode, projectDir, worktree, branch, startedAt, finishedAt,
-        objective: task, acceptance: acceptance ?? [], verificationProfile: verification ?? null,
+        objective: task, acceptance: acceptance ?? [], verificationProfile: verification ?? null, ...(continuedFrom ? { continuedFrom } : {}), ...(stakes ? { stakes } : {}),
+        ...(mutation ? { mutation: { ...mutation, elapsedSeconds: Math.round((mutationElapsedMs ?? 0) / 1000) } } : {}),
+        ...(jevClaims || judged ? { validators: { ...(jevClaims ? { jev: { check: "report-claims", verdict: jevClaims.verdict, flagged: Boolean(jevClaims.flag), error: jevClaims.error, truncated: Boolean(jevClaims.truncated), inputTokens: jevClaims.usage } } : {}), ...(judged ? { judge: { agent: judge?.agent ?? null, provider: judge?.provider ?? null, model: judge?.model ?? null, answer: judged.answer, flags: judged.flags, error: judged.error } } : {}) } } : {}),
         outcome: finalOutcome.outcome, recovered: finalOutcome.recovered, recoveryAttempted: finalOutcome.recoveryAttempted,
         reportRecoveryAttempted, reportRecovered,
         reviewRequired: finalOutcome.reviewRequired || record.testChanges.reviewRequired,
-        coordinatorStatus, issues, reportValidation, independentVerification,
+        coordinatorStatus, issues: [...issues, ...(independentVerification.issues ?? [])], runnerNotes, reportValidation, independentVerification,
         // Original, unsubstituted regressionCheck (real "restore_failed" status
         // visible here even though resolveOutcome above only ever saw a
         // not_run-substituted view) -- full transparency for the caller.
@@ -446,7 +621,7 @@ export function createExecutor(deps) {
   // the worktree, so a scout that wrote to its snapshot cannot forge evidence.
   // A clean scout worktree holds no work and is removed; a dirty one is retained
   // because a scout that wrote is a scout that misbehaved, and that is worth a look.
-  async function executeScout({ task, acceptance, base, jobId, jobDir, runtimeDir, timeoutSeconds, profile, reasoning, pool = null, subscriptionWorker = null, onBehalfOf = null, model = null, reportSize = null, workerId, progress, jobStartedMs }) {
+  async function executeScout({ task, acceptance, base, jobId, jobDir, runtimeDir, timeoutSeconds, profile, reasoning, pool = null, subscriptionWorker = null, onBehalfOf = null, model = null, reportSize = null, workerId, progress, jobStartedMs, reviews = null }) {
     const mode = "scout", worktree = path.join(jobDir, "worktree");
     let worktreeRetained = false;
     try {
@@ -469,7 +644,10 @@ export function createExecutor(deps) {
       const workerStartedMs = Date.now();
       progress("worker");
       try {
-        result = await runOpenClaw({ task, acceptance, verification: null, mode, cwd: worktree, baseRef: base.ref, baseSha: base.sha, timeoutSeconds, runtimeDir, profile, reasoning, pool, subscriptionWorker, onBehalfOf, model, reportSize, jobDir, workerId: workerId || jobId, evidenceTool: evidencePlaced ? evidenceTool : null });
+        // The run gets the work share; the reserve stays for report recovery
+        // below. Scouts had none, so a timed-out scout left about 0 seconds
+        // and its findings were never recovered.
+        result = await runOpenClaw({ task, acceptance, verification: null, mode, cwd: worktree, baseRef: base.ref, baseSha: base.sha, timeoutSeconds: deriveTimeBudget({ timeoutSeconds }).workTimeoutSeconds, runtimeDir, profile, reasoning, pool, subscriptionWorker, onBehalfOf, model, reportSize, jobDir, workerId: workerId || jobId, evidenceTool: evidencePlaced ? evidenceTool : null });
       } catch (error) {
         workerFailed = true;
         // error.timedOut is set only by our own spawn timer (run(), above) --
@@ -499,18 +677,26 @@ export function createExecutor(deps) {
 
       // See shouldAttemptScoutRecovery's own doc comment: this only fires when
       // the report is genuinely unusable, gated by whatever time is actually
-      // left against the caller's original timeout (scout has no reserved
-      // report-phase budget the way implement does).
+      // left against the caller's original timeout (the reserve held back
+      // from the run above).
       let reportRecoveryAttempted = false, reportRecovered = false;
-      const remainingSeconds = timeoutSeconds - Math.round(workerElapsedMs / 1000);
+      // At least the reserve, as implement's recovery gets: nomArmy's own kill
+      // lands 30s after OpenClaw's timer, which would otherwise eat it.
+      const remainingSeconds = Math.max(deriveTimeBudget({ timeoutSeconds }).reportReserveSeconds, timeoutSeconds - Math.round(workerElapsedMs / 1000));
       if (shouldAttemptScoutRecovery({ workerFailed, workerTimedOut, report, remainingSeconds })) {
         reportRecoveryAttempted = true;
+        // What the first run left, in case the follow-up can't see its session.
+        let filesRead = [], earlierReply = reportText;
+        try {
+          const first = await readOpenClawTranscript(path.join(runtimeDir, "state"));
+          if (first.available) { filesRead = first.filesRead ?? []; earlierReply = earlierReply || first.lastAssistantText || ""; }
+        } catch { /* the reply alone still helps */ }
         try {
           const recoveryResult = await runOpenClaw({
             task, acceptance, verification: null, mode, cwd: worktree, baseRef: base.ref, baseSha: base.sha,
             timeoutSeconds: remainingSeconds, runtimeDir, profile, reasoning, pool, subscriptionWorker, onBehalfOf, model, reportSize, jobDir, workerId: workerId || jobId,
             evidenceTool: evidencePlaced ? evidenceTool : null,
-            overridePrompt: scoutReportRecoveryPrompt({ report: used.report.scout }), logSuffix: "-recovery",
+            overridePrompt: scoutReportRecoveryPrompt({ report: used.report.scout, question: task, acceptance, earlierReply, filesRead }), logSuffix: "-recovery",
           });
           const recoveryReport = parseScoutReport(finalText(recoveryResult), (recoveryResult?.budgetsUsed ?? used).scout);
           if (!isScoutReportUnusable(recoveryReport)) {
@@ -530,7 +716,16 @@ export function createExecutor(deps) {
       const dirty = record.repoStatusFiles.length > 0;
       const readFile = async p => { try { return await gitRaw(["show", `${base.sha}:${p}`], projectDir); } catch { return null; } };
       const verified = await verifyCitations(report.findings, { readFile, limits: used.scout });
+      // Jev: do the cited lines support each finding? Only adds flags.
+      let jevCitations = null;
+      const jevScout = jevSettingsFor("scout-citations");
+      if (jevScout && verified?.findings?.length) {
+        try { jevCitations = await checkScoutCitations({ findings: verified.findings, settings: jevScout, readFile }); }
+        catch (error) { jevCitations = { flags: [], checked: 0, errors: [error.message], usage: 0, verdicts: [] }; }
+        for (const f of jevCitations.flags) verified.findings[f.index].jev = { verdict: f.verdict, probability: f.probability };
+      }
       const outcome = resolveScoutOutcome({ report, verified, workerFailed, workerTimedOut, dirty });
+      if (jevCitations?.flags.length) outcome.reviewRequired = true;
 
       progress("record");
       if (outcome.retainWorktree) worktreeRetained = true;
@@ -539,8 +734,15 @@ export function createExecutor(deps) {
       const worker = workerMetadata(result ?? attempted);
       const issues = [...outcome.reasons];
       if (workerError) issues.push(`scout error: ${String(workerError).split("\n")[0]}`);
+      // OpenClaw's own cleanup failing after a finished run is benign once the report
+      // is recovered, and it happens on most Codex jobs: kept on the record for
+      // stats, out of the issues the General reviews.
+      const runnerNotes = [];
+      if ((result ?? attempted)?.salvaged) runnerNotes.push(`runner cleanup failed after the run (${(result ?? attempted).salvagedFrom}); the scout's report was recovered from the run's transcript`);
       const failures = worker.toolSummary?.failures ?? 0; if (failures > 0) issues.push(`scout recorded ${failures} tool failure(s)`);
       if (dirty) issues.push(`snapshot changed: ${record.repoStatusFiles.join(", ")}`);
+      if (jevCitations?.flags.length) issues.push(`CITATIONS MAY NOT SUPPORT FINDINGS (Jev): ${jevCitations.flags.map((f) => `"${String(verified.findings[f.index].text).slice(0, 80)}${String(verified.findings[f.index].text).length > 80 ? "..." : ""}" (${f.verdict}, ${f.probability.toFixed(2)})`).join("; ")}. Read those cited lines before relying on them; they're marked [JEV] in the report.`);
+      if (jevCitations?.errors.length) issues.push(`Jev citation check skipped or incomplete (${jevCitations.errors.join("; ")}); this job's result doesn't depend on it`);
       if (reportRecoveryAttempted) {
         issues.push(reportRecovered
           ? "scout report recovered via a follow-up call after the first reply was cut off"
@@ -577,8 +779,9 @@ export function createExecutor(deps) {
         displacement_verdict: displacement.verdict
       };
       const manifest = { version: VERSION, jobId, workerId: workerId || jobId, mode, projectDir, worktree: worktreeRetained ? worktree : null, branch: null, baseSha: base.sha, startedAt, finishedAt,
-        objective: task, mustCover: acceptance ?? [],
-        outcome: outcome.outcome, coordinatorStatus: outcome.coordinatorStatus, reviewRequired: outcome.reviewRequired, issues,
+        objective: task, mustCover: acceptance ?? [], ...(reviews ? { reviews } : {}),
+        ...(jevCitations ? { validators: { jev: { check: "scout-citations", checked: jevCitations.checked, flags: jevCitations.flags, errors: jevCitations.errors, inputTokens: jevCitations.usage } } } : {}),
+        outcome: outcome.outcome, coordinatorStatus: outcome.coordinatorStatus, reviewRequired: outcome.reviewRequired, issues, runnerNotes,
         scout: { question: report.question, confidence: report.confidence, notFound: report.notFound,
           findings: verified.findings, supported: verified.supported, unsupported: verified.unsupported, weak: verified.weak,
           excerptLinesUsed: verified.excerptLinesUsed, excerptTruncated: verified.excerptTruncated,
@@ -670,6 +873,11 @@ export function createExecutor(deps) {
       const worker = workerMetadata(result ?? attempted);
       const issues = [...outcome.reasons];
       if (workerError) issues.push(`decompose error: ${String(workerError).split("\n")[0]}`);
+      // OpenClaw's own cleanup failing after a finished run is benign once the report
+      // is recovered, and it happens on most Codex jobs: kept on the record for
+      // stats, out of the issues the General reviews.
+      const runnerNotes = [];
+      if ((result ?? attempted)?.salvaged) runnerNotes.push(`runner cleanup failed after the run (${(result ?? attempted).salvagedFrom}); the decomposer's report was recovered from the run's transcript`);
       const failures = worker.toolSummary?.failures ?? 0; if (failures > 0) issues.push(`decomposer recorded ${failures} tool failure(s)`);
       if (dirty) issues.push(`snapshot changed: ${record.repoStatusFiles.join(", ")}`);
       if (overlaps.length) issues.push(`${overlaps.length} subtask pair(s) claim overlapping files; not safe to dispatch as independent jobs as proposed`);
@@ -699,7 +907,7 @@ export function createExecutor(deps) {
       };
       const manifest = { version: VERSION, jobId, workerId: workerId || jobId, mode, projectDir, worktree: worktreeRetained ? worktree : null, branch: null, baseSha: base.sha, startedAt, finishedAt,
         objective: task, constraints: acceptance ?? [],
-        outcome: outcome.outcome, coordinatorStatus: outcome.coordinatorStatus, reviewRequired: outcome.reviewRequired, issues,
+        outcome: outcome.outcome, coordinatorStatus: outcome.coordinatorStatus, reviewRequired: outcome.reviewRequired, issues, runnerNotes,
         decompose: { objective: report.objective, confidence: report.confidence, notSplittable: report.notSplittable,
           subtasks: report.subtasks.map((s, i) => ({ task: s.task, acceptance: s.acceptance, citations: verified.findings[i]?.citations ?? [], supported: verified.findings[i]?.supported ?? false, weak: verified.findings[i]?.weak ?? false })),
           overlaps, supported: verified.supported, unsupported: verified.unsupported, weak: verified.weak,

package/lib/git-record.mjs

@@ -65,7 +65,7 @@ export function isRuntimeJunk(file) {
  * gave one, else the task's first sentence (the army role header and an
  * "OBJECTIVE:" label dropped). The job id stays, as a trailer.
  */
-export function coordinatorCommitMessage({ task = "", subject = null, note = null, jobId, workerId = null, recovered = false, provider = null, model = null }) {
+export function coordinatorCommitMessage({ task = "", subject = null, note = null, jobId, workerId = null, recovered = false, provider = null, model = null, continuedFrom = null }) {
   const oneLine = (t) => String(t ?? "").replace(/\s+/g, " ").trim();
   let body = String(task ?? "");
   if (/^\[nomArmy role:/.test(body)) body = body.includes("\n\n") ? body.slice(body.indexOf("\n\n") + 2) : "";
@@ -78,6 +78,7 @@ export function coordinatorCommitMessage({ task = "", subject = null, note = nul
   const cleanNote = oneLine(note);
   if (cleanNote) lines.push("", ...wrapText(cleanNote, 72));
   lines.push("", `nomArmy-Job: ${jobId}`);
+  if (continuedFrom) lines.push(`nomArmy-Continues: ${continuedFrom}`);
   if (provider || model) lines.push(`nomArmy-Worker: ${[provider, model].filter(Boolean).join("/")}`);
   return lines.join("\n");
 }
@@ -138,9 +139,26 @@ export function createGitRecord({ run, git, gitRaw }) {
   // idleMinElapsedMs of the work phase has passed (an early snapshot mid-first-
   // edit looks identical to no edit at all). A worktree read failing mid-write
   // is expected, not an error; it just means "nothing to report this tick."
-  function makeIdleDiffTick(cwd, { idleMs, minElapsedMs }) {
-    let lastHash = null, lastChangeAtMs = 0, sawChange = false;
+  //
+  // An unchanged worktree isn't enough on its own: a frontier worker running
+  // a suite of thousands of tests, or reading before its next edit, changes
+  // no file for minutes. Three Senti jobs were cut off at 7 to 13 minutes
+  // while still working. With `activity` (the worker's transcript: event
+  // count and whether a tool call is still running), the breaker waits while
+  // the worker is active, and stops only when it has also gone quiet for
+  // idleMs, or when nothing has changed for ACTIVE_CAP times idleMs however
+  // busy it looks (a worker looping without progress).
+  function makeIdleDiffTick(cwd, { idleMs, minElapsedMs, activity = null }) {
+    const ACTIVE_CAP = 3;
+    let lastHash = null, lastChangeAtMs = 0, sawChange = false, lastEvents = null, lastActivityAtMs = 0, inFlight = false;
     return async elapsedMs => {
+      if (activity) {
+        try {
+          const a = await activity();
+          if (a && a.events !== lastEvents) { lastEvents = a.events; lastActivityAtMs = elapsedMs; }
+          inFlight = Boolean(a?.toolInFlight);
+        } catch { /* a failed read just means no activity signal this tick */ }
+      }
       let statusOut;
       try { statusOut = await gitRaw(["status", "--porcelain=v1", "-z", "--untracked-files=all"], cwd); }
       catch { return { stop: false }; }
@@ -181,6 +199,12 @@ export function createGitRecord({ run, git, gitRaw }) {
       if (!sawChange || elapsedMs < minElapsedMs) return { stop: false };
       const idleForMs = elapsedMs - lastChangeAtMs;
       if (idleForMs < idleMs) return { stop: false };
+      if (activity) {
+        const quietForMs = elapsedMs - lastActivityAtMs;
+        const active = inFlight || quietForMs < idleMs;
+        if (active && idleForMs < idleMs * ACTIVE_CAP) return { stop: false };
+        if (active) return { stop: true, reason: "idle_diff", detail: `worktree unchanged for ${Math.round(idleForMs / 1000)}s while the worker kept working (${ACTIVE_CAP}x the idle limit)` };
+      }
       return { stop: true, reason: "idle_diff", detail: `worktree unchanged for ${Math.round(idleForMs / 1000)}s` };
     };
   }

package/lib/harness-schema.mjs

@@ -0,0 +1,61 @@
+import { z } from "zod";
+
+const text = z.string().trim().min(1, "must not be empty");
+const name = z.string().regex(/^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/, "must be kebab-case");
+const relativePath = text.refine((value) => !value.startsWith("/") && !value.includes("\\") && !value.includes(":") && !value.split("/").includes(".."), "must be a repository-relative path");
+const detect = z.union([
+  z.object({ file: relativePath }).strict(),
+  z.object({ package: text }).strict(),
+  z.object({ lockfile: relativePath }).strict(),
+]);
+
+const env = z.record(z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/), z.string());
+const pinnedImage = text.refine((value) => {
+  if (/\s/.test(value) || value.startsWith("-")) return false;
+  const reference = value.split("@")[0];
+  if (reference.split("/").at(-1).endsWith(":latest")) return false;
+  return /^[a-z0-9][a-z0-9._:/-]*@sha256:[a-f0-9]{64}$/.test(value)
+    || /^[a-z0-9][a-z0-9._:/-]*:[A-Za-z0-9_][A-Za-z0-9_.-]*$/.test(value)
+      && value.split("/").at(-1).includes(":");
+}, "image must have an explicit non-latest tag or sha256 digest");
+const service = z.object({
+  name, image: pinnedImage, port: z.number().int().min(1).max(65535),
+  env: env.optional(),
+  health: z.string().regex(/^\/(?!\/)[^\s\\#]*$/, "health must be an HTTP path").optional(),
+}).strict();
+
+export const harnessSchema = z.object({
+  name,
+  summary: text,
+  detect: z.array(detect),
+  after: z.array(name).default([]),
+  image: z.union([
+    z.object({ builtin: name }).strict(),
+    z.object({ apt: z.array(text), run: z.array(text) }).strict(),
+  ]),
+  verification: z.record(text, z.union([text, z.array(text).min(1)])).default({}),
+  artifacts: z.array(relativePath).default([]),
+  requires: z.object({
+    memoryMb: z.number().int().positive().optional(),
+    shmMb: z.number().int().positive().optional(),
+    kvm: z.boolean().optional(),
+  }).strict().default({}),
+  network: z.enum(["none", "services", "allowlist"]).default("none"),
+  services: z.array(service).optional(),
+  env: env.optional(),
+  suggestedRole: z.object({ name, description: text }).strict().optional(),
+  docs: relativePath.default("README.md"),
+}).strict().superRefine((spec, ctx) => {
+  if (new Set(spec.services?.map((service) => service.name)).size !== (spec.services?.length ?? 0)) {
+    ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["services"], message: "service names must be unique" });
+  }
+  if (spec.services !== undefined && spec.network === "none") {
+    ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["services"], message: "services requires network services or allowlist" });
+  }
+});
+
+export function harnessSchemaFor(folderName) {
+  return harnessSchema.refine((spec) => spec.name === folderName, {
+    path: ["name"], message: "name must equal its folder name",
+  });
+}