nomarmy 0.1.0-alpha.2 → 0.1.0-alpha.21

package/lib/admission.mjs

@@ -1,14 +1,19 @@
 import fs from "node:fs";
 import path from "node:path";
 import { executionMode } from "./execution.mjs";
-import { clampInt } from "./budget-state.mjs";
+import { jobLabel, jobElapsedSeconds } from "./job-format.mjs";
+import { continuationProblem } from "./continue-from.mjs";
+import { loadConfig } from "./config.mjs";
+import { maxJobs } from "./limits.mjs";
 import { checkBrief, assessAdmission, describeBudgets } from "./budget.mjs";
 import { parseStatusPorcelainZ, isRuntimeJunk } from "./git-record.mjs";
 import { readJson } from "./openclaw-run.mjs";
 import { readOpenClawTranscriptTail } from "./transcript.mjs";
 import { readClaudeSessionTranscript } from "./claude-transcript.mjs";
 import { notify } from "./notify.mjs";
-import { recentModelRefusal } from "./health.mjs";
+import { readCodexRateLimits, recordUsageSnapshot, readUsageSnapshots, usageStatus, usageDisplayText, refreshStaleOverLimitReadings, staleUsageRefreshedNote, USAGE_STALE_MINUTES } from "./usage-limits.mjs";
+import { projectDirProblem } from "./server-context.mjs";
+import { recentModelRefusal, refusedModelIn, recordModelRefusal, recordProbeSuccess, modelProven, modelRefusals, claimModelRefusalRetry } from "./health.mjs";
 import { writeLease, removeLease, liveLeases, liveSlots, acquireSlot } from "./slots.mjs";
 import { loadRun, runTotals, runAdmissionProblems, recordRunJob, detectUsageLimit } from "./runs.mjs";
 import { agentProviderId, hostToolsImplementProblem } from "./agents.mjs";
@@ -21,7 +26,7 @@ import { policyAdmissionProblems } from "./outcome.mjs";
 // Claude or Codex job took llama-server's only slot and blocked local work
 // it never competed with -- reported from a real Senti run.
 export function jobLane(job) {
-  return job.pool || job.subscription_worker ? "remote" : "local";
+  return job.mode === "verify" || job.pool || job.subscription_worker ? "remote" : "local";
 }
 
 // A static, operator-declared ceiling on how many remote jobs (api and
@@ -33,8 +38,9 @@ export function jobLane(job) {
 // benefit of spreading load across providers with their own separate rate
 // limits. Not rate-limit-aware (see config/providers.yml.example); read
 // fresh each call, matching currentMaxWorkers()'s own env-read pattern.
+/** Api and subscription jobs at once, machine-wide: limits.yml's max_jobs (see lib/limits.mjs). */
 export function currentMaxPoolWorkers() {
-  return clampInt(process.env.NOMARMY_MAX_POOL_WORKERS, 1, 32, 4);
+  return maxJobs().value;
 }
 
 // Pure partition of a batch's ORIGINAL indices by lane -- pulled out of
@@ -113,7 +119,7 @@ export function createJobRuntime(deps) {
    * batch queue for a slot instead of failing.
    */
   function withAgentSlot(args, jobId, fn, { waitMs = 0 } = {}) {
-    const max = args.agentName ? agentMaxConcurrent(args.agentName) : null;
+    const max = args.mode !== "verify" && args.agentName ? agentMaxConcurrent(args.agentName) : null;
     if (!max) return fn();
     return (async () => {
       const slot = await acquireSlot(slotsRoot, args.agentName, max, { jobId, waitMs });
@@ -121,6 +127,19 @@ export function createJobRuntime(deps) {
       try { return await fn(); } finally { slot.release(); }
     })();
   }
+  // Who ran the job and for whom, onto its finished record, so `nomarmy
+  // stats` can count by role and agent, and by repo for verify runs too
+  // (their records didn't carry the repo). Best-effort; never fails the job.
+  function stampJobRecord(entry, result) {
+    const file = path.join(result?.jobDir ?? path.join(jobsRoot, entry.jobId), "metadata.json");
+    try {
+      const record = JSON.parse(fs.readFileSync(file, "utf8"));
+      record.labels = { role: entry.role ?? null, agent: entry.agent ?? null, label: entry.label ?? null, runId: entry.runId ?? null };
+      record.projectDir = record.projectDir ?? projectDir;
+      fs.writeFileSync(`${file}.tmp`, JSON.stringify(record, null, 2));
+      fs.renameSync(`${file}.tmp`, file);
+    } catch { /* no record (a refused or crashed job): nothing to label */ }
+  }
   function track(jobId, meta, promise) {
     const entry = { ...meta, jobId, startedAt: new Date().toISOString(), settled: false, result: null, error: null, promise: null };
     // A machine-wide lease for as long as the job runs, so every session's
@@ -129,7 +148,7 @@ export function createJobRuntime(deps) {
     if (meta.lane) writeLease(leasesRoot, jobId, { lane: meta.lane, agent: meta.agent ?? null, runId: meta.runId ?? null, role: meta.role ?? null, model: meta.model ?? null, repo: projectDir });
     const release = () => removeLease(leasesRoot, jobId);
     entry.promise = promise.then(
-      r => { entry.settled = true; entry.result = r; release(); notifyJobFinished(entry, r, null); return r; },
+      r => { entry.settled = true; entry.result = r; release(); stampJobRecord(entry, r); notifyJobFinished(entry, r, null); return r; },
       e => { entry.settled = true; entry.error = e; release(); notifyJobFinished(entry, null, e); throw e; });
     entry.promise.catch(() => {});
     activeJobs.set(jobId, entry);
@@ -140,18 +159,29 @@ export function createJobRuntime(deps) {
    * watching hears about it from any coordinator without polling.
    */
   function notifyJobFinished(entry, result, error) {
+    try {
+      const provider = entry.agent ? agentProviderId(agentsConfig().agents[entry.agent]) : null;
+      const snapshot = provider ? readCodexRateLimits(path.join(jobsRoot, entry.jobId)) : null;
+      if (snapshot) recordUsageSnapshot(stateRoot, provider, snapshot);
+    } catch { /* Usage telemetry must never affect the job result. */ }
     if (!entry.lane) return; // only tracked jobs, never internal helpers
     const m = result?.manifest ?? {};
+    // Remember a model its vendor refused, until something on it works.
+    try {
+      const refused = refusedModelIn(m);
+      if (refused) recordModelRefusal(stateRoot, refused, String(m.workerError ?? m.worker?.error ?? "").split("\n")[0].slice(0, 300) || null);
+      else if (m.worker?.provider && m.worker?.model && !m.workerError && !error) recordProbeSuccess(stateRoot, `${m.worker.provider}/${m.worker.model}`);
+    } catch { /* best-effort, like the usage reading */ }
     const outcome = error ? "failed" : String(m.outcome ?? (result?.ok ? "done" : "finished")).toLowerCase().replace(/_/g, " ");
-    const who = entry.agent ? `${entry.agent}${entry.model ? `/${entry.model}` : ""}` : "local model";
+    const who = entry.mode === "verify" ? "verification runner" : entry.agent ? `${entry.agent}${entry.model ? `/${entry.model}` : ""}` : "local model";
     const took = Math.round((Date.now() - Date.parse(entry.startedAt)) / 60000);
     const ok = !error && (result?.ok || m.coordinatorStatus === "complete");
-    notify(`nomArmy: ${entry.role ?? entry.mode ?? "job"} ${ok ? "done" : outcome}`, `${entry.workerId ?? entry.jobId} on ${who}: ${outcome} after ${took}m. ${ok ? "Ready for the General's review." : "Needs a look."}`);
+    notify(`nomArmy: ${entry.role ?? entry.mode ?? "job"} ${ok ? "done" : outcome}`, `${entry.label ? `${entry.label} (${entry.workerId ?? entry.jobId})` : entry.workerId ?? entry.jobId} on ${who}: ${outcome} after ${took}m. ${ok ? "Ready for the General's review." : "Needs a look."}`);
   }
   function admissionHardware() {
     return executionMode(deps.env).managesModelServer ? deps.budgetState.hardwareSnapshot : null;
   }
-  function capacitySnapshot() {
+  function capacitySnapshot(usageOverride = null) {
     const admission = assessAdmission({ hardware: admissionHardware(), runningJobs: runningCount("local"), slots: deps.budgetState.contextInfo.slots, maxWorkers: currentMaxWorkers() });
     return {
       // The local model's budget. An api or subscription job's scales with
@@ -161,14 +191,55 @@ export function createJobRuntime(deps) {
       admission,
       memory: deps.budgetState.hardwareSnapshot?.memory ?? null,
       running: [...activeJobs.values()].filter(j => !j.settled).map(j => ({ jobId: j.jobId, workerId: j.workerId, mode: j.mode, lane: j.lane, startedAt: j.startedAt, phase: readJson(path.join(jobsRoot, j.jobId, "status.json"))?.phase ?? "starting" })),
+      usageLimits: Object.fromEntries(Object.entries(usageOverride ?? readUsageSnapshots(stateRoot)).map(([provider, snapshot]) => {
+        const status = usageStatus(snapshot);
+        return [provider, { ...status, text: usageDisplayText(status) }];
+      })),
       maxWorkers: currentMaxWorkers(),
-      remote: { running: runningCount("remote"), maxWorkers: currentMaxPoolWorkers(), note: "api and subscription agents; each agent's own max_concurrent also applies" }
+      remote: (() => { const limit = maxJobs(); return { running: runningCount("remote"), maxWorkers: limit.value, setBy: limit.source === "file" ? limit.path : limit.source === "env" ? "NOMARMY_MAX_POOL_WORKERS" : "default", note: "api and subscription agents; each agent's own max_concurrent also applies. Change with `nomarmy config max-jobs <n>`." }; })()
     };
   }
   async function admit(jobs) {
-    await deps.budgetState.refresh();
-    if (jobs.some((j) => jobLane(j) === "remote")) await modelCatalogReady();
+    if (jobs.some(j => j.mode !== "verify")) await deps.budgetState.refresh();
+    if (jobs.some((j) => j.mode !== "verify" && jobLane(j) === "remote")) await modelCatalogReady();
     const problems = [];
+    const notARepo = (deps.projectDirProblem ?? projectDirProblem)(projectDir);
+    if (notARepo) problems.push(notARepo);
+    // Every job needs its sandbox; the server wires the Podman check (tests don't).
+    const noSandbox = deps.sandboxProblem ? deps.sandboxProblem() : null;
+    if (noSandbox) problems.push(noSandbox);
+    const usageNow = deps.now?.() ?? Date.now();
+    let snapshots = readUsageSnapshots(stateRoot);
+    const providerFor = (job) => {
+      if (job.mode === "verify" || !job.agentName || job.confirm_over_limit === true) return null;
+      try { return agentProviderId(agentsConfig().agents[job.agentName]) ?? null; } catch { return null; }
+    };
+    const held = [];
+    jobs.forEach((j, i) => {
+      const provider = providerFor(j);
+      if (!provider || !snapshots[provider]) return;
+      const status = usageStatus(snapshots[provider], usageNow);
+      if (status.level === "over") held.push({ i, j, provider, status });
+    });
+    const usageNotes = [];
+    let failedProviders = new Set();
+    let refreshError = null;
+    if (held.some((item) => item.status.ageMinutes >= USAGE_STALE_MINUTES)) {
+      const refreshed = await refreshStaleOverLimitReadings(stateRoot, { run: deps.usageRun, now: usageNow, snapshots, timeoutMs: deps.usageRefreshTimeoutMs });
+      snapshots = refreshed.snapshots;
+      refreshError = refreshed.error;
+      failedProviders = new Set(refreshed.failedProviders ?? []);
+    }
+    const cleared = new Set();
+    for (const item of held) {
+      const status = snapshots[item.provider] ? usageStatus(snapshots[item.provider], usageNow) : item.status;
+      if (status.level !== "over") {
+        if (!cleared.has(item.provider)) { cleared.add(item.provider); usageNotes.push(staleUsageRefreshedNote(item.provider)); }
+        continue;
+      }
+      const extra = failedProviders.has(item.provider) ? ` ${refreshError ?? "OpenClaw usage refresh failed"}.` : "";
+      problems.push(`${jobs.length > 1 ? `job ${item.i + 1}: ` : ""}agent "${item.j.agentName}" is held at its usage limit: ${status.text} (reading ${status.ageMinutes} minutes old).${extra} Ask the operator before resubmitting with confirm_over_limit: true, or send the job to another agent.`);
+    }
     // A pool-routed job is checked against that pool's OWN (model-dependent)
     // budget, not the local-derived global one -- see budgetsForPool. Which
     // specific entry pickProvider will land on isn't known yet at admission
@@ -178,12 +249,36 @@ export function createJobRuntime(deps) {
     // (see budgetsForSubscriptionWorker) -- there's no "which entry" unknown
     // the way a weighted pool has, since the name given IS the entry.
     jobs.forEach((j, i) => {
+      if (j.mode === "verify") {
+        try {
+          const profiles = Object.keys(loadConfig(projectDir)?.config?.verification ?? {});
+          if (!j.verification || !profiles.includes(j.verification)) problems.push(`${jobs.length > 1 ? `job ${i + 1}: ` : ""}verify requires a verification profile from .nomarmy.yml; available profiles: ${profiles.join(", ") || "(none)"}`);
+        } catch (error) { problems.push(error.message); }
+        return;
+      }
       const jobBudgets = budgetsForJob(j);
       for (const p of checkBrief(j, jobBudgets)) problems.push(jobs.length > 1 ? `job ${i + 1}: ${p}` : p);
     });
     // verify_regression re-runs `verification`; with no profile set there is
     // nothing to re-run. Refuse before starting anything, matching every other
     // admission check here, rather than silently no-op at runtime.
+    // stakes applies to what gets committed; reviews names the job a scout reviews.
+    jobs.forEach((j, i) => {
+      const at = jobs.length > 1 ? `job ${i + 1}: ` : "";
+      if (j.stakes === "high" && (j.mode ?? "implement") !== "implement") problems.push(`${at}stakes: high applies to implement jobs; for a review, send a scout with reviews: <job id>`);
+      if (j.reviews) {
+        if (j.mode !== "scout") problems.push(`${at}reviews: <job id> marks a scout as a review of that job; this job is ${j.mode ?? "implement"}`);
+        else if (!fs.existsSync(path.join(jobsRoot, j.reviews, "metadata.json"))) problems.push(`${at}reviews: no finished job ${j.reviews} to review (see local_worker_jobs)`);
+      }
+    });
+    // continue_from: the retained job must exist, be unfinished and uncommitted,
+    // and belong to this repo; see lib/continue-from.mjs.
+    jobs.forEach((j, i) => {
+      if (!j.continue_from) return;
+      const { problem } = continuationProblem({ continueFrom: j.continue_from, mode: j.mode, baseRef: j.base_ref ?? null, jobsRoot, projectDir,
+        isRunning: (id) => activeJobs.get(id)?.settled === false });
+      if (problem) problems.push(`${jobs.length > 1 ? `job ${i + 1}: ` : ""}${problem}`);
+    });
     jobs.forEach((j, i) => {
       if (j.verify_regression && !j.verification) {
         problems.push(`${jobs.length > 1 ? `job ${i + 1}: ` : ""}verify_regression requires a verification profile; there is nothing to run twice without one`);
@@ -196,6 +291,7 @@ export function createJobRuntime(deps) {
     // clean, so a missing on_behalf_of is never reported twice in two
     // different shapes.
     jobs.forEach((j, i) => {
+      if (j.mode === "verify") return;
       const fieldProblems = subscriptionJobFieldProblems(j);
       for (const p of fieldProblems) problems.push(jobs.length > 1 ? `job ${i + 1}: ${p}` : p);
       if (fieldProblems.length === 0 && j.on_behalf_of) {
@@ -211,26 +307,63 @@ export function createJobRuntime(deps) {
     // Claude CLI) isn't bounded by the sandbox, so it's refused unless that
     // agent says allow_host_tools (lib/agents.mjs). Scouts and reviews still run.
     jobs.forEach((j, i) => {
-      if (!j.agentName || (j.mode ?? "implement") !== "implement") return;
+      if (j.mode === "verify" || !j.agentName || (j.mode ?? "implement") !== "implement") return;
       let problem = null;
       try { problem = hostToolsImplementProblem(j.agentName, agentsConfig().agents[j.agentName]); } catch { return; }
       if (problem) problems.push(`${jobs.length > 1 ? `job ${i + 1}: ` : ""}${problem}`);
     });
-    // A model its vendor refused on a job today, with nothing working on it
-    // since, isn't sent another job (lib/health.mjs recentModelRefusal).
+    // Re-test stale refusals once per day before making an admission decision.
+    const now = deps.now?.() ?? Date.now();
+    const refusedJobs = new Map();
     jobs.forEach((j, i) => {
-      if (!j.agentName || !j.model) return;
+      if (j.mode === "verify" || !j.agentName || !j.model) return;
       let provider = null;
       try { provider = agentProviderId(agentsConfig().agents[j.agentName]); } catch { return; }
       if (!provider) return;
-      const refusal = recentModelRefusal(stateRoot, `${provider}/${j.model}`);
-      if (refusal) problems.push(`${jobs.length > 1 ? `job ${i + 1}: ` : ""}model_not_found: ${provider}/${j.model} was refused on an earlier job today and hasn't worked since, so this job wasn't sent. Use another model (the job's \`model\`, or \`nomarmy army assign\`); \`nomarmy army assign <role> ${j.agentName} ${j.model}\` re-tests it, and a passing test clears this.`);
+      const key = `${provider}/${j.model}`;
+      if (!recentModelRefusal(stateRoot, key, { now })) return;
+      if (!refusedJobs.has(key)) refusedJobs.set(key, { provider, model: j.model, jobs: [] });
+      refusedJobs.get(key).jobs.push(i);
     });
+    await Promise.all([...refusedJobs.entries()].map(async ([key, entry]) => {
+      const hadRefusal = Boolean(modelRefusals(stateRoot)[key]);
+      if (hadRefusal && deps.probeModel && claimModelRefusalRetry(stateRoot, key, { now })) {
+        try {
+          const result = await deps.probeModel({ provider: entry.provider, model: entry.model, stateRoot });
+          if (result.ok) { recordProbeSuccess(stateRoot, key, { now }); return; }
+          if (result.refused) recordModelRefusal(stateRoot, key, result.reason, { now });
+        } catch { /* A failed probe is inconclusive; keep the original refusal. */ }
+      }
+      for (const i of entry.jobs) problems.push(`${jobs.length > 1 ? `job ${i + 1}: ` : ""}model_not_found: ${key} was refused on an earlier job and hasn't worked since, so this job wasn't sent. Use another model or re-test with nomarmy army assign.`);
+    }));
+    // A model that has never worked for its agent gets one test call first
+    // (lib/model-probe.mjs): a refusal then costs seconds, not a whole job.
+    // The server wires the probe; without one (tests), nothing is probed.
+    if (deps.probeModel && !problems.length) {
+      const pending = new Map();
+      jobs.forEach((j, i) => {
+        if (j.mode === "verify" || !j.agentName || !j.model) return;
+        let provider = null;
+        try { provider = agentProviderId(agentsConfig().agents[j.agentName]); } catch { return; }
+        const key = provider && `${provider}/${j.model}`;
+        if (!key || modelProven(stateRoot, key) || modelRefusals(stateRoot)[key]) return;
+        if (!pending.has(key)) pending.set(key, { provider, model: j.model, jobs: [] });
+        pending.get(key).jobs.push({ i, agentName: j.agentName });
+      });
+      await Promise.all([...pending.entries()].map(async ([key, p]) => {
+        let result;
+        try { result = await deps.probeModel({ provider: p.provider, model: p.model, stateRoot }); } catch { return; }
+        if (result.ok) { recordProbeSuccess(stateRoot, key); return; }
+        if (!result.refused) return; // inconclusive: let the job find out
+        recordModelRefusal(stateRoot, key, result.reason);
+        for (const { i, agentName } of p.jobs) problems.push(`${jobs.length > 1 ? `job ${i + 1}: ` : ""}model_not_found: ${key} refused a test call before this job was sent${result.reason ? ` (${result.reason})` : ""}. Pick another of ${agentName}'s models (the job's \`model\`; the \`army\` tool lists them).`);
+      }));
+    }
     // An agent's max_concurrent, machine-wide. Batch jobs on the same agent
     // queue for its slot at launch instead (withAgentSlot's waitMs).
     if (jobs.length === 1) {
       const [j] = jobs;
-      const max = j.agentName ? agentMaxConcurrent(j.agentName) : null;
+      const max = j.mode !== "verify" && j.agentName ? agentMaxConcurrent(j.agentName) : null;
       const held = max ? liveSlots(slotsRoot, j.agentName) : 0;
       if (max && held >= max) problems.push(`not admitted (capacity): agent "${j.agentName}" already has ${held} job(s) running across this machine's nomArmy sessions, at its max_concurrent of ${max}`);
     }
@@ -241,7 +374,7 @@ export function createJobRuntime(deps) {
         const run = loadRun(runsRoot, j.run_id);
         if (run.repo !== projectDir) problems.push(`${jobs.length > 1 ? `job ${i + 1}: ` : ""}run "${run.id}" belongs to ${run.repo}, not this repository`);
         const running = liveLeases(leasesRoot, { runId: run.id }).length + jobs.slice(0, i).filter((o) => o.run_id === run.id).length;
-        for (const p of runAdmissionProblems(run, { agentName: j.agentName ?? "local", running })) problems.push(jobs.length > 1 ? `job ${i + 1}: ${p}` : p);
+        for (const p of runAdmissionProblems(run, { agentName: j.mode === "verify" ? null : j.agentName ?? "local", running })) problems.push(jobs.length > 1 ? `job ${i + 1}: ${p}` : p);
       } catch (error) { problems.push(jobs.length > 1 ? `job ${i + 1}: ${error.message}` : error.message); }
     });
     // Slot capacity only concerns local jobs: a remote job's inference runs
@@ -254,17 +387,31 @@ export function createJobRuntime(deps) {
     const anyLocal = jobs.some((j) => jobLane(j) === "local");
     const admission = anyLocal
       ? assessAdmission({ hardware: admissionHardware(), runningJobs: runningCount("local"), slots: deps.budgetState.contextInfo.slots, maxWorkers: currentMaxWorkers() })
-      : assessAdmission({ hardware: admissionHardware(), runningJobs: 0, slots: null, maxWorkers: Infinity });
+      : assessAdmission({ hardware: jobs.every(j => j.mode === "verify") ? null : admissionHardware(), runningJobs: 0, slots: null, maxWorkers: Infinity });
     if (!admission.admit) problems.push(...admission.reasons.map(r => `not admitted (${admission.level}): ${r}`));
     if (jobs.some((j) => jobLane(j) === "remote")) {
       const remoteCeiling = currentMaxPoolWorkers(), runningRemote = runningCount("remote");
       if (runningRemote >= remoteCeiling) {
-        problems.push(`not admitted (capacity): ${runningRemote} remote job(s) (api or subscription agents) already running, at NOMARMY_MAX_POOL_WORKERS=${remoteCeiling}`);
+        problems.push(`not admitted (capacity): ${runningRemote} remote job(s) (api or subscription agents) already running, the limit of ${remoteCeiling} at once (raise it with \`nomarmy config max-jobs <n>\`)`);
       }
     }
+    for (const note of usageNotes) admission.reasons.push(note);
     return { problems, admission };
   }
 
+  async function displayedCapacity() {
+    const now = deps.now?.() ?? Date.now();
+    const refreshed = await refreshStaleOverLimitReadings(stateRoot, { run: deps.usageRun, now, timeoutMs: deps.usageRefreshTimeoutMs });
+    const view = capacitySnapshot(refreshed.snapshots);
+    if (refreshed.error) {
+      for (const provider of refreshed.failedProviders ?? []) {
+        if (!view.usageLimits[provider]) continue;
+        view.usageLimits[provider] = { ...view.usageLimits[provider], text: `${view.usageLimits[provider].text}. ${refreshed.error}` };
+      }
+    }
+    return view;
+  }
+
   function refusal(problems) {
     return toolText(refusalText(problems, capacitySnapshot), true);
   }
@@ -285,17 +432,17 @@ export function createJobRuntime(deps) {
    */
   function recordJobInRun(args, jobId, result, error = null) {
     if (!args.run_id) return;
-    const kind = args.pool ? "api" : args.subscription_worker ? "subscription" : "local";
+    const kind = args.mode === "verify" ? "verify" : args.pool ? "api" : args.subscription_worker ? "subscription" : "local";
     const m = result?.manifest ?? {};
     const errorLines = [m.worker?.error, error?.message,
       ...String(m.workerError ?? "").split(/\r?\n/).filter((l) => /error|limit|429/i.test(l))].filter(Boolean).join("\n");
-    const usageLimit = kind === "local" ? null : detectUsageLimit(errorLines);
+    const usageLimit = (kind === "local" || kind === "verify") ? null : detectUsageLimit(errorLines);
     try {
       const before = runTotals(loadRun(runsRoot, args.run_id)).warnings;
       const updated = recordRunJob(runsRoot, args.run_id, {
-        jobId, agent: args.agentName ?? "local", kind, model: args.model ?? null, role: args.armyRole ?? null, mode: args.mode,
-        outcome: m.outcome ?? (error ? "ERROR" : null), costUsd: m.metrics?.worker_cost_usd ?? null,
-        tokens: m.metrics?.worker_tokens_total ?? null, usageLimit,
+        jobId, agent: kind === "verify" ? null : args.agentName ?? "local", kind, model: kind === "verify" ? null : args.model ?? null, role: args.armyRole ?? null, mode: args.mode,
+        outcome: m.outcome ?? (error ? "ERROR" : null), costUsd: kind === "verify" ? 0 : m.metrics?.worker_cost_usd ?? null,
+        tokens: kind === "verify" ? 0 : m.metrics?.worker_tokens_total ?? null, usageLimit,
       });
       // A limit crossed or an agent paused by this job is worth interrupting for.
       const fresh = runTotals(updated).warnings.filter((w) => !before.includes(w) && /OVER|paused/.test(w));
@@ -311,7 +458,7 @@ export function createJobRuntime(deps) {
   function launch(args) {
     const workerId = args.worker_id || null;
     const jobId = slug(workerId || (args.mode === "scout" ? "scout" : "worker"));
-    return trackInRun(args, track(jobId, { mode: args.mode, workerId: workerId || jobId, lane: jobLane(args), agent: args.agentName ?? null, runId: args.run_id ?? null, role: args.armyRole ?? null, model: args.model ?? null },
+    return trackInRun(args, track(jobId, { mode: args.mode, workerId: workerId || jobId, lane: jobLane(args), agent: args.agentName ?? null, runId: args.run_id ?? null, role: args.armyRole ?? null, model: args.model ?? null, label: jobLabel(args) },
       withAgentSlot(args, jobId, () => executeJob({ ...jobArgs(args, workerId), jobId }))));
   }
   // Best-effort progress signal for a job still mid-run: a plain "phase: worker,
@@ -357,12 +504,13 @@ export function createJobRuntime(deps) {
 
   async function summarize(entry, files, jobDir = null) {
     const status = files.status, meta = files.meta ?? files.failure;
-    const elapsedSeconds = status?.startedAt ? Math.round((Date.now() - Date.parse(status.startedAt)) / 1000) : entry ? Math.round((Date.now() - Date.parse(entry.startedAt)) / 1000) : null;
+    const elapsedSeconds = jobElapsedSeconds({ status, meta, entry });
     const out = { jobId: entry?.jobId ?? status?.jobId ?? meta?.jobId ?? null, workerId: entry?.workerId ?? status?.workerId ?? meta?.workerId ?? null,
       mode: entry?.mode ?? status?.mode ?? meta?.mode ?? null, state: null, phase: status?.phase ?? "starting", elapsedSeconds,
       timeoutSeconds: status?.timeoutSeconds ?? null, coordinatorStatus: meta?.coordinatorStatus ?? null, outcome: meta?.outcome ?? null,
       reviewRequired: meta?.reviewRequired ?? null, issues: (meta?.issues ?? []).slice(0, 6), worktree: meta?.worktree ?? null, branch: meta?.branch ?? null,
       commit: meta?.commit?.sha ?? null, scout: meta?.scout ? { supported: meta.scout.supported, unsupported: meta.scout.unsupported } : null };
+    if (meta?.mode === "verify") Object.assign(out, { verification: meta.verification, baseRef: meta.baseRef, baseSha: meta.baseSha });
     if (entry && !entry.settled) out.state = "running";
     else if (entry?.error) { out.state = "failed"; out.error = String(entry.error.message ?? entry.error).split("\n")[0]; }
     else if (meta) out.state = "finished";
@@ -372,5 +520,5 @@ export function createJobRuntime(deps) {
     return out;
   }
 
-  return { WORKER_START_STAGGER_MS, activeJobs, runningCount, agentMaxConcurrent, withAgentSlot, track, notifyJobFinished, capacitySnapshot, admit, refusal, runBrief, recordJobInRun, trackInRun, launch, liveProgress, summarize };
+  return { WORKER_START_STAGGER_MS, activeJobs, runningCount, agentMaxConcurrent, withAgentSlot, track, notifyJobFinished, capacitySnapshot, displayedCapacity, admit, refusal, runBrief, recordJobInRun, trackInRun, launch, liveProgress, summarize };
 }

package/lib/agents.mjs

@@ -30,12 +30,14 @@ import fs from "node:fs";
 import path from "node:path";
 import YAML from "yaml";
 import { z } from "zod";
+import { typeError, isMissingField, isUnknownDiscriminator } from "./zod-issues.mjs";
 
 import { providerEntrySchema, formatDispatchIssues, openclawProviderId, PROVIDER_TYPES, ID_RE, HOST_TOOL_PROVIDERS, agentRunsToolsOnHost } from "./dispatch-schema.mjs";
 
 export { HOST_TOOL_PROVIDERS, agentRunsToolsOnHost };
 import { findProviderConflicts, describeProviderConflict } from "./subscription-config.mjs";
 import { privateConfigProblem } from "./army.mjs";
+import { THINKING_LEVELS } from "./thinking.mjs";
 
 export const AGENTS_FILENAME = "agents.yml";
 export const AGENT_KINDS = Object.freeze(["local", "api", "subscription"]);
@@ -45,10 +47,10 @@ export const RESERVED_AGENT_NAMES = Object.freeze(["__proto__", "constructor", "
 export const BUILTIN_LOCAL_AGENT = Object.freeze({ kind: "local", slot: "coder" });
 
 const requiredString = () =>
-  z.string({ required_error: "is required", invalid_type_error: "must be a string" })
+  z.string({ error: typeError("a string") })
     .refine((value) => value.trim().length > 0, { message: "must not be empty" });
-const thinkingSchema = z.union([z.boolean(), z.enum(["low", "medium", "high"])]).default(true);
-const positiveInt = () => z.number({ invalid_type_error: "must be a number" }).int("must be a whole number").positive("must be a positive number");
+const thinkingSchema = z.union([z.boolean(), z.enum(THINKING_LEVELS)]).default(true);
+const positiveInt = () => z.number({ error: typeError("a number") }).int("must be a whole number").positive("must be a positive number");
 
 const localAgentSchema = z.object({
   kind: z.literal("local"),
@@ -62,7 +64,7 @@ const localAgentSchema = z.object({
 // providerEntrySchema in agentsFileSchema's superRefine, so they exist once.
 const apiAgentSchema = z.object({
   kind: z.literal("api"),
-  provider: z.enum(API_PROVIDER_TYPES, { errorMap: () => ({ message: `must be one of ${API_PROVIDER_TYPES.join(", ")}` }) }),
+  provider: z.enum(API_PROVIDER_TYPES, { error: `must be one of ${API_PROVIDER_TYPES.join(", ")}` }),
   model: requiredString().optional(),
   auth_env: requiredString(),
   base_url: z.string().optional(),
@@ -95,7 +97,7 @@ export function hostToolsImplementProblem(name, agent) {
 }
 
 export const agentSchema = z.discriminatedUnion("kind", [localAgentSchema, apiAgentSchema, subscriptionAgentSchema], {
-  errorMap: (issue, ctx) => (issue.code === "invalid_union_discriminator" ? { message: `kind must be one of ${AGENT_KINDS.join(", ")}` } : { message: ctx.defaultError }),
+  error: (issue) => (isUnknownDiscriminator(issue) ? `kind must be one of ${AGENT_KINDS.join(", ")}` : undefined),
 });
 
 /** An api agent as the pool entry the dispatch path understands. `model` may be absent (see resolveAgentModel). */
@@ -148,7 +150,7 @@ export function formatAgentIssues(error) {
   const lines = error.issues.map((issue) => {
     const where = issue.path.length ? issue.path.join(".") : "config";
     if (issue.code === "unrecognized_keys") return `${where}: unexpected field(s) ${issue.keys.map((k) => `"${k}"`).join(", ")}`;
-    if (issue.code === "invalid_type" && issue.received === "undefined") return `${where}: is required`;
+    if (isMissingField(issue)) return `${where}: is required`;
     return `${where}: ${issue.message}`;
   });
   return [...new Set(lines)];

package/lib/army.mjs

@@ -29,7 +29,9 @@ import os from "node:os";
 import path from "node:path";
 import YAML from "yaml";
 import { z } from "zod";
+import { typeError } from "./zod-issues.mjs";
 import { agentRunsToolsOnHost } from "./dispatch-schema.mjs";
+import { usageStatus, usageDisplayText } from "./usage-limits.mjs";
 
 export const GLOBAL_CONFIG_FILENAME = "config.yml";
 export const PROJECT_CONFIG_FILENAMES = Object.freeze([".nomarmy.yml", ".nomarmy.yaml"]);
@@ -115,7 +117,7 @@ const roleSchema = z.object({
   disabled: z.boolean().optional(),
 }).strict();
 
-const positiveNumber = () => z.number({ invalid_type_error: "must be a number" }).positive("must be positive");
+const positiveNumber = () => z.number({ error: typeError("a number") }).positive("must be positive");
 // Limits on one /feature run (lib/runs.mjs). Personal, like the General:
 // global or local only, never a committed project file.
 const runLimitsSchema = z.object({
@@ -152,11 +154,13 @@ function formatIssues(error) {
   });
 }
 
-// The global and local files hold only `army:` for now (the global one is
-// where machine-level settings such as the local model will move later);
-// strict, so a typo'd key is an error rather than silently ignored. The
-// project file is .nomarmy.yml, whose other sections lib/schema.mjs owns.
+// Global configuration is army-only; the local file also admits the separately
+// validated verification network policy. Both stay strict against unknown keys.
+// The project file is .nomarmy.yml, whose other sections lib/schema.mjs owns.
 const armyOnlyFileSchema = z.object({ army: armySchema.optional() }).strict();
+// Verification owns validation and use of this operator-local section. Army
+// updates must preserve it without permitting it in global configuration.
+const localArmyFileSchema = armyOnlyFileSchema.extend({ verification_network: z.unknown().optional() });
 
 /** Parse one layer's file and validate its `army:` section. Missing file or no section -> null. */
 export function readArmyFile(filePath, { armyOnly = false } = {}) {
@@ -168,7 +172,7 @@ export function readArmyFile(filePath, { armyOnly = false } = {}) {
     throw new ArmyConfigError(`${filePath} is not valid YAML: ${error.message}`, { path: filePath, errors: [`config: is not valid YAML: ${error.message}`] });
   }
   const doc = parsed ?? {};
-  const result = armyOnly ? armyOnlyFileSchema.safeParse(doc) : armySchema.optional().safeParse(doc.army);
+  const result = armyOnly ? (path.basename(filePath) === LOCAL_CONFIG_FILENAME ? localArmyFileSchema : armyOnlyFileSchema).safeParse(doc) : armySchema.optional().safeParse(doc.army);
   if (result.success) return (armyOnly ? result.data.army : result.data) ?? null;
   const errors = formatIssues(result.error).map((line) => (armyOnly ? line : `army.${line}`));
   throw new ArmyConfigError(`${filePath} has an invalid army section:\n${errors.map((l) => `  - ${l}`).join("\n")}`, { path: filePath, errors });
@@ -361,11 +365,22 @@ export function generalOverlap(army, agents = {}) {
  * description, phase, suggested mode, agent, any problem or overlap with
  * the General, and which layer set each field.
  */
-export function describeArmy(loaded, { agents = {}, describeAgent = null } = {}) {
+export function describeArmy(loaded, { agents = {}, describeAgent = null, usageSnapshots = null, agentProviderId = null, now = Date.now(), usageRefreshError = null, usageRefreshFailed = null } = {}) {
   const army = loaded.army;
   const problems = armyTargetProblems(army, agents);
   const overlap = generalOverlap(army, agents);
   const runsOn = (name) => (name && agents[name] && describeAgent ? describeAgent(agents[name]) : null);
+  const usage = (name) => {
+    if (!name || !agents[name] || !usageSnapshots || !agentProviderId) return null;
+    let provider;
+    try { provider = agentProviderId(agents[name]); } catch { return null; }
+    const snapshot = usageSnapshots[provider];
+    if (!snapshot) return null;
+    const status = usageStatus(snapshot, now);
+    const failed = new Set(usageRefreshFailed ?? []);
+    const text = failed.has(provider) && usageRefreshError ? `${usageDisplayText(status)}. ${usageRefreshError}` : usageDisplayText(status);
+    return { level: status.level, text };
+  };
   // A subscription whose owner isn't the General's own is worth a word,
   // not a refusal: it may be the same person's other account (a real Senti
   // General skipped a role's agent for exactly this, unsure whose it was).
@@ -378,7 +393,7 @@ export function describeArmy(loaded, { agents = {}, describeAgent = null } = {})
   const roles = Object.fromEntries(Object.entries(army.roles).map(([name, role]) => [name, {
     description: role.description ?? null, phase: role.phase ?? null, mode: role.mode ?? null,
     agent: role.agent ?? null, model: role.model ?? agents[role.agent]?.model ?? null, modelIsAuto: role.model === "auto",
-    agentRunsOn: runsOn(role.agent),
+    agentRunsOn: runsOn(role.agent), usage: usage(role.agent),
     // Its agent's own tools run on this machine (dispatch-schema.mjs): an
     // implement role there is refused per job unless the agent allows it.
     hostTools: agentRunsToolsOnHost(agents[role.agent]) ? { allowed: agents[role.agent].allow_host_tools === true, implementRole: (role.mode ?? "implement") === "implement" } : null,
@@ -388,13 +403,13 @@ export function describeArmy(loaded, { agents = {}, describeAgent = null } = {})
     ? "not defined -- run `nomarmy army general <agent>` so nomArmy can flag roles that share the General's model or usage"
     : !Object.prototype.hasOwnProperty.call(agents, army.general) ? `agent "${army.general}" is not defined in your agents.yml` : null;
   return {
-    general: { ...GENERAL, agent: army.general, agentRunsOn: runsOn(army.general), problem: generalProblem, setBy: loaded.sources.general },
+    general: { ...GENERAL, agent: army.general, agentRunsOn: runsOn(army.general), usage: usage(army.general), problem: generalProblem, setBy: loaded.sources.general },
     workflow: army.workflow,
     runLimits: army.runLimits ?? {},
     roles,
     layers: loaded.layers,
     howToDispatch: Object.keys(roles).length
-      ? "Pass army_role: \"<role>\" on a job (plus on_behalf_of when its agent is a subscription). The role's description goes at the top of the brief; set `mode` on the job yourself, the role's is only a suggestion. A role with model \"auto\" needs model on the job, picked from that agent's models below; any job may pass model to override the role's. Use agent: \"<name>\" instead to pick an agent directly."
+      ? "Pass army_role: \"<role>\" on a job (plus on_behalf_of when its agent is a subscription). The role's description goes at the top of the brief; set `mode` on the job yourself, the role's is only a suggestion. A role with model \"auto\" needs model on the job, picked from that agent's models below (never its refusedModels: its vendor turned those down on a job); any job may pass model to override the role's. Use agent: \"<name>\" instead to pick an agent directly."
       : "No army is configured. Run `nomarmy army init` for the default roster, or dispatch with agent: \"<name>\".",
   };
 }

package/lib/codex-link.mjs

@@ -0,0 +1,37 @@
+import os from "node:os";
+import path from "node:path";
+import { parseOpenclawAuthProfiles } from "./health.mjs";
+import { CODEX_IMPORT_ARGS, CODEX_IMPORT_RECOVERY, emailOpenaiProfiles, usableCodexImport } from "./openclaw-runtime-health.mjs";
+
+// Never relay migration output: --include-secrets may expose credential data.
+export async function linkCodex({ run, command = "openclaw", isTTY = false, removeEmailProfiles = false, confirm = async () => false, print = () => {}, codexDir = path.join(os.homedir(), ".codex"), now = Date.now() }) {
+  const read = async () => {
+    const result = await run(command, ["models", "auth", "list", "--json"]);
+    return result.ok ? parseOpenclawAuthProfiles(result.stdout) : null;
+  };
+  const profiles = await read();
+  if (!profiles) { print("Cannot inspect OpenClaw auth profiles; nothing imported."); return false; }
+  const emails = emailOpenaiProfiles(profiles);
+  if (emails.length) {
+    print(`Email-keyed profiles will capture the Codex import: ${emails.map((p) => p.id).join(", ")}.`);
+    if (!removeEmailProfiles && !(isTTY && await confirm("Remove these email-keyed profiles before importing?", { defaultYes: true }))) {
+      print("Stopped; use --remove-email-profiles to allow removal non-interactively.");
+      return false;
+    }
+    for (const p of emails) {
+      if (!(await run(command, ["models", "auth", "logout", p.id])).ok) { print("OpenClaw profile logout failed; nothing imported."); return false; }
+    }
+    const remaining = await read();
+    if (!remaining || emailOpenaiProfiles(remaining).length) { print("Email-keyed profiles still exist; nothing imported."); return false; }
+  }
+  print(`Linking OpenClaw: ${CODEX_IMPORT_RECOVERY}`);
+  const args = CODEX_IMPORT_ARGS.map((arg) => arg === "~/.codex" ? codexDir : arg);
+  if (!(await run(command, args)).ok) { print("Codex import failed."); return false; }
+  const imported = await read();
+  if (!imported || emailOpenaiProfiles(imported).length || !usableCodexImport(imported, now)) {
+    print("Sign-in has no usable auth profile: expected an unexpired openai:account- profile labeled (Codex import).");
+    return false;
+  }
+  print("Confirmed an unexpired openai:account- (Codex import) profile.");
+  return true;
+}

package/lib/config.mjs

@@ -10,6 +10,8 @@
 import fs from "node:fs";
 import path from "node:path";
 import YAML from "yaml";
+import { validateEnabledHarnesses } from "./harnesses.mjs";
+import { readRegistrySecrets } from "./registry-secrets.mjs";
 
 import {
   collectElevated,
@@ -125,6 +127,13 @@ export function validateConfig(input) {
     };
   }
 
+  if (Object.hasOwn(candidate, "verification_network")) {
+    return { valid: false, config: null, errors: ["verification_network is operator-local only: put it in gitignored .nomarmy.local.yml, never .nomarmy.yml"], elevated: emptyElevated() };
+  }
+  if (Object.hasOwn(candidate, "registries")) {
+    return { valid: false, config: null, errors: ["registries: allowed only in .nomarmy.local.yml, never in committed repository configuration"], elevated: emptyElevated() };
+  }
+
   const result = configSchema.safeParse(candidate);
   if (!result.success) {
     return {
@@ -135,6 +144,10 @@ export function validateConfig(input) {
     };
   }
 
+  try { validateEnabledHarnesses(result.data.harnesses); }
+  catch (error) {
+    return { valid: false, config: null, errors: [error.message], elevated: emptyElevated() };
+  }
   return {
     valid: true,
     config: result.data,
@@ -154,6 +167,8 @@ export function validateConfig(input) {
  * @returns {{ found: boolean, path: string|null, config: object|null, elevated: { shared: string[], remote: string[] } }}
  */
 export function loadConfig(repoDir = process.cwd()) {
+  try { readRegistrySecrets(repoDir); }
+  catch (error) { throw new ConfigError(error.message, { errors: [error.message] }); }
   const configPath = findConfigFile(repoDir);
   if (!configPath) {
     return { found: false, path: null, config: null, elevated: emptyElevated() };