nomarmy 0.1.0-alpha.2 → 0.1.0-alpha.21

package/lib/verify.mjs

@@ -13,7 +13,7 @@
 // to repo-controlled code, the exact privilege the worker itself is denied.
 //
 // Therefore commands only ever execute inside the Podman sandbox image, as a
-// non-root user, with `--network none`. If Podman is missing, the image is
+// non-root user, with `--network none` or a verification-only internal service network. If Podman is missing, the image is
 // absent, or the container fails to start, the verdict is `not_run`. There is
 // no host fallback path in this file, deliberately: a missing sandbox is
 // absence of evidence, not permission to take a shortcut.
@@ -22,12 +22,17 @@
 // is spawned with an argv array and `shell: false`. A command string is handed
 // to `/bin/sh -c` *inside* the container, which is inside the sandbox boundary.
 
+import { randomUUID, randomBytes } from "node:crypto";
 import { spawn } from "node:child_process";
 import fs from "node:fs";
+import os from "node:os";
+import { collectVerificationArtifacts } from "./verification-artifacts.mjs";
 import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadVerificationNetwork, redactCredentials } from "./verification-network.mjs";
 
 import { loadConfig as defaultLoadConfig } from "./config.mjs";
-import { resolveSandboxImage, nodeDependencyFiles, linkNodePackages, SANDBOX_NPM_ENV } from "./sandbox-images.mjs";
+import { resolveSandboxImage, sandboxHarnesses, nodeDependencyFiles, linkNodePackages, SANDBOX_NPM_ENV } from "./sandbox-images.mjs";
 
 /** Sandbox image used when `NOMARMY_AGENT_IMAGE` is unset. */
 export const DEFAULT_AGENT_IMAGE = "openclaw-nomarmy-coder:bookworm";
@@ -130,10 +135,19 @@ function tailOf(result) {
  * command that never started is absence of evidence, so if nothing ever ran the
  * verdict is `not_run`, never `fail`.
  *
+ * A command guarded on nomArmy's changed-file variables (`if [ -n
+ * "$NOMARMY_CHANGED_TEST_FILES" ]; then ...; fi`) exits 0 without running
+ * anything when those are empty, as on an unchanged checkout in mode: verify.
+ * That's skipped, not passed: exit 0, no output at all, and every
+ * NOMARMY_CHANGED_* variable the command names empty. A command with a
+ * fallback (`${NOMARMY_CHANGED_TEST_FILES:-tests/}`) prints test output, so
+ * it counts as run. If every command skipped, nothing was verified: not_run.
+ *
  * @param {Array<object>} results
- * @returns {{ status: "pass"|"fail"|"not_run", detail: string }}
+ * @param {{ env?: Record<string, string> }} [context] the NOMARMY_CHANGED_* values the commands saw
+ * @returns {{ status: "pass"|"fail"|"not_run", detail: string, basis?: string }}
  */
-export function classifyResults(results) {
+export function classifyResults(results, { env = {} } = {}) {
   const list = Array.isArray(results) ? results.filter(Boolean) : [];
   if (list.length === 0) {
     return { status: "not_run", detail: "no commands were executed" };
@@ -171,9 +185,28 @@ export function classifyResults(results) {
     }
   }
 
+  const skipped = list.map((result, index) => ({ index, why: guardedNoOp(result, env) })).filter((s) => s.why);
+  if (skipped.length === total) {
+    return { status: "not_run", basis: "nothing-changed", detail: `every command is scoped to changed files and nothing changed, so none ran (${[...new Set(skipped.map((s) => s.why))].join("; ")}); add an unscoped profile to run the full suite` };
+  }
+  if (skipped.length) {
+    const which = skipped.map((s) => `command ${s.index + 1} (\`${list[s.index].command ?? "?"}\`) did nothing: ${s.why}`).join("; ");
+    return { status: "pass", detail: `${total - skipped.length} of ${total} commands passed; ${skipped.length} skipped: ${which}` };
+  }
   return { status: "pass", detail: `${total} of ${total} commands passed` };
 }
 
+const CHANGED_VAR_RE = /\$\{?(NOMARMY_CHANGED_[A-Z_]+)/g;
+
+/** Why a zero-exit, silent command was a guarded no-op, or null if it ran. */
+function guardedNoOp(result, env) {
+  if (result.exitCode !== 0 || result.timedOut || result.started === false) return null;
+  if (result.silent === false || (result.silent === undefined && `${result.stdout ?? ""}${result.stderr ?? ""}`.trim())) return null;
+  const names = [...new Set([...String(result.command ?? "").matchAll(CHANGED_VAR_RE)].map((m) => m[1]))];
+  if (!names.length || names.some((name) => String(env[name] ?? "").trim())) return null;
+  return `${names.join(" and ")} ${names.length > 1 ? "are" : "is"} empty`;
+}
+
 // ---------------------------------------------------------------------------
 // pure: output capping
 // ---------------------------------------------------------------------------
@@ -229,6 +262,8 @@ export function buildPodmanArgs({
   workdir = DEFAULT_WORKDIR,
   nodeModulesSource = null,
   env = {},
+  secretEnv = [],
+  shmMb = null,
 } = {}) {
   const args = [
     "run",
@@ -256,7 +291,7 @@ export function buildPodmanArgs({
   // the var unquoted inside its OWN `/bin/sh -c` still word-splits on spaces
   // as usual -- that is the operator's shell, not this one, and is worth
   // knowing when a repo's paths might contain spaces.
-  for (const [name, value] of Object.entries(env)) args.push("--env", `${name}=${value}`);
+  for (const [name, value] of Object.entries(env)) args.push("--env", secretEnv.includes(name) ? name : `${name}=${value}`);
   // `git worktree add` never copies node_modules, and this container has
   // --network none, so a Node repo's own verification commands can never
   // install what they need. The coordinator's own already-installed tree is
@@ -268,6 +303,7 @@ export function buildPodmanArgs({
   if (nodeModulesSource) {
     args.push("--mount", `type=bind,source=${nodeModulesSource},target=${workdir}/node_modules,readonly`);
   }
+  if (shmMb > 0) args.push(`--shm-size=${shmMb}m`);
   if (jobId) args.push("--label", `nomarmy.job=${jobId}`);
   args.push("--entrypoint", "/bin/sh", image, "-c", command);
   return args;
@@ -277,13 +313,13 @@ export function buildPodmanArgs({
 // the real Podman executor (the only place a host process is spawned)
 // ---------------------------------------------------------------------------
 
-function spawnCollect(file, args, { timeoutMs, maxOutputBytes, cwd } = {}) {
+function spawnCollect(file, args, { timeoutMs, maxOutputBytes, cwd, env } = {}) {
   return new Promise((resolve) => {
     let child;
     try {
       // shell:false is the whole point: nothing here is ever parsed by a host
       // shell, so a config value cannot break out of its argv slot.
-      child = spawn(file, args, { cwd, shell: false, windowsHide: true });
+      child = spawn(file, args, { cwd, env, shell: false, windowsHide: true });
     } catch (error) {
       resolve({ spawned: false, code: null, stdout: "", stderr: String(error?.message || error), timedOut: false });
       return;
@@ -337,11 +373,116 @@ function spawnCollect(file, args, { timeoutMs, maxOutputBytes, cwd } = {}) {
  * Executor backed by the real Podman CLI. Tests inject a fake in its place.
  * @param {{ podman?: string }} [options]
  */
-export function createPodmanExecutor({ podman = "podman" } = {}) {
+export function createPodmanExecutor({ podman = "podman", collect = spawnCollect, healthTimeoutMs = 30_000, healthIntervalMs = 250 } = {}) {
   return {
+    /** Own only resources created by this run; cleanup is also used on setup failure. */
+    async startServices({ services, image, jobId, allow = [], token }) {
+      const network = `nomarmy-${String(jobId || randomUUID()).replace(/[^a-zA-Z0-9_-]/g, "-")}`;
+      const containers = [];
+      let created = false;
+      let externalCreated = false;
+      const external = `${network}-external`;
+      const proxyName = `${network}-egress`;
+      const call = (args, timeoutMs = 20_000) => collect(podman, args, { timeoutMs, maxOutputBytes: 4096, env: { ...process.env, NOMARMY_EGRESS_TOKEN: token } });
+      const ok = (result) => result.spawned && !result.timedOut && result.code === 0;
+      const requireSuccess = async (args, label, timeoutMs) => {
+        if (!ok(await call(args, timeoutMs))) throw new Error(label);
+      };
+      const cleanup = async () => {
+        const failures = [];
+        try { await this.cleanupVerification({ jobId }); }
+        catch { failures.push("could not remove verification containers"); }
+        for (const name of containers.reverse()) {
+          try { await requireSuccess(["rm", "--force", "--ignore", name], `could not remove service container ${name}`); }
+          catch (error) { failures.push(error.message); }
+        }
+        if (externalCreated) {
+          try { await requireSuccess(["network", "rm", external], "could not remove egress network"); }
+          catch (error) { failures.push(error.message); }
+        }
+        if (created) {
+          try { await requireSuccess(["network", "rm", network], `could not remove service network ${network}`); }
+          catch (error) { failures.push(error.message); }
+        }
+        if (failures.length) throw new Error(failures.join("; "));
+      };
+      try {
+        await requireSuccess(["network", "create", "--internal", network], `could not create internal service network ${network}`);
+        created = true;
+        if (allow.length) {
+          await requireSuccess(["network", "create", external], "could not create egress network");
+          externalCreated = true;
+          containers.push(proxyName);
+          await requireSuccess(["run", "--detach", "--name", proxyName,
+            "--network", `${network}:alias=egress`, "--network", external,
+            `--user=${DEFAULT_CONTAINER_USER}`, "--cap-drop=ALL", "--security-opt=no-new-privileges",
+            "--read-only", "--pids-limit=128", "--memory=128m", "--cpus=1",
+            "--mount", `type=bind,source=${fileURLToPath(new URL("./egress-proxy.mjs", import.meta.url))},target=/egress-proxy.mjs,readonly`,
+            "--env", "NOMARMY_EGRESS_TOKEN",
+            "--env", `NOMARMY_EGRESS_ALLOW=${JSON.stringify(allow)}`,
+            "--entrypoint", "node", DEFAULT_AGENT_IMAGE, "/egress-proxy.mjs"], "could not start egress proxy");
+          const probeName = `${network}-egress-health`;
+          containers.push(probeName);
+          const script = `const net=await import('node:net');const end=Date.now()+${healthTimeoutMs};while(Date.now()<end){const ok=await new Promise(r=>{const s=net.connect(3128,'egress');s.setTimeout(1000);s.on('connect',()=>{s.destroy();r(true)});s.on('error',()=>r(false));s.on('timeout',()=>{s.destroy();r(false)});});if(ok)process.exit(0);await new Promise(r=>setTimeout(r,${healthIntervalMs}));}process.exit(1);`;
+          await requireSuccess(["run", "--rm", "--name", probeName, "--network", network,
+            `--user=${DEFAULT_CONTAINER_USER}`, "--cap-drop=ALL", "--security-opt=no-new-privileges",
+            "--entrypoint", "node", DEFAULT_AGENT_IMAGE, "--input-type=module", "-e", script], "egress proxy did not become ready", healthTimeoutMs + 5000);
+        }
+        for (const service of services) {
+          if (!ok(await call(["image", "exists", service.image]))) {
+            await requireSuccess(["pull", service.image], `could not pull service image ${service.image}`, 120_000);
+          }
+          const name = `${network}-service-${service.name}`;
+          containers.push(name);
+          await requireSuccess(["run", "--detach", "--name", name, "--network", network,
+            "--network-alias", service.name, "--cap-drop=ALL", "--security-opt=no-new-privileges",
+            "--pids-limit=512", ...Object.entries(service.env ?? {}).flatMap(([k, v]) => ["--env", `${k}=${v}`]),
+            service.image], `could not start service ${service.name}`);
+        }
+        for (const service of services.filter((spec) => spec.health)) {
+          const probeName = `${network}-health-${service.name}`;
+          containers.push(probeName);
+          const url = `http://${service.name}:${service.port}${service.health}`;
+          // Poll inside the same internal network, never on the host. The base
+          // verification image supplies Node, so services need no curl binary.
+          const script = `const deadline=Date.now()+${healthTimeoutMs}; while(Date.now()<deadline){try{const r=await fetch(${JSON.stringify(url)},{redirect:'error',signal:AbortSignal.timeout(Math.min(2000,Math.max(1,deadline-Date.now())))});if(r.status>=200&&r.status<300)process.exit(0);}catch{}await new Promise(r=>setTimeout(r,${healthIntervalMs}));}process.exit(1);`;
+          await requireSuccess(["run", "--rm", "--name", probeName, "--network", network,
+            `--user=${DEFAULT_CONTAINER_USER}`, "--cap-drop=ALL", "--security-opt=no-new-privileges",
+            "--entrypoint", "node", image, "--input-type=module", "-e", script],
+            `service ${service.name} did not become healthy within ${healthTimeoutMs}ms (${url})`, healthTimeoutMs + 5000);
+        }
+        return { network, cleanup, ...(allow.length ? { logs: async () => {
+          // Freeze the audit trail before reading it; no late DNS completion or
+          // service connection may append a verdict after this snapshot.
+          await requireSuccess(["stop", "--time", "2", proxyName], "could not stop egress proxy");
+          const logLimit = 4 * 1024 * 1024;
+          const result = await collect(podman, ["logs", proxyName], { timeoutMs: 20_000, maxOutputBytes: logLimit });
+          // Never accept verification with a silently incomplete audit trail.
+          if (!ok(result) || Buffer.byteLength(result.stdout) >= logLimit) throw new Error("could not capture complete egress verdict log");
+          // Accept only the proxy's fixed verdict format; never relay diagnostics.
+          return result.stdout.split("\n").filter(line => /^[a-z0-9.-]+:[0-9]+ (connected|denied)$/.test(line)).join("\n");
+        } } : {}) };
+      } catch (error) {
+        try { await cleanup(); } catch (cleanupError) { throw new Error(`${error.message}; ${cleanupError.message}`); }
+        throw error;
+      }
+    },
+
+    async cleanupVerification({ jobId }) {
+      if (!jobId) return;
+      const listed = await collect(podman, ["ps", "-aq", "--filter", `label=nomarmy.job=${jobId}`], { timeoutMs: 20_000, maxOutputBytes: 1024 * 1024 });
+      if (!listed.spawned || listed.timedOut || listed.code !== 0 || Buffer.byteLength(listed.stdout) >= 1024 * 1024) throw new Error("could not list verification containers");
+      const ids = listed.stdout.trim().split(/\s+/).filter(Boolean);
+      if (ids.some(id => !/^[a-f0-9]+$/.test(id))) throw new Error("invalid verification container id");
+      if (ids.length) {
+        const removed = await collect(podman, ["rm", "--force", ...ids], { timeoutMs: 20_000, maxOutputBytes: 4096 });
+        if (!removed.spawned || removed.timedOut || removed.code !== 0) throw new Error("could not remove verification containers");
+      }
+    },
+
     /** Is a usable sandbox present? Never throws. */
     async probe({ image }) {
-      const version = await spawnCollect(podman, ["version", "--format", "{{.Server.Version}}"], {
+      const version = await collect(podman, ["version", "--format", "{{.Server.Version}}"], {
         timeoutMs: 20_000,
         maxOutputBytes: 4096,
       });
@@ -358,7 +499,7 @@ export function createPodmanExecutor({ podman = "podman" } = {}) {
         };
       }
 
-      const inspect = await spawnCollect(podman, ["image", "inspect", image], {
+      const inspect = await collect(podman, ["image", "inspect", image], {
         timeoutMs: 20_000,
         maxOutputBytes: 4096,
       });
@@ -372,10 +513,10 @@ export function createPodmanExecutor({ podman = "podman" } = {}) {
     },
 
     /** Run one command inside the sandbox. Never throws. */
-    async run({ command, cwd, image, jobId, network, user, workdir, timeoutMs, maxOutputBytes, nodeModulesSource, env }) {
-      const args = buildPodmanArgs({ image, cwd, command, jobId, network, user, workdir, nodeModulesSource, env });
+    async run({ command, cwd, image, jobId, network, user, workdir, timeoutMs, maxOutputBytes, nodeModulesSource, env, secretEnv, shmMb }) {
+      const args = buildPodmanArgs({ image, cwd, command, jobId, network, user, workdir, nodeModulesSource, env, secretEnv, shmMb });
       const started = Date.now();
-      const outcome = await spawnCollect(podman, args, { timeoutMs, maxOutputBytes });
+      const outcome = await collect(podman, args, { timeoutMs, maxOutputBytes, env: { ...process.env, ...Object.fromEntries((secretEnv ?? []).filter(name => Object.hasOwn(env ?? {}, name)).map(name => [name, env[name]])) } });
       const durationMs = Date.now() - started;
 
       if (!outcome.spawned) {
@@ -465,6 +606,11 @@ function pluralCommands(count, name) {
   return `${count} command${count === 1 ? "" : "s"} in profile '${name}'`;
 }
 
+/** Largest matched shared-memory requirement in MiB; zero leaves Podman defaults. */
+export function verificationShmMb(cwd, config, selection = sandboxHarnesses(cwd, config)) {
+  return Math.max(0, ...selection.matched.map((name) => selection.harnesses[name].requires.shmMb ?? 0));
+}
+
 /**
  * Build the verification runner to hand to `registerVerificationRunner`.
  *
@@ -506,8 +652,24 @@ export function createVerificationRunner(options = {}) {
     sandboxImageRun = undefined,
   } = options;
 
-  return async function runVerification(context = {}) {
-    const { profile = null, cwd = null, jobId = null, record = null } = context;
+  // Snapshot operator authority before any worker runs. Never load local policy
+  // from context.cwd, config objects, or harness definitions.
+  let networkPolicy = null, networkPolicyError = null;
+  try { networkPolicy = (options.loadVerificationNetwork ?? loadVerificationNetwork)(hostProjectDir); }
+  catch (error) { networkPolicyError = error.message; }
+
+  async function runVerification(context = {}) {
+    let { profile = null, cwd = null, jobId = randomUUID(), record = null } = context;
+    jobId ||= randomUUID();
+    let registryNote = null;
+    const notRun = (reason, basis, extra = {}) => ({ status: "not_run", basis, reason, detail: registryNote, ...extra });
+    const onRegistryNote = (note) => {
+      registryNote = note;
+      if (record) {
+        record.issues ??= [];
+        if (!record.issues.includes(note)) record.issues.push(note);
+      }
+    };
 
     if (!cwd || typeof cwd !== "string") {
       return notRun("no worktree path was supplied, so nothing could be verified", "no-worktree");
@@ -537,17 +699,46 @@ export function createVerificationRunner(options = {}) {
     } catch (error) {
       // A broken contract is not a failing test suite. Say so and stop.
       return notRun(
-        `.nomarmy.yml could not be loaded: ${String(error?.message || error).split("\n")[0]}`,
+        `.nomarmy.yml could not be loaded: ${error.errors?.find(line => line.startsWith("verification_network ")) ?? String(error?.message || error).split("\n")[0]}`,
         "config-error",
       );
     }
     const config = loaded && loaded.found ? loaded.config : null;
+    if (config && Object.hasOwn(config, "verification_network")) return notRun("verification_network is operator-local only: use .nomarmy.local.yml", "config-error");
+    if (networkPolicyError) return notRun(networkPolicyError, "config-error");
+    const credentialEnv = {}, secrets = [];
+    const networkRecord = networkPolicy ? { allowlist: networkPolicy.allow, reached: [], credentials: Object.keys(networkPolicy.env) } : null;
+    for (const [name, source] of Object.entries(networkPolicy?.env ?? {})) {
+      const value = (options.hostEnv ?? process.env)[source];
+      if (typeof value !== "string" || !value.length) return notRun(`missing host environment variable ${source}`, "missing-credential", { network: networkRecord });
+      credentialEnv[name] = value;
+      secrets.push(value);
+    }
+    const credentialsInUse = secrets.length > 0;
+    const withheld = "output withheld: credentials were in use (set verification_network.keep_output: true in .nomarmy.local.yml to keep it, redacted)";
+    const hideOutput = credentialsInUse && networkPolicy.keep_output !== true;
+    const token = networkPolicy ? randomBytes(32).toString("hex") : null;
+    if (token) secrets.push(token);
+    const redact = value => redactCredentials(value, secrets);
+    const networkInfo = networkRecord ? { network: networkRecord, issues: [`verification had network access to ${networkRecord.allowlist.join(", ")}`] } : {};
+
+    let selection;
+    try { selection = sandboxHarnesses(cwd, config); }
+    catch (error) { return notRun(error.message, "config-error"); }
+    const specs = selection.matched.map((name) => selection.harnesses[name]);
+    if (!networkPolicy && specs.some((spec) => spec.network === "allowlist")) return notRun("allowlist harnesses require operator-local provisioning", "environment-not-provisioned");
+    const services = specs.flatMap((spec) => spec.services ?? []);
+    if (new Set(services.map((service) => service.name)).size !== services.length) return notRun("duplicate harness service names", "config-error");
+    if (networkPolicy && services.some(s => s.name === "egress")) return notRun("service alias egress is reserved for the verification proxy", "config-error");
+    const harnessEnv = Object.assign({}, ...specs.map((spec) => spec.env ?? {}));
+    const shmMb = verificationShmMb(cwd, config, selection);
 
     const explicitImage = image || process.env.NOMARMY_AGENT_IMAGE || null;
     let sandboxImage;
     try {
       sandboxImage = resolveSandboxImage({
         cwd, explicitImage, defaultImage: DEFAULT_AGENT_IMAGE, config,
+        trustedDir: hostProjectDir, onNote: onRegistryNote,
         ...(sandboxImageRun ? { run: sandboxImageRun } : {}),
       });
     } catch (error) {
@@ -615,85 +806,147 @@ export function createVerificationRunner(options = {}) {
       nodeModulesSource = mount.source;
     }
 
-    // --- execute, sequentially, inside the sandbox ----------------------
-    const deadline = now() + overallTimeoutMs;
-    const results = [];
-
-    for (const command of commands) {
-      const remaining = deadline - now();
-      if (remaining <= 0) {
-        results.push({
-          command,
-          started: false,
-          timedOut: true,
-          reason: `overall verification budget of ${overallTimeoutMs}ms was exhausted before this command started`,
-          timeoutMs: overallTimeoutMs,
-        });
-        break;
+    let disposable;
+    try {
+      // Whenever a proxy token or credentials are issued, run on a throwaway
+      // copy: anything a command writes (a token in a new file) can't reach
+      // the worktree nomArmy commits from.
+      if (credentialsInUse || token) {
+        disposable = fs.mkdtempSync(path.join(os.tmpdir(), "nomarmy-verification-"));
+        const copy = path.join(disposable, "worktree");
+        fs.cpSync(cwd, copy, { recursive: true, dereference: false, verbatimSymlinks: true, filter: source => path.basename(source) !== ".git" });
+        cwd = copy;
       }
+      // --- execute, sequentially, inside the sandbox ----------------------
+      const deadline = now() + overallTimeoutMs;
+      const results = [];
 
-      const timeoutMs = Math.max(1, Math.min(commandTimeoutMs, remaining));
-      let raw;
+      let serviceRun;
+      let proxyLog = "";
+      const renderOutput = () => (proxyLog ? `${proxyLog}\n\n` : "") + results.map((r) => `$ ${r.command}  (exit ${r.exitCode ?? "none"}${r.timedOut ? ", timed out" : ""}${networkPolicy ? `, ${r.durationMs ?? 0}ms` : ""})\n${r.stdout ?? ""}${r.stderr ? `\n[stderr]\n${r.stderr}` : ""}`).join("\n\n");
       try {
-        raw = await executor.run({
-          command,
-          cwd,
-          image: sandboxImage,
-          jobId,
-          network,
-          user,
-          workdir,
-          timeoutMs,
-          maxOutputBytes,
-          nodeModulesSource,
-          env: verificationEnv,
-        });
+        if (services.length || networkPolicy) serviceRun = await executor.startServices({ services, image: sandboxImage, jobId, ...(networkPolicy ? { allow: networkPolicy.allow, token } : {}) });
       } catch (error) {
-        raw = { started: false, reason: `sandbox execution threw: ${String(error?.message || error)}` };
+        return notRun(redact(`service setup failed: ${error.message}`), "services-unavailable");
+      }
+      const proxyEnv = networkPolicy ? Object.fromEntries([
+        ...["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"].map(name => [name, `http://nomarmy:${token}@egress:3128`]),
+        ...["NO_PROXY", "no_proxy"].map(name => [name, services.map(s => s.name).join(",")]),
+      ]) : {};
+      try {
+        for (const command of commands) {
+          const remaining = deadline - now();
+          if (remaining <= 0) {
+            results.push({
+              command,
+              started: false,
+              timedOut: true,
+              reason: `overall verification budget of ${overallTimeoutMs}ms was exhausted before this command started`,
+              timeoutMs: overallTimeoutMs,
+            });
+            break;
+          }
+
+          const timeoutMs = Math.max(1, Math.min(commandTimeoutMs, remaining));
+          let raw;
+          try {
+            raw = await executor.run({
+              command,
+              cwd,
+              image: sandboxImage,
+              jobId,
+              network: serviceRun?.network ?? network,
+              user,
+              workdir,
+              timeoutMs,
+              maxOutputBytes,
+              nodeModulesSource,
+              env: { ...harnessEnv, ...verificationEnv, ...credentialEnv, ...proxyEnv },
+              secretEnv: [...Object.keys(credentialEnv), ...["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"]],
+              shmMb,
+            });
+          } catch (error) {
+            raw = { started: false, reason: `sandbox execution threw: ${String(error?.message || error)}` };
+          }
+
+          const stdout = hideOutput ? { text: withheld, dropped: 0 } : capOutput(redact(raw?.stdout), maxOutputBytes);
+          const stderr = capOutput(hideOutput ? "" : redact(raw?.stderr), maxOutputBytes);
+          const entry = {
+            command: redact(command),
+            started: raw?.started !== false,
+            timedOut: Boolean(raw?.timedOut),
+            exitCode: Number.isInteger(raw?.exitCode) ? raw.exitCode : null,
+            reason: raw?.reason == null ? null : hideOutput ? withheld : redact(raw.reason),
+            durationMs: Number.isFinite(raw?.durationMs) ? raw.durationMs : null,
+            timeoutMs,
+            stdout: stdout.text,
+            stderr: stderr.text,
+            // Whether the command printed nothing at all, judged before any
+            // output is withheld, so a guarded no-op is still recognized.
+            silent: !String(raw?.stdout ?? "").trim() && !String(raw?.stderr ?? "").trim(),
+            truncated: { stdout: stdout.dropped, stderr: stderr.dropped },
+          };
+          results.push(entry);
+
+          // Stop at the first command that did not cleanly pass: later commands
+          // would run against an already-broken tree and add nothing.
+          if (!entry.started || entry.timedOut || entry.exitCode !== 0) break;
+        }
+
+      } finally {
+        if (serviceRun) {
+          let logError;
+          try { if (serviceRun.logs) proxyLog = redactCredentials(await serviceRun.logs(), secrets, { truncated: false }); }
+          catch { logError = true; }
+          if (networkRecord) networkRecord.reached = [...new Set(proxyLog.split("\n").filter(line => line.endsWith(" connected")).map(line => line.split(" ")[0]))];
+          try { await serviceRun.cleanup(); }
+          catch (error) { return notRun(redact(`service cleanup failed: ${error.message}`), "services-cleanup-failed", { ...networkInfo, output: renderOutput() }); }
+          if (logError) return notRun("could not capture egress verdict log", "egress-log-failed", { ...networkInfo, output: renderOutput() });
+        } else if (executor.cleanupVerification) {
+          await executor.cleanupVerification({ jobId });
+        }
       }
 
-      const stdout = capOutput(raw?.stdout, maxOutputBytes);
-      const stderr = capOutput(raw?.stderr, maxOutputBytes);
-      const entry = {
-        command,
-        started: raw?.started !== false,
-        timedOut: Boolean(raw?.timedOut),
-        exitCode: Number.isInteger(raw?.exitCode) ? raw.exitCode : null,
-        reason: raw?.reason ?? null,
-        durationMs: Number.isFinite(raw?.durationMs) ? raw.durationMs : null,
-        timeoutMs,
-        stdout: stdout.text,
-        stderr: stderr.text,
-        truncated: { stdout: stdout.dropped, stderr: stderr.dropped },
-      };
-      results.push(entry);
-
-      // Stop at the first command that did not cleanly pass: later commands
-      // would run against an already-broken tree and add nothing.
-      if (!entry.started || entry.timedOut || entry.exitCode !== 0) break;
-    }
+      const verdict = classifyResults(results, { env: verificationEnv });
+      const executed = results.filter((r) => r.started).length;
+      const dropped = results.reduce(
+        (total, r) => total + (r.truncated?.stdout ?? 0) + (r.truncated?.stderr ?? 0),
+        0,
+      );
 
-    const verdict = classifyResults(results);
-    const executed = results.filter((r) => r.started).length;
-    const dropped = results.reduce(
-      (total, r) => total + (r.truncated?.stdout ?? 0) + (r.truncated?.stderr ?? 0),
-      0,
-    );
+      const detailSuffix = dropped > 0
+        ? ` [${dropped} bytes of output dropped by the ${maxOutputBytes}-byte cap]`
+        : "";
 
-    const detailSuffix = dropped > 0
-      ? ` [${dropped} bytes of output dropped by the ${maxOutputBytes}-byte cap]`
-      : "";
+      if (verdict.status === "not_run") {
+        return notRun(verdict.detail, verdict.basis ?? "sandbox-unavailable", { ...networkInfo, output: renderOutput() });
+      }
 
-    if (verdict.status === "not_run") {
-      return notRun(verdict.detail, "sandbox-unavailable");
+      return {
+        ...networkInfo,
+        // No evidence files are kept while credentials or a proxy token are in
+        // use: a scan can't catch every encoding (UTF-16, wrapped, compressed).
+        ...(credentialsInUse || token ? { artifacts: [], artifactsCapped: false, artifactsNote: "artifacts not kept: credentials or a proxy token were in use" } : {}),
+        status: verdict.status,
+        basis: `${basis}; ${executed} executed in ${sandboxImage}`,
+        reason: null,
+        detail: `${verdict.detail}${detailSuffix}${registryNote ? "; " + registryNote : ""}`,
+        // Each command's full (capped) output, for a caller that keeps a log.
+        output: renderOutput(),
+      };
+    } finally {
+      if (disposable) fs.rmSync(disposable, { recursive: true, force: true });
     }
-
-    return {
-      status: verdict.status,
-      basis: `${basis}; ${executed} executed in ${sandboxImage}`,
-      reason: null,
-      detail: `${verdict.detail}${detailSuffix}`,
-    };
+  }
+  return async (context = {}) => {
+    let result;
+    try { result = await runVerification(context); }
+    catch (error) {
+      if (!networkPolicy) throw error;
+      result = notRun("verification could not complete safely", "runner-error");
+    }
+    if (networkPolicy && !result.network) result.network = { allowlist: networkPolicy.allow, reached: [], credentials: Object.keys(networkPolicy.env) };
+    return result;
   };
 }
 

package/lib/worker-prompt.mjs

@@ -69,10 +69,13 @@ export function describeRecoveryChanges(record) {
   return `${record.repoStatusFiles.length} file(s) differ from a clean checkout: ${record.repoStatusFiles.join(", ")}`;
 }
 
-export function reportRecoveryPrompt({ report = { targetTokens: 256, hardCapTokens: 512 }, changes = null } = {}) {
+export function reportRecoveryPrompt({ report = { targetTokens: 256, hardCapTokens: 512 }, changes = null, task = null } = {}) {
+  // The objective itself, so STATUS is judged against it even when the
+  // resumed session lost it with the cut-off reply.
+  const objective = task ? `\nThe objective you were working on:\n${String(task).trim()}\n` : "";
   const changesLine = changes
     ? `\nThe repository (checked independently just now, not from your memory of this session) already shows: ${changes}. Trust this over any uncertainty about what you did or did not do.\n`
     : `\nThe repository (checked independently just now, not from your memory of this session) shows no changes at all.\n`;
-  return `Your previous reply ended without the required final report, or was cut off before completing it.\n${changesLine}\nDo not repeat, redo, retry, or describe any action you already took. Do not call any tool. Reply with ONLY the four lines below, nothing before them, nothing after them:\n\nSTATUS: done | partial | blocked\nTESTS: pass | fail | not_run\nNOT_DONE: none | <brief>\nNOTE: <brief implementation or risk note>\n\nUse the exact field names above, including the underscore in NOT_DONE. Target ${report.targetTokens} tokens; ${report.hardCapTokens} is the hard cap. Base STATUS on the repository state above, not on what you recall attempting: if it shows the edit landed, you may report done; if it shows nothing relevant, report blocked or partial rather than guessing done.`;
+  return `Your previous reply ended without the required final report, or was cut off before completing it.\n${objective}${changesLine}\nDo not repeat, redo, retry, or describe any action you already took. Do not call any tool. Reply with ONLY the four lines below, nothing before them, nothing after them:\n\nSTATUS: done | partial | blocked\nTESTS: pass | fail | not_run\nNOT_DONE: none | <brief>\nNOTE: <brief implementation or risk note>\n\nUse the exact field names above, including the underscore in NOT_DONE. Target ${report.targetTokens} tokens; ${report.hardCapTokens} is the hard cap. Base STATUS on the repository state above, not on what you recall attempting: if it shows the edit landed, you may report done; if it shows nothing relevant, report blocked or partial rather than guessing done.`;
 }