c8ctl-plugin-nano 1.56.4 → 1.57.1

package/c8ctl-plugin.js

@@ -42,6 +42,7 @@ import {
   readdirSync,
   chmodSync,
   renameSync,
+  linkSync,
   realpathSync,
   statfsSync,
   lstatSync,
@@ -141,6 +142,10 @@ const READINESS_TIMEOUT_MS = 60_000;
 const READINESS_POLL_MS = 500;
 const HEALTH_TIMEOUT_MS = 1_500;
 const STOP_GRACE_MS = 8_000;
+// #202: the signal a graceful `supervisor stop`/`workforce stop` sends each
+// `nano work` child to quiesce it — stop leasing new jobs, finish in-flight work,
+// then exit. SIGTERM/SIGINT remain the FORCE abort (kill harness, yield jobs).
+const SUPERVISOR_DRAIN_SIGNAL = 'SIGUSR2';
 // Upper bound on one `--auto` engine-read reconcile (enumerate deployed
 // definitions + fetch each BPMN). A read that stalls past this is treated as a
 // transient failure so the running poller set is KEPT and, crucially, shutdown
@@ -4423,9 +4428,16 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, envelope,
   return out;
 }
 
-// Reap leftover job workspaces under the runs root. Age-gated, skips in-flight
-// run dirs, best-effort, bounded to our own directory (never touches anything we
-// did not create).
+// LEGACY flat reaper (pre-issue-#205). Reaps `run-*`/`res-*` directly under the
+// runs ROOT — the shared-namespace design that caused the cross-worker data-loss
+// incident. It is NO LONGER CALLED by the worker (which now allocates a private
+// `worker-<incarnation>/` namespace and uses `reapOwnedNamespace` +
+// `reclaimOrphanNamespaces`), and is retained ONLY so that (a) a mixed-version
+// host with an OLD worker still running this code stays interoperable, and (b)
+// the mixed-version regression test can assert this flat sweep never descends
+// into a new `worker-*` namespace (it only matches the `run-`/`res-` prefixes at
+// the root, so `worker-*` dirs are invisible to it). Do not reintroduce it into
+// the worker cleanup path.
 function reapAgentRunDirs({ maxAgeMs = 0, liveRunDirs = new Set() } = {}) {
   let reaped = 0;
   const root = agentRunsRoot();
@@ -4454,13 +4466,321 @@ function reapAgentRunDirs({ maxAgeMs = 0, liveRunDirs = new Set() } = {}) {
   return { reaped };
 }
 
+// ---- Cross-process-safe worker run/result namespaces (issue #205) ----------
+// The old shared flat `agent-runs/{run,res}-*` layout let ANY worker process's
+// age-gated reaper delete another live worker's active checkout or result
+// channel: `reapAgentRunDirs` excludes only ITS OWN caller's in-flight dirs, so
+// a sibling process (empty `liveRunDirs`) treated an aged-but-active dir as
+// garbage and removed it (data loss — see the incident in issue #205). Age is
+// not evidence of completion; editing files inside a checkout does not refresh
+// the enclosing dir's mtime.
+//
+// The fix isolates each worker PROCESS INCARNATION under its own
+// `agent-runs/worker-<incarnation>/` namespace and splits cleanup in two:
+//
+//   * OWNER-SCOPED ordinary cleanup — a worker's startup/periodic reaper only
+//     ever traverses its OWN namespace, where its `liveRunDirs` set is the sole
+//     authority. It can never see, let alone delete, a sibling's dir.
+//   * ORPHAN RECLAMATION — a separate, cross-process-safe sweep that may reclaim
+//     ANOTHER incarnation's namespace ONLY on positive proof the owner process
+//     is dead (PID-reuse-safe) and no harness it spawned survives, under an
+//     exclusive lock with a final recheck. Unknown ownership/liveness always
+//     RETAINS and logs an actionable diagnostic, never deletes.
+//
+// The `worker-` namespace prefix is deliberately invisible to the legacy flat
+// `run-*`/`res-*` sweep, so an OLD worker still running the pre-fix code cannot
+// enter a new namespace, and the new reaper never touches unowned legacy flat
+// dirs. See issue #205 and the mixed-version rollout note in the README.
+
+const WORKER_NS_PREFIX = 'worker-';
+const OWNER_RECORD = 'owner.json';
+const RECLAIM_LOCK = '.reclaiming';
+const LIVE_MARKER_DIR = 'live';
+
+// A process-start fingerprint for `pid`, used to defeat PID reuse: a recorded
+// owner is only "the same process" if the PID is alive AND its start token still
+// matches. Best-effort + cross-platform: Linux reads field 22 (starttime) from
+// `/proc/<pid>/stat`; elsewhere (macOS/BSD) it shells out to `ps -o lstart=`.
+// Returns null when neither source is available — callers treat a null/absent
+// token conservatively (cannot prove reuse ⇒ never reclaim).
+function pidStartToken(pid = process.pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return null;
+  try {
+    const stat = readFileSync(`/proc/${pid}/stat`, 'utf-8');
+    // comm (field 2) is parenthesised and may itself contain spaces/parens, so
+    // split AFTER the last ')'. starttime is overall field 22 ⇒ index 19 of the
+    // remaining whitespace-separated fields.
+    const rparen = stat.lastIndexOf(')');
+    if (rparen !== -1) {
+      const rest = stat.slice(rparen + 2).trim().split(/\s+/);
+      const starttime = rest[19];
+      if (starttime && /^\d+$/.test(starttime)) return `lx:${starttime}`;
+    }
+  } catch { /* not Linux / no procfs */ }
+  try {
+    const out = execFileSync('ps', ['-o', 'lstart=', '-p', String(pid)], { encoding: 'utf-8', timeout: 5_000 }).trim();
+    if (out) return `ps:${out}`;
+  } catch { /* best effort */ }
+  return null;
+}
+
+// A fresh, unique-per-process incarnation id. NOT the reusable configured worker
+// name: two processes running the same profile must land in distinct namespaces.
+function newIncarnationId() {
+  return `${process.pid}-${Date.now().toString(36)}-${randomBytes(6).toString('hex')}`;
+}
+
+// Sanitise an incarnation id for safe use as a single path segment.
+function workerNamespaceDir(incarnation, root = agentRunsRoot()) {
+  return join(root, `${WORKER_NS_PREFIX}${String(incarnation).replace(/[^\w.#-]/g, '_')}`);
+}
+
+function readOwnerRecord(nsDir) {
+  try {
+    const rec = JSON.parse(readFileSync(join(nsDir, OWNER_RECORD), 'utf-8'));
+    if (!rec || typeof rec !== 'object' || typeof rec.incarnation !== 'string') return null;
+    return rec;
+  } catch { return null; }
+}
+
+// Allocate (idempotently) this incarnation's namespace and publish its immutable
+// ownership record ATOMICALLY *before* any reapable child dir can exist inside
+// it. A concurrent reclaimer that catches the dir mid-creation sees no owner
+// record and RETAINS it (never reclaims an incompletely-registered namespace).
+function allocateWorkerNamespace({ incarnation, worker = null, pid = process.pid, pidStart = null, version = null, root = agentRunsRoot() } = {}) {
+  const nsDir = workerNamespaceDir(incarnation, root);
+  mkdirSync(nsDir, { recursive: true });
+  const ownerFile = join(nsDir, OWNER_RECORD);
+  if (!existsSync(ownerFile)) {
+    const owner = { schema: 1, incarnation, worker, pid, pidStart, host: hostname(), createdAt: new Date().toISOString(), version };
+    const tmp = `${ownerFile}.${process.pid}.${Date.now()}.tmp`;
+    writeFileSync(tmp, JSON.stringify(owner, null, 2));
+    // Publish EXCLUSIVELY: linkSync is an atomic create-if-absent on POSIX and
+    // Windows, so a racing allocator can never overwrite an already-published
+    // owner record (the immutability guarantee). EEXIST ⇒ a peer won the race,
+    // which is success — the record is immutable, so whoever wrote it is fine.
+    try {
+      linkSync(tmp, ownerFile);
+    } catch (err) {
+      if (err?.code !== 'EEXIST') { try { rmSync(tmp, { force: true }); } catch { /* */ } throw err; }
+    } finally {
+      try { rmSync(tmp, { force: true }); } catch { /* */ }
+    }
+  }
+  mkdirSync(join(nsDir, LIVE_MARKER_DIR), { recursive: true });
+  return { nsDir, owner: readOwnerRecord(nsDir) };
+}
+
+// Classify a recorded owner's incarnation as 'alive' | 'dead' | 'unknown'.
+// PID-reuse-safe: a live PID whose start token no longer matches the record is a
+// DIFFERENT process ⇒ the recorded incarnation is 'dead'. A missing pid, or a
+// live pid we cannot re-fingerprint (null token on either side), is 'unknown' —
+// never 'dead' — so reclamation errs toward preservation.
+function incarnationLiveness(owner, { isAlive = isPidAlive, startToken = pidStartToken } = {}) {
+  if (!owner || !Number.isInteger(owner.pid) || owner.pid <= 0) return 'unknown';
+  let alive;
+  try { alive = isAlive(owner.pid); } catch { return 'unknown'; }
+  if (!alive) return 'dead';
+  if (owner.pidStart == null) return 'unknown'; // never recorded ⇒ can't disprove reuse
+  let current;
+  try { current = startToken(owner.pid); } catch { current = null; }
+  if (current == null) return 'unknown'; // can't re-fingerprint ⇒ conservative
+  return current === owner.pidStart ? 'alive' : 'dead';
+}
+
+function jobMarkerPath(nsDir, jobKey) {
+  return join(nsDir, LIVE_MARKER_DIR, `${String(jobKey).replace(/[^\w.#-]/g, '_')}.json`);
+}
+
+// Record that a job is in-flight in this namespace (a harness may be spawned).
+// Written BEFORE the harness starts so a crash mid-spawn still leaves evidence
+// that a harness could be orphaned (an empty `harnessPids` ⇒ retain conservatively).
+function writeJobMarker(nsDir, { jobKey, workerPid = process.pid, incarnation = null }) {
+  try {
+    mkdirSync(join(nsDir, LIVE_MARKER_DIR), { recursive: true });
+    const p = jobMarkerPath(nsDir, jobKey);
+    writeFileSync(p, JSON.stringify({ jobKey: String(jobKey), workerPid, incarnation, harnessPids: [], startedAt: new Date().toISOString() }));
+    return p;
+  } catch { return null; }
+}
+
+// Append the spawned harness's PID (its process-group leader) to the in-flight
+// job marker, so orphan reclamation can probe whether it survived the worker.
+function recordHarnessPid(nsDir, jobKey, pid) {
+  if (!nsDir || !Number.isInteger(pid) || pid <= 0) return;
+  try {
+    const p = jobMarkerPath(nsDir, jobKey);
+    const rec = JSON.parse(readFileSync(p, 'utf-8'));
+    if (!Array.isArray(rec.harnessPids)) rec.harnessPids = [];
+    if (!rec.harnessPids.includes(pid)) {
+      rec.harnessPids.push(pid);
+      const tmp = `${p}.${process.pid}.tmp`;
+      writeFileSync(tmp, JSON.stringify(rec));
+      renameSync(tmp, p);
+    }
+  } catch { /* best effort */ }
+}
+
+function removeJobMarker(nsDir, jobKey) {
+  try { rmSync(jobMarkerPath(nsDir, jobKey), { force: true }); } catch { /* best effort */ }
+}
+
+function readJobMarkers(nsDir) {
+  let names;
+  try { names = readdirSync(join(nsDir, LIVE_MARKER_DIR)); }
+  catch (err) {
+    // ENOENT ⇒ the live/ dir never existed: genuinely no in-flight markers.
+    // Any other error (EACCES, transient FS) means marker liveness is UNKNOWN,
+    // so return a synthetic malformed marker to force a conservative retain
+    // (never reclaim a namespace whose harness state we couldn't determine).
+    if (err && err.code === 'ENOENT') return [];
+    return [{ jobKey: '(unreadable)', harnessPids: [], malformed: true }];
+  }
+  const out = [];
+  for (const n of names) {
+    if (!n.endsWith('.json')) continue;
+    try { out.push(JSON.parse(readFileSync(join(nsDir, LIVE_MARKER_DIR, n), 'utf-8'))); }
+    catch { out.push({ jobKey: n, harnessPids: [], malformed: true }); }
+  }
+  return out;
+}
+
+// Does a (presumed-dead-owner) namespace still have a possibly-live harness? An
+// in-flight marker that is malformed or records no harness pid (a registration
+// race — the job started but the harness pid was not yet recorded) is treated as
+// possibly-live (conservative). Otherwise probe each recorded pid.
+function namespaceHasLiveHarness(nsDir, { isAlive = isPidAlive } = {}) {
+  for (const m of readJobMarkers(nsDir)) {
+    const pids = Array.isArray(m.harnessPids) ? m.harnessPids : [];
+    if (m.malformed || pids.length === 0) return true;
+    for (const pid of pids) { try { if (isAlive(pid)) return true; } catch { return true; } }
+  }
+  return false;
+}
+
+// Shared eligibility used by BOTH the startup and periodic owner-scoped sweeps
+// (so they can never drift): reap the `run-*`/`res-*` children of `dir` that are
+// (a) not in `liveRunDirs`, (b) older than `maxAgeMs`, (c) a real directory (an
+// lstat rejects a symlink — never followed). `onReap(path)` fires per removal for
+// evidence logging. Confined to `dir`; unrelated entries (owner.json, live/,
+// .reclaiming, operator files) are ignored.
+function reapChildRunDirs(dir, { maxAgeMs = 0, liveRunDirs = new Set(), now = Date.now(), onReap } = {}) {
+  let reaped = 0;
+  try {
+    if (!existsSync(dir)) return { reaped };
+    for (const name of readdirSync(dir)) {
+      if (!name.startsWith('run-') && !name.startsWith('res-')) continue;
+      const p = join(dir, name);
+      if (liveRunDirs.has(p)) continue;
+      try {
+        const st = lstatSync(p);
+        if (!st.isDirectory()) continue;
+        if (maxAgeMs > 0 && now - st.mtimeMs < maxAgeMs) continue;
+        rmSync(p, { recursive: true, force: true });
+        reaped++;
+        if (onReap) { try { onReap(p); } catch { /* */ } }
+      } catch { /* skip */ }
+    }
+  } catch (err) {
+    return { reaped, error: err.message };
+  }
+  return { reaped };
+}
+
+// OWNER-SCOPED ordinary cleanup: reap aged, not-in-flight job/result dirs inside THIS
+// worker's own namespace only. Safe by construction — no sibling shares this
+// namespace, and `liveRunDirs` authoritatively excludes in-flight dirs.
+function reapOwnedNamespace(nsDir, { maxAgeMs = 0, liveRunDirs = new Set(), now = Date.now(), logger, incarnation } = {}) {
+  return reapChildRunDirs(nsDir, {
+    maxAgeMs,
+    liveRunDirs,
+    now,
+    onReap: (p) => { if (logger) logger.info(`[reaper] ${new Date(now).toISOString()} incarnation=${incarnation ?? '?'} removed own aged, not-in-flight ${basename(p)} (owner-scoped)`); },
+  });
+}
+
+// ORPHAN RECLAMATION (cross-process-safe). Sweep sibling `worker-*` namespaces
+// and reclaim ONLY those whose owner incarnation is provably dead and which have
+// no surviving harness, under an exclusive per-namespace lock with a final
+// recheck. Everything uncertain — a missing/malformed owner record, a live or
+// unknown owner, a possibly-live harness, a lock held by another reclaimer, a
+// namespace younger than `minAgeMs`, a symlink, or a path escaping `root` — is
+// RETAINED with a diagnostic. Legacy flat `run-*`/`res-*` (no `worker-` prefix)
+// are never considered. Injectable probes (`liveness`, `harnessAlive`, `now`)
+// make every branch deterministically testable.
+function reclaimOrphanNamespaces({
+  root = agentRunsRoot(),
+  selfIncarnation = null,
+  now = Date.now(),
+  minAgeMs = 0,
+  liveness = incarnationLiveness,
+  harnessAlive = isPidAlive,
+  logger,
+} = {}) {
+  const reclaimed = [];
+  const retained = [];
+  const stamp = () => new Date(now).toISOString();
+  const note = (name, reason, owner) => {
+    retained.push({ name, reason });
+    if (logger) logger.info(`[reclaim] ${stamp()} actor=${selfIncarnation ?? '?'} target=${name} owner=${owner?.incarnation ?? 'unknown'}(pid ${owner?.pid ?? '?'}) RETAINED — ${reason}`);
+  };
+  let rootReal;
+  try { rootReal = realpathSync(root); } catch (err) {
+    if (logger) logger.info(`[reclaim] ${stamp()} actor=${selfIncarnation ?? '?'} RETAINED all — could not resolve runs root ${root} (${err.code || err.message})`);
+    return { reclaimed, retained };
+  }
+  let names;
+  try { names = readdirSync(root); } catch (err) {
+    if (logger) logger.info(`[reclaim] ${stamp()} actor=${selfIncarnation ?? '?'} RETAINED all — could not scan runs root ${root} (${err.code || err.message})`);
+    return { reclaimed, retained };
+  }
+  for (const name of names) {
+    if (!name.startsWith(WORKER_NS_PREFIX)) continue;
+    const nsDir = join(root, name);
+    let st;
+    try { st = lstatSync(nsDir); } catch (err) { note(name, `could not lstat (${err.code || err.message}) — retained`, null); continue; }
+    if (!st.isDirectory()) { note(name, 'not a directory (symlink?) — skipped', null); continue; }
+    if (minAgeMs > 0 && now - st.mtimeMs < minAgeMs) { note(name, 'younger than min reclaim age', null); continue; }
+    // Containment: the resolved namespace must sit directly under the resolved root.
+    let nsReal;
+    try { nsReal = realpathSync(nsDir); } catch (err) { note(name, `could not resolve real path (${err.code || err.message}) — retained`, null); continue; }
+    if (dirname(nsReal) !== rootReal) { note(name, 'path escapes runs root — skipped', null); continue; }
+    const owner = readOwnerRecord(nsDir);
+    if (!owner) { note(name, 'missing/malformed owner record — not garbage', null); continue; }
+    if (selfIncarnation && owner.incarnation === selfIncarnation) continue; // never our own
+    const state = liveness(owner);
+    if (state !== 'dead') { note(name, `owner ${state}`, owner); continue; }
+    if (namespaceHasLiveHarness(nsDir, { isAlive: harnessAlive })) { note(name, 'a spawned harness may still be alive', owner); continue; }
+    // Exclusive reclamation ownership: an atomic mkdir lock. A loser retains.
+    const lock = join(nsDir, RECLAIM_LOCK);
+    try { mkdirSync(lock); } catch { note(name, 'another reclaimer holds the lock', owner); continue; }
+    try {
+      // FINAL recheck under the lock: identity + liveness + harness must all hold.
+      const owner2 = readOwnerRecord(nsDir);
+      if (!owner2 || owner2.incarnation !== owner.incarnation) { note(name, 'ownership changed under lock', owner2 || owner); continue; }
+      if (liveness(owner2) !== 'dead') { note(name, 'owner became alive under lock', owner2); continue; }
+      if (namespaceHasLiveHarness(nsDir, { isAlive: harnessAlive })) { note(name, 'harness became live under lock', owner2); continue; }
+      let real2;
+      try { real2 = realpathSync(nsDir); } catch (err) { note(name, `could not resolve real path under lock (${err.code || err.message}) — retained`, owner2); continue; }
+      if (dirname(real2) !== rootReal) { note(name, 'path escaped runs root under lock', owner2); continue; }
+      rmSync(nsDir, { recursive: true, force: true });
+      reclaimed.push({ name, owner: owner2 });
+      if (logger) logger.info(`[reclaim] ${stamp()} actor=${selfIncarnation ?? '?'} target=${name} owner=${owner2.incarnation}(pid ${owner2.pid}) RECLAIMED — owner proven dead, no surviving harness`);
+    } finally {
+      // Drop the lock unless the whole namespace was reclaimed (lock gone with it).
+      try { if (existsSync(nsDir)) rmSync(lock, { recursive: true, force: true }); } catch { /* best effort */ }
+    }
+  }
+  return { reclaimed, retained };
+}
+
 // ---- One-shot capture (shared by host + container executors) ---------------
 const MAX_CAPTURE_BYTES = 1_048_576; // 1 MiB per stream
 
 // Spawn a child, pipe `stdinData`, capture byte-capped stdout/stderr, enforce a
 // timeout (invoking `onTimeout(child)` to tear the child down), and resolve to a
 // uniform result. Used by both the host and container executors.
-function spawnCaptureOneShot({ command, args = [], shell = false, detached = false, cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, onTimeout, stream = false, streamPrefix = '', onStreamOut, onStreamErr, relayTap = null }) {
+function spawnCaptureOneShot({ command, args = [], shell = false, detached = false, cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, onTimeout, stream = false, streamPrefix = '', onStreamOut, onStreamErr, relayTap = null, abortSignal = null, onSpawn = null }) {
   return new Promise((resolve) => {
     let child;
     const stdoutChunks = [];
@@ -4472,6 +4792,7 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
     let settled = false;
     let timer = null;
     let idleMon = null;
+    let onAbort = null;
 
     // Live "spy" tee (--stream): mirror the child's output line-by-line to a
     // caller-supplied emitter (the worker routes these through c8ctl's
@@ -4509,6 +4830,7 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
       settled = true;
       if (timer) clearTimeout(timer);
       if (idleMon) idleMon.stop();
+      if (onAbort && abortSignal) { try { abortSignal.removeEventListener('abort', onAbort); } catch { /* best effort */ } onAbort = null; }
       if (teeOut) teeOut('', true);
       if (teeErr) teeErr('', true);
       resolve(result);
@@ -4520,6 +4842,25 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
       finish({ ok: false, exitCode: null, stdout: '', stderr: '', error: err.message, truncated: false, stderrTruncated: false });
       return;
     }
+    if (onSpawn) { try { onSpawn(child.pid); } catch { /* best effort */ } }
+
+    // #202: a `stop --force`/abort aborts this signal — kill the harness process
+    // group (via the same onTimeout kill that the hard-cap/idle paths use) and
+    // settle as aborted so the worker fails/yields the job for immediate retry
+    // instead of letting the detached child outlive the interrupt (orphaned to
+    // init) and the lock lapse.
+    if (abortSignal) {
+      if (abortSignal.aborted) {
+        try { if (onTimeout) onTimeout(child); } catch { /* best effort */ }
+        finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: 'aborted', aborted: true, truncated: stdoutTruncated, stderrTruncated });
+        return;
+      }
+      onAbort = () => {
+        try { if (onTimeout) onTimeout(child); } catch { /* best effort */ }
+        finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: 'aborted', aborted: true, truncated: stdoutTruncated, stderrTruncated });
+      };
+      abortSignal.addEventListener('abort', onAbort);
+    }
 
     timer = timeoutMs && timeoutMs > 0
       ? setTimeout(() => {
@@ -4630,7 +4971,7 @@ function ptyAvailable(ptyFactory) {
 // spawnCaptureOneShot. A PTY merges stdout+stderr into one stream, so stderr is
 // always '' here; that is expected for a live terminal. `ptyFactory` is
 // injectable for tests (defaults to node-pty).
-function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, cols = 120, rows = 30, ptyFactory, relayTap = null, stream = false, streamPrefix = '', onStreamOut }) {
+function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, cols = 120, rows = 30, ptyFactory, relayTap = null, stream = false, streamPrefix = '', onStreamOut, abortSignal = null, onSpawn = null }) {
   return new Promise((resolve) => {
     const factory = ptyFactory || loadPtyModule();
     if (!factory || typeof factory.spawn !== 'function') {
@@ -4646,6 +4987,7 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
     let idleMon = null;
     let detachSteer = null;
     let term;
+    let onAbort = null;
 
     // Live "spy" tee (--stream), line-buffered, mirroring spawnCaptureOneShot.
     const STREAM_TEE_LINE_CAP = 64 * 1024;
@@ -4676,6 +5018,7 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
       if (timer) clearTimeout(timer);
       if (idleMon) idleMon.stop();
       if (detachSteer) { try { detachSteer(); } catch { /* best effort */ } detachSteer = null; }
+      if (onAbort && abortSignal) { try { abortSignal.removeEventListener('abort', onAbort); } catch { /* best effort */ } onAbort = null; }
       if (teeSink) tee('', true);
       resolve(result);
     };
@@ -4686,6 +5029,19 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
       finish({ ok: false, exitCode: null, stdout: '', stderr: '', error: `pty spawn failed: ${err?.message || err}`, truncated: false, stderrTruncated: false });
       return;
     }
+    if (onSpawn) { try { onSpawn(term?.pid); } catch { /* best effort */ } }
+
+    // #202: abort (stop --force) kills the PTY and settles as aborted so the job
+    // is failed/yielded for immediate retry rather than left to lock-lapse.
+    if (abortSignal) {
+      const abortNow = () => {
+        killTerm();
+        finish({ ok: false, exitCode: null, stdout: joinCapped(chunks), stderr: '', error: 'aborted', aborted: true, truncated, stderrTruncated: false });
+      };
+      if (abortSignal.aborted) { abortNow(); return; }
+      onAbort = abortNow;
+      abortSignal.addEventListener('abort', onAbort);
+    }
 
     timer = timeoutMs && timeoutMs > 0
       ? setTimeout(() => {
@@ -4883,7 +5239,7 @@ const ACP_MAX_LINE_BYTES = 8 * 1024 * 1024; // 8 MiB
 // and every caller work unchanged. Because the raw stream is JSON-RPC (not human
 // output), `stdout` here is the accumulated human-readable transcript text (what
 // we relay), and `stderr` is the child's real stderr (agent diagnostics).
-function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, relayTap = null, stream = false, streamPrefix = '', onStreamOut, onStreamErr, permission = 'yolo', shell = false, onAcpUpdate = null }) {
+function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, relayTap = null, stream = false, streamPrefix = '', onStreamOut, onStreamErr, permission = 'yolo', shell = false, onAcpUpdate = null, abortSignal = null, onSpawn = null }) {
   return new Promise((resolve) => {
     const logger = getLogger();
     const humanChunks = [];
@@ -4897,6 +5253,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
     let idleMon = null;
     let detachSteer = null;
     let child;
+    let onAbort = null;
     let sessionId = null;
     let nextId = 1;
     const pending = new Map();
@@ -4985,6 +5342,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
       if (idleMon) idleMon.stop();
       if (settleTimer) { clearTimeout(settleTimer); settleTimer = null; }
       if (detachSteer) { try { detachSteer(); } catch { /* best effort */ } detachSteer = null; }
+      if (onAbort && abortSignal) { try { abortSignal.removeEventListener('abort', onAbort); } catch { /* best effort */ } onAbort = null; }
       // #137: a turn that completed but produced no mappable session/update gets a
       // synthesised structured floor so the cockpit drill-in isn't empty. Only for
       // a resolved turn (`promptResolved`) — a handshake/spawn failure has nothing
@@ -5253,6 +5611,16 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
       finish({ ok: false, exitCode: null, stdout: '', stderr: '', error: err.message, truncated: false, stderrTruncated: false });
       return;
     }
+    if (onSpawn) { try { onSpawn(child.pid); } catch { /* best effort */ } }
+
+    // #202: abort (stop --force) — finish() reaps the still-alive child via
+    // killTree, so settle as aborted and let it reap the ACP harness group.
+    if (abortSignal) {
+      const abortNow = () => finish({ ok: false, exitCode: null, stdout: humanStdout(), stderr: joinCapped(stderrChunks), error: 'aborted', aborted: true, truncated: humanTruncated, stderrTruncated });
+      if (abortSignal.aborted) { abortNow(); return; }
+      onAbort = abortNow;
+      abortSignal.addEventListener('abort', onAbort);
+    }
 
     timer = timeoutMs && timeoutMs > 0
       ? setTimeout(() => {
@@ -5554,7 +5922,7 @@ function baseAgentEnv(profile, job) {
  * Both paths resolve to the same result contract.
  */
 function runAgentJob(profile, job, opts = {}) {
-  const { timeoutMs, idleTimeoutMs, recoveryWindowMs, envelope, sandbox = 'none', image, runId, secretEnv = {}, passThroughSecretNames = [], cwd, extraEnv = {}, profileEnv = {}, resultFile, stream = false, streamPrefix = '', onStreamOut, onStreamErr, args: commandArgs, terminal = 'pipe', protocol = 'pipe', permission = 'yolo', relaySession = null, ptyFactory, nudgePayload = null, onAcpUpdate = null } = opts;
+  const { timeoutMs, idleTimeoutMs, recoveryWindowMs, envelope, sandbox = 'none', image, runId, secretEnv = {}, passThroughSecretNames = [], cwd, extraEnv = {}, profileEnv = {}, resultFile, stream = false, streamPrefix = '', onStreamOut, onStreamErr, args: commandArgs, terminal = 'pipe', protocol = 'pipe', permission = 'yolo', relaySession = null, ptyFactory, nudgePayload = null, onAcpUpdate = null, abortSignal = null, onSpawn = null } = opts;
   // #110: `protocol`/`permission` drive the ACP executor branch below. The
   // pipe/PTY paths are unchanged, so `protocol === 'pipe'` behaviour is identical.
   // A `nudgePayload` (#678) carries the bespoke "re-emit your result" prompt for a
@@ -5643,6 +6011,8 @@ function runAgentJob(profile, job, opts = {}) {
         onStreamErr,
         permission,
         onAcpUpdate,
+        abortSignal,
+        onSpawn,
       });
     }
 
@@ -5665,6 +6035,8 @@ function runAgentJob(profile, job, opts = {}) {
         stream,
         streamPrefix,
         onStreamOut,
+        abortSignal,
+        onSpawn,
       });
     }
 
@@ -5688,6 +6060,8 @@ function runAgentJob(profile, job, opts = {}) {
       onStreamOut,
       onStreamErr,
       relayTap,
+      abortSignal,
+      onSpawn,
     });
   }
 
@@ -5759,10 +6133,16 @@ function runAgentJob(profile, job, opts = {}) {
     onStreamOut,
     onStreamErr,
     relayTap,
+    // #205: thread the harness-PID callback through the container path too, so a
+    // crashed worker's in-flight job marker records the spawned client PID and
+    // orphan reclamation can tell a possibly-surviving harness from an abandoned
+    // namespace (an empty `harnessPids` otherwise forces indefinite retention).
+    onSpawn,
     onTimeout: (child) => {
       try { spawnSync(engine, ['rm', '-f', containerName], { timeout: 15_000 }); } catch { /* best effort */ }
       try { killTree(child); } catch { /* best effort */ }
     },
+    abortSignal,
   });
 }
 
@@ -7165,15 +7545,50 @@ async function workAgent(req, flags) {
   let reaperTimer = null;
   let runDirTimer = null;
 
+  // --- Cross-process-safe run/result namespace (issue #205) ----------------
+  // Every worker PROCESS gets a fresh incarnation id and its OWN namespace under
+  // agent-runs/worker-<incarnation>/. All of this process's run-*/res- dirs live
+  // there, and its ordinary reaper only ever traverses that namespace, so it can
+  // never delete a sibling worker's active checkout or result channel (the
+  // data-loss defect this issue fixes). The immutable ownership record is
+  // published atomically before any child dir is created.
+  const workerIncarnation = newIncarnationId();
+  const workerPidStart = pidStartToken(process.pid);
+  let pluginVersion = null;
+  try { pluginVersion = JSON.parse(readFileSync(join(pluginDir, 'package.json'), 'utf-8')).version ?? null; } catch { /* best effort */ }
+  let workerNsDir;
+  try {
+    ({ nsDir: workerNsDir } = allocateWorkerNamespace({
+      incarnation: workerIncarnation,
+      worker: profile?.name ?? null,
+      pid: process.pid,
+      pidStart: workerPidStart,
+      version: pluginVersion,
+    }));
+    logger.info(`Worker namespace ${basename(workerNsDir)} (incarnation ${workerIncarnation}, pid ${process.pid}) — run/result dirs are isolated here.`);
+  } catch (err) {
+    logger.error(`Could not allocate the worker run/result namespace under ${agentRunsRoot()}: ${err.message}`);
+    process.exit(1);
+  }
+
   // Run-dir hygiene runs regardless of sandbox: any sandbox=none job that carries
-  // a repository clones a throwaway workspace under the runs root, and a crashed
-  // worker can leave one behind. Bounded to our own directory, age-gated.
+  // a repository clones a throwaway workspace under this worker's namespace, and a
+  // crashed job handler can leave one behind. OWNER-SCOPED: bounded to our own
+  // namespace, age-gated, and skipping in-flight dirs (liveRunDirs) — never a
+  // sibling's namespace or a legacy flat run-*/res- dir. A SEPARATE,
+  // cross-process-safe reclamation sweep handles other incarnations' abandoned namespaces.
   {
-    const initialRuns = reapAgentRunDirs({ maxAgeMs: reapAgeMs, liveRunDirs });
-    if (initialRuns.reaped > 0) logger.info(`Reaped ${initialRuns.reaped} leftover job workspace(s) at startup.`);
+    const initialRuns = reapOwnedNamespace(workerNsDir, { maxAgeMs: reapAgeMs, liveRunDirs, logger, incarnation: workerIncarnation });
+    if (initialRuns.reaped > 0) logger.info(`Reaped ${initialRuns.reaped} leftover job workspace(s) in this worker's namespace at startup.`);
+    if (initialRuns.error) logger.warn(`Startup workspace reap warning: ${initialRuns.error}`);
+    const reclaimStartup = reclaimOrphanNamespaces({ selfIncarnation: workerIncarnation, minAgeMs: reapAgeMs, logger });
+    if (reclaimStartup.reclaimed.length > 0) logger.info(`Reclaimed ${reclaimStartup.reclaimed.length} abandoned worker namespace(s) at startup.`);
     runDirTimer = setInterval(() => {
-      const r = reapAgentRunDirs({ maxAgeMs: reapAgeMs, liveRunDirs });
-      if (r.reaped > 0) logger.info(`Reaper removed ${r.reaped} finished job workspace(s).`);
+      const r = reapOwnedNamespace(workerNsDir, { maxAgeMs: reapAgeMs, liveRunDirs, logger, incarnation: workerIncarnation });
+      if (r.reaped > 0) logger.info(`Reaper removed ${r.reaped} aged, not-in-flight job workspace(s) from this worker's namespace.`);
+      if (r.error) logger.warn(`Workspace reaper warning: ${r.error}`);
+      const rc = reclaimOrphanNamespaces({ selfIncarnation: workerIncarnation, minAgeMs: reapAgeMs, logger });
+      if (rc.reclaimed.length > 0) logger.info(`Reclaimed ${rc.reclaimed.length} abandoned worker namespace(s).`);
     }, reapIntervalMs);
     if (typeof runDirTimer.unref === 'function') runDirTimer.unref();
   }
@@ -7376,7 +7791,7 @@ async function workAgent(req, flags) {
   // dispatch claim/release lifecycle keyed by this worker's instance drives the
   // cockpit's jobKeys — so the recorders no longer poke a per-process channel.
   const recordJobStart = (job, jobType) => {
-    activeJobs.set(String(job.jobKey), { type: jobType, since: Date.now() });
+    activeJobs.set(String(job.jobKey), { type: jobType, since: Date.now(), retries: Number(job.retries) });
     writeActivity();
   };
   const recordJobEnd = (job) => {
@@ -7525,7 +7940,7 @@ async function workAgent(req, flags) {
   // the per-process 1+N reconcile crawl, and the per-job lock extender.
   let settle;
   const runner = {
-    run: async (job) => {
+    run: async (job, abortSignal) => {
       const jobType = job.type;
       recordJobStart(job, jobType);
       try {
@@ -7671,8 +8086,8 @@ async function workAgent(req, flags) {
           const authRef = envelope.repository.authRef;
           repoToken = githubCloneToken({ provider, authRef, secretResolver }); // absent → anonymous clone
           try {
-            mkdirSync(agentRunsRoot(), { recursive: true });
-            runDir = mkdtempSync(join(agentRunsRoot(), 'run-'));
+            mkdirSync(workerNsDir, { recursive: true });
+            runDir = mkdtempSync(join(workerNsDir, 'run-'));
             liveRunDirs.add(runDir);
             provisioned = provisionRepo({ envelope, token: repoToken, runDir, timeoutMs: cloneTimeoutMs });
             if (provisioned.baseFetchError) {
@@ -7721,14 +8136,14 @@ async function workAgent(req, flags) {
           // `cd` to a known absolute path). True confinement is the container
           // increment; a provisioned repository envelope stays the preferred path.
           try {
-            mkdirSync(agentRunsRoot(), { recursive: true });
-            runDir = mkdtempSync(join(agentRunsRoot(), 'run-'));
+            mkdirSync(workerNsDir, { recursive: true });
+            runDir = mkdtempSync(join(workerNsDir, 'run-'));
             liveRunDirs.add(runDir);
             cwd = runDir;
           } catch (err) {
             if (runDir) { try { rmSync(runDir, { recursive: true, force: true }); } catch { /* best effort */ } liveRunDirs.delete(runDir); runDir = null; }
             const retries = Math.max(0, (Number(job.retries) || 1) - 1);
-            const msg = `could not create a temp workspace under the runs root: ${err.message}`;
+            const msg = `could not create a temp workspace under the worker namespace: ${err.message}`;
             logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
             return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
           }
@@ -7752,18 +8167,35 @@ async function workAgent(req, flags) {
         let resultFile = null;
         try {
           try {
-            mkdirSync(agentRunsRoot(), { recursive: true });
-            resultDir = mkdtempSync(join(agentRunsRoot(), 'res-'));
+            mkdirSync(workerNsDir, { recursive: true });
+            resultDir = mkdtempSync(join(workerNsDir, 'res-'));
             resultFile = join(resultDir, 'result.json');
             // Track it so the run-dir reaper skips it while in-flight and reaps it
             // (as a `res-*` dir) if this worker crashes before the cleanup below.
             liveRunDirs.add(resultDir);
           } catch { resultDir = null; resultFile = null; }
 
+          // Record an in-flight job marker in this worker's namespace BEFORE the
+          // harness starts (#205). A later orphan reclamation of a dead owner uses
+          // it to detect a possibly-surviving harness: an empty `harnessPids`
+          // (registration race) forces conservative retention; once `onSpawn`
+          // records the harness PID, reclamation can probe whether it outlived us.
+          writeJobMarker(workerNsDir, { jobKey: job.jobKey, workerPid: process.pid, incarnation: workerIncarnation });
+
           const runOpts = {
             timeoutMs: effectiveHardCapMs,
             idleTimeoutMs: effectiveIdleTimeoutMs,
             recoveryWindowMs: effectiveRecoveryWindowMs,
+            // #205: record the spawned harness's PID (its process-group leader) in
+            // the in-flight job marker so cross-process orphan reclamation can tell
+            // a still-running orphaned harness from a genuinely abandoned namespace.
+            onSpawn: (pid) => recordHarnessPid(workerNsDir, job.jobKey, pid),
+            // #202: the supervisor fiber's interruption AbortSignal (a
+            // `stop --force`/abort). runAgentJob wires it to killTree the harness
+            // process group so an abort CANCELS the work instead of orphaning the
+            // detached agent grandchild to init. Absent (undefined) on the normal
+            // and graceful-drain paths, where the harness runs to completion.
+            abortSignal,
             envelope,
             sandbox,
             image,
@@ -7799,6 +8231,17 @@ async function workAgent(req, flags) {
           };
           result = await runAgentJob(profile, job, runOpts);
 
+          // #202: a `stop --force`/abort killed the harness mid-run. Do NOT settle
+          // here — the worker's force-stop handler yields the job (settle.fail with
+          // its retries preserved) so it is immediately retryable, and the process
+          // is about to exit. Settling here (or nudging/finalizing git) would race
+          // that yield. The finally below still runs (container/run-dir/relay
+          // cleanup); recordJobEnd fires in the outer finally.
+          if (result && result.aborted) {
+            logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop) — harness killed; yielding job for retry.`);
+            return;
+          }
+
           // Gap 2 (#678): a clean run that emitted no machine-readable result gets
           // ONE bounded re-emit nudge in the same workspace, feeding back its own
           // output, before we accept an empty result. Runs before finalizeGit so
@@ -7861,6 +8304,10 @@ async function workAgent(req, flags) {
           }
         } finally {
           if (isContainer) liveRunIds.delete(runId);
+          // #205: clear the in-flight job marker now the harness has stopped — the
+          // owning lifecycle's finally is the authoritative "job finished" signal,
+          // so this namespace no longer holds a surviving harness for this job.
+          removeJobMarker(workerNsDir, job.jobKey);
           if (runDir && !keepRuns) { try { rmSync(runDir, { recursive: true, force: true }); } catch { /* best effort */ } }
           if (runDir) liveRunDirs.delete(runDir);
           // Emit the relay session's `phase:close` lifecycle event and drain its
@@ -8087,23 +8534,29 @@ async function workAgent(req, flags) {
     });
   }
 
-  // Keep the process alive until a stop signal, then interrupt the runtime loop
-  // and tear down visibility. Interrupting the supervisor fiber runs the runtime's
-  // bracketed teardown (release slots, stop the heartbeat + agentic scope).
+  // Keep the process alive until a stop signal. Three stop modes (issue #202):
+  //  - SIGUSR2  → GRACEFUL DRAIN: quiesce the activation loop (lease no new job),
+  //    wait for in-flight jobs to settle normally, then interrupt the idle runtime
+  //    and exit 0. No harness is killed — work completes.
+  //  - SIGTERM / SIGINT → FORCE ABORT: interrupt the runtime (which aborts each
+  //    running job's AbortSignal → runAgentJob killTree's the harness process
+  //    group) and yield each in-flight job (settle.fail, retries preserved) so it
+  //    is immediately retryable, then exit.
+  // A SIGTERM arriving mid-drain ESCALATES the drain to a force abort.
   await new Promise((resolve) => {
-    const stop = async (signal) => {
-      if (draining) return;
-      draining = true;
-      if (!autoMode) unwatchFile(configFile);
-      logger.info(`Received ${signal} — stopping worker...`);
+    let quiescing = false;
+    let aborting = false;
+    let finished = false;
+
+    // Shared teardown: stop timers, deregister presence over the still-live
+    // connection, interrupt the runtime fiber (bracketed teardown: release slots,
+    // stop heartbeats, tear down the agentic scope), then resolve.
+    const teardownAndExit = async (signal) => {
+      if (finished) return;
+      finished = true;
+      if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
       if (reaperTimer) clearInterval(reaperTimer);
       if (runDirTimer) clearInterval(runDirTimer);
-      // Deregister this worker's presence BEFORE interrupting the runtime: emit
-      // the explicit deregister while the multiplexed host connection is still
-      // live, so the worker disappears from the cockpit cleanly rather than
-      // lingering until its heartbeat lapses. Best-effort — teardown must never
-      // hang. Interrupting the fiber then runs the runtime's bracketed teardown
-      // (release slots, stop the heartbeat + tear down the agentic scope).
       if (agenticPlane) {
         try { agenticPlane.deregister(`worker stopped (${signal})`); } catch { /* best effort */ }
       }
@@ -8113,10 +8566,84 @@ async function workAgent(req, flags) {
       } catch (err) {
         logger.warn(`supervisor shutdown error — runtime loop may not have shut down cleanly: ${err?.message || err}`);
       }
+      // #205: tear down this worker's own namespace on a clean exit. All in-flight
+      // jobs' finallys have run by now (markers cleared, run/res dirs removed), so
+      // nothing here is in use. Skipped under --keep-runs so kept workspaces stay
+      // for debugging and age out via cross-process reclamation, exactly like the
+      // pre-fix age-gated behavior.
+      if (workerNsDir && !keepRuns) { try { rmSync(workerNsDir, { recursive: true, force: true }); } catch { /* best effort */ } }
+      resolve();
+    };
+
+    const inFlightCount = () => {
+      try { return SupervisorEffect.runSync(workerRegistry.activeCount); }
+      catch { return activeJobs.size; }
+    };
+
+    const forceAbort = async (signal) => {
+      if (aborting || finished) return;
+      aborting = true;
+      draining = true;
+      if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
+      logger.info(`Received ${signal} — aborting in-flight work and stopping worker...`);
+      // Snapshot in-flight jobs (with their retry budget) BEFORE the interrupt
+      // clears the ownership registry, so we can yield each one afterwards.
+      const inflight = [...activeJobs.entries()].map(([jobKey, info]) => ({ jobKey, retries: info?.retries }));
+      // Interrupt the runtime: this aborts each running job's AbortSignal (the
+      // makeJobRunner seam) so runAgentJob killTree's the harness process group,
+      // and runs dispatch's bracketed teardown (release ownership + slot). The
+      // runner sees the aborted result and, by contract, does NOT settle — we do.
+      if (agenticPlane) { try { agenticPlane.deregister(`worker aborted (${signal})`); } catch { /* best effort */ } }
+      try {
+        await SupervisorEffect.runPromise(SupervisorFiber.interrupt(supervisorFiber));
+      } catch (err) {
+        logger.warn(`supervisor abort error — runtime loop may not have shut down cleanly: ${err?.message || err}`);
+      }
+      // Yield each in-flight job so the broker re-activates it at once (retries
+      // preserved — a force-stop doesn't consume an attempt). Best-effort: a
+      // failed yield just lets the lock lapse (the honest fallback).
+      for (const { jobKey, retries } of inflight) {
+        try {
+          await SupervisorEffect.runPromise(settle.fail(jobKey, {
+            errorMessage: `worker force-stopped (${signal}); job yielded for retry`,
+            retries: Number.isFinite(retries) && retries > 0 ? retries : 1,
+            retryBackOff: 0,
+          }));
+          logger.info(`  yielded job ${jobKey} for immediate retry.`);
+        } catch (err) {
+          logger.warn(`  could not yield job ${jobKey} (${err?.message || err}); its lock will lapse and the broker will reclaim it.`);
+        }
+      }
+      finished = true; // teardown already interrupted the fiber; just resolve.
+      logger.info('Worker stopped.');
       resolve();
     };
-    process.once('SIGINT', () => { stop('SIGINT'); });
-    process.once('SIGTERM', () => { stop('SIGTERM'); });
+
+    const gracefulDrain = async (signal) => {
+      if (quiescing || aborting || finished) return;
+      quiescing = true;
+      draining = true;
+      // Authoritative quiesce: the activation loop leases no new job even if the
+      // --auto reconcile rewrites this worker's job types (registry.quiesce wins).
+      try { SupervisorEffect.runSync(workerRegistry.quiesce()); } catch { /* best effort */ }
+      if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
+      const n0 = inFlightCount();
+      logger.info(`Received ${signal} — draining: polling stopped; waiting on ${n0} in-flight job(s) to finish (send SIGTERM to abort).`);
+      // Poll until every in-flight job has settled and released its slot, then
+      // interrupt the now-idle runtime and exit. Wait INDEFINITELY — a force
+      // abort (SIGTERM) is the only escape hatch.
+      const tick = async () => {
+        if (aborting || finished) return; // escalated to a force abort — let it own exit
+        const n = inFlightCount();
+        if (n <= 0) { await teardownAndExit(signal); return; }
+        setTimeout(tick, 200);
+      };
+      setTimeout(tick, 200);
+    };
+
+    process.once(SUPERVISOR_DRAIN_SIGNAL, () => { gracefulDrain(SUPERVISOR_DRAIN_SIGNAL); });
+    process.once('SIGINT', () => { forceAbort('SIGINT'); });
+    process.once('SIGTERM', () => { forceAbort('SIGTERM'); });
   });
 }
 
@@ -8860,8 +9387,10 @@ function waitForChildExit(child, timeoutMs) {
   return new Promise((resolve) => {
     if (!child || child.exitCode !== null || child.signalCode !== null) return resolve();
     let done = false;
-    const finish = () => { if (done) return; done = true; clearTimeout(t); resolve(); };
-    const t = setTimeout(finish, timeoutMs);
+    // #202: a null/undefined timeout means WAIT INDEFINITELY (graceful drain) —
+    // no timer is armed, so we only resolve when the child actually exits.
+    const t = timeoutMs == null ? null : setTimeout(() => finish(), timeoutMs);
+    function finish() { if (done) return; done = true; if (t) clearTimeout(t); resolve(); }
     child.once('exit', finish);
   });
 }
@@ -8939,7 +9468,21 @@ async function runSupervisorDaemon() {
 
   const workers = new Map();
   const attachClients = new Set();
+  // #202: sockets that issued a `stop`/drain and are waiting for the daemon to
+  // finish. They get a terminal `stopped` (final) frame when shutdown completes,
+  // so the streaming `stop` client (which waits indefinitely) sees a clean
+  // end-of-response instead of a bare socket close. A one-shot `supervisorRequest`
+  // has a fixed 15s `SUPERVISOR_RESPONSE_TIMEOUT_MS` deadline, so it only observes
+  // that frame when the drain completes within it; a longer drain times out
+  // client-side while the daemon keeps draining in the background.
+  const stopClients = new Set();
   let shuttingDown = false;
+  // #202: a graceful-drain shutdown is in progress (SIGUSR2 sent to workers,
+  // awaiting them to finish in-flight jobs and exit). `forcing` records that a
+  // `stop --force` has escalated that drain to a hard abort. Both gate the
+  // restart-on-exit path and let a second `stop --force` escalate a live drain.
+  let draining = false;
+  let forcing = false;
   // Live-view monitor: tracks the last-broadcast fleet signature so we push a
   // refreshed status to attached consoles only on real change (see below).
   let monitorTimer = null;
@@ -9070,7 +9613,23 @@ async function runSupervisorDaemon() {
       try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
       const ranMs = Date.now() - (w.spawnedAt || Date.now());
       if (ranMs >= SUPERVISOR_HEALTHY_UPTIME_MS) w.restarts = 0;
-      if (w.stopping || shuttingDown || !workers.has(w.id)) { persist(); return; }
+      if (w.stopping || shuttingDown || !workers.has(w.id)) {
+        // #202 drain-remove: a worker flagged for removal has now drained and
+        // exited — delete it and announce its removal (mirrors the synchronous
+        // force-remove path). Do this before persist() so the state reflects it.
+        if (w.removeOnExit && workers.get(w.id) === w) {
+          workers.delete(w.id);
+          try { rmSync(w.activityFile || supervisorWorkerActivityFile(w.id), { force: true }); } catch { /* best effort */ }
+          dlog(`worker '${w.id}' removed (drained)`);
+          broadcast({ type: 'event', event: 'worker-remove', id: w.id });
+        }
+        persist();
+        // #202: during a graceful drain, push a fresh status so an attached
+        // `stop` client sees the in-flight count shrink as each worker finishes
+        // and exits — even when the periodic monitor is disabled.
+        if (draining) { try { broadcast(statusFrame(false)); } catch { /* best effort */ } }
+        return;
+      }
       const delay = supervisorBackoffMs(w.restarts);
       w.restarts += 1;
       dlog(`worker '${w.id}' down (${reason}); restarting in ${delay}ms (restart #${w.restarts})`);
@@ -9109,23 +9668,55 @@ async function runSupervisorDaemon() {
     return w;
   };
 
-  const stopWorker = async (id) => {
-    const w = workers.get(id);
-    if (!w) return false;
-    w.stopping = true;
-    if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
+  const forceStopWorker = async (w) => {
     const pid = w.pid;
     if (w.child && pid) {
       try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
       await waitForChildExit(w.child, STOP_GRACE_MS);
       if (isPidAlive(pid)) { try { process.kill(pid, 'SIGKILL'); } catch { /* ignore */ } }
     }
+  };
+
+  const stopWorker = async (id, { force = true } = {}) => {
+    const w = workers.get(id);
+    if (!w) return false;
+    w.stopping = true;
+    if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
+    const pid = w.pid;
+    if (!force) {
+      // #202 graceful drain: send SIGUSR2 so the child quiesces its activation
+      // loop, lets in-flight jobs finish, and exits on its own. Wait INDEFINITELY
+      // — a `stop --force` escalation (forceStopWorker) is the only way to cut a
+      // stuck drain short, and it resolves this same wait when the child dies.
+      if (w.child && pid) {
+        try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ }
+        await waitForChildExit(w.child, null);
+      }
+      return true;
+    }
+    await forceStopWorker(w);
     return true;
   };
 
-  const removeWorker = async (id) => {
-    if (!workers.has(id)) return false;
-    await stopWorker(id);
+  const removeWorker = async (id, { force = true } = {}) => {
+    const w = workers.get(id);
+    if (!w) return false;
+    if (!force) {
+      // #202 drain-remove (used by `workforce stop`): quiesce this worker and let
+      // it finish its in-flight jobs, then let the death handler delete it once
+      // it exits. Return immediately so the control loop keeps serving (the
+      // client polls status to watch the worker disappear) rather than blocking
+      // the request queue for the whole — possibly long — drain.
+      w.stopping = true;
+      w.removeOnExit = true;
+      if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
+      const pid = w.pid;
+      if (w.child && pid) { try { process.kill(pid, SUPERVISOR_DRAIN_SIGNAL); } catch { /* already gone */ } }
+      else { workers.delete(id); try { rmSync(supervisorWorkerActivityFile(id), { force: true }); } catch { /* best effort */ } broadcast({ type: 'event', event: 'worker-remove', id }); persist(); }
+      dlog(`worker '${id}' draining for removal`);
+      return true;
+    }
+    await stopWorker(id, { force: true });
     workers.delete(id);
     try { rmSync(supervisorWorkerActivityFile(id), { force: true }); } catch { /* best effort */ }
     dlog(`worker '${id}' removed`);
@@ -9166,16 +9757,45 @@ async function runSupervisorDaemon() {
     ...(final ? { final: true } : {}),
   });
 
-  const shutdown = async (signal) => {
-    if (shuttingDown) return;
+  const shutdown = async (signal, { force = true } = {}) => {
+    if (shuttingDown) {
+      // A shutdown is already running. A `stop --force` arriving mid-DRAIN
+      // escalates it: hard-stop every worker still finishing its jobs so the
+      // operator isn't stuck waiting. Idempotent — only the first force escalates.
+      if (force && draining && !forcing) {
+        forcing = true;
+        dlog('stop --force received during drain — escalating to hard abort');
+        broadcast({ type: 'event', event: 'draining-escalated' });
+        await Promise.all([...workers.values()].map((w) => forceStopWorker(w)));
+      }
+      return;
+    }
     shuttingDown = true;
+    draining = !force;
     // Let any in-flight mutation finish before we snapshot the worker set, so
     // an add/restart racing the shutdown can't leave an orphaned child behind.
     try { await opQueue; } catch { /* mutation already logged */ }
+    dlog(`received ${signal || 'stop'} — ${force ? 'stopping' : 'draining'} ${workers.size} worker(s)`);
+    if (!force) broadcast({ type: 'event', event: 'draining', workers: [...workers.values()].map(workerPublic) });
+    await Promise.all([...workers.keys()].map((id) => stopWorker(id, { force })));
     if (monitorTimer) { try { clearInterval(monitorTimer); } catch { /* ignore */ } monitorTimer = null; }
-    dlog(`received ${signal || 'stop'} — stopping ${workers.size} worker(s)`);
-    await Promise.all([...workers.keys()].map((id) => stopWorker(id)));
     broadcast({ type: 'event', event: 'daemon-stop' });
+    // Terminal frame for every waiting `stop` client (streaming or one-shot).
+    // We must FLUSH these before exiting: `process.exit()` does NOT drain pending
+    // socket I/O, so an immediate exit can drop the final frame and turn the
+    // clean end-of-response the streaming stop client waits for into a bare
+    // socket close. `end(frame, cb)` writes the frame then sends FIN, and its
+    // callback fires once the bytes are handed off; await them all (with a short
+    // timeout guard so a wedged/slow client can't block the exit indefinitely).
+    await Promise.all([...stopClients].map((s) => new Promise((resolve) => {
+      let done = false;
+      const finish = () => { if (!done) { done = true; resolve(); } };
+      try {
+        const timer = setTimeout(finish, 2000);
+        if (typeof timer.unref === 'function') timer.unref();
+        s.end(encodeFrame({ ok: true, type: 'stopped', final: true }), () => { clearTimeout(timer); finish(); });
+      } catch { finish(); /* client gone */ }
+    })));
     try { server.close(); } catch { /* ignore */ }
     if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
     clearSupervisorState();
@@ -9213,12 +9833,17 @@ async function runSupervisorDaemon() {
         }
         case 'remove': {
           if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
+          // #202: `req.force === false` drains each worker (finish in-flight jobs
+          // then exit); the default remains a fast force-stop (used by reconcile
+          // and interactive remove). A drain-remove returns immediately and the
+          // worker disappears from status once it has drained.
+          const removeForce = req.force !== false;
           const removed = await serializeOp(async () => {
             const ids = resolveTargets(req.target);
-            for (const id of ids) await removeWorker(id);
+            for (const id of ids) await removeWorker(id, { force: removeForce });
             return ids;
           });
-          sock.write(encodeFrame({ ok: true, type: 'removed', removed, final: true }));
+          sock.write(encodeFrame({ ok: true, type: 'removed', removed, draining: !removeForce, final: true }));
           break;
         }
         case 'restart': {
@@ -9235,10 +9860,25 @@ async function runSupervisorDaemon() {
           attachClients.add(sock);
           sock.write(encodeFrame(statusFrame(false)));
           break;
-        case 'stop':
-          sock.write(encodeFrame({ ok: true, type: 'stopping', final: true }));
-          setTimeout(() => shutdown('stop'), 50);
+        case 'stop': {
+          // #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
+          // jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
+          // yield jobs) and can also ESCALATE a drain already in progress.
+          const force = !!req.force;
+          // Register this client as an attach consumer so it streams drain
+          // progress (shrinking in-flight counts), and as a stop client so it
+          // gets a terminal `stopped` frame when the daemon has finished.
+          attachClients.add(sock);
+          stopClients.add(sock);
+          sock.write(encodeFrame(force
+            ? { ok: true, type: 'stopping', force: true }
+            : { ok: true, type: 'draining' }));
+          sock.write(encodeFrame(statusFrame(false)));
+          // Kick the shutdown asynchronously; don't await it here so the control
+          // loop keeps serving (streaming status, accepting a force escalation).
+          setTimeout(() => { shutdown('stop', { force }); }, 20);
           break;
+        }
         default:
           sock.write(encodeFrame({ ok: false, error: `unknown op "${op}"`, final: true }));
       }
@@ -9276,8 +9916,8 @@ async function runSupervisorDaemon() {
         queue = queue.then(() => handleRequest(req, sock)).catch((err) => dlog(`request error: ${err?.message || err}`));
       }
     });
-    sock.on('close', () => attachClients.delete(sock));
-    sock.on('error', () => attachClients.delete(sock));
+    sock.on('close', () => { attachClients.delete(sock); stopClients.delete(sock); });
+    sock.on('error', () => { attachClients.delete(sock); stopClients.delete(sock); });
   });
 
   // Create the control socket owner-only from the start. The socket file lives
@@ -9622,7 +10262,29 @@ async function supervisorRestartCmd(req) {
   else { logger.error(res.error); process.exit(1); }
 }
 
-async function supervisorStopCmd() {
+/**
+ * Count the in-flight jobs across a fleet snapshot (array of
+ * `summarizeSupervisorWorker` results) — the number an operator is waiting on
+ * when draining.
+ */
+function countSupervisorInFlight(workers) {
+  let n = 0;
+  for (const w of workers || []) {
+    if (w && w.activity && Array.isArray(w.activity.jobs)) n += w.activity.jobs.length;
+  }
+  return n;
+}
+
+/**
+ * Stop the supervisor. By default (issue #202) this GRACEFULLY DRAINS: the
+ * daemon quiesces its workers (they stop leasing new jobs, finish the ones in
+ * flight, and exit), streaming the shrinking in-flight count to this client the
+ * whole time. Ctrl-C DETACHES — the daemon keeps draining in the background.
+ * `force` hard-aborts instead: each worker's harness is killed and its job is
+ * yielded for immediate retry. A `force` request also escalates a drain that is
+ * already in progress.
+ */
+async function supervisorStopCmd(force = false) {
   const logger = getLogger();
   const running = await liveSupervisor();
   if (!running) {
@@ -9630,35 +10292,100 @@ async function supervisorStopCmd() {
     else logger.warn('Supervisor is not running — nothing to stop.');
     return;
   }
-  try {
-    await supervisorRequest({ op: 'stop' });
-  } catch {
-    // Socket unreachable — fall back to signalling the daemon pid directly.
-    try { process.kill(running.pid, 'SIGTERM'); } catch { /* already gone */ }
-  }
-  // Gate the wait loop and the SIGKILL fallback on the daemon pid we captured,
-  // not on runningSupervisor()/the state file: the daemon clears its state file
-  // as part of shutting down (and liveSupervisor()/external cleanup can remove
-  // it too), so a state-file check can report "gone" while the process is still
-  // alive — which would break the loop early and skip the SIGKILL fallback,
-  // leaving a wedged daemon and its worker process group running.
-  const deadline = Date.now() + STOP_GRACE_MS + 2_000;
-  while (Date.now() < deadline) {
-    if (!isPidAlive(running.pid)) break;
-    await new Promise((r) => setTimeout(r, 150));
+  const socketPath = running.socket || getSupervisorSocketPath();
+  const daemonPid = running.pid;
+
+  // Stream drain/stop progress. Resolves with the outcome that ended the stream.
+  const outcome = await new Promise((resolve) => {
+    let sock = null;
+    let buf = '';
+    let done = false;
+    let lastCount = null;
+    let onSigint = null;
+    const cleanup = () => {
+      if (onSigint) { try { process.removeListener('SIGINT', onSigint); } catch { /* ignore */ } }
+      try { if (sock) sock.end(); } catch { /* ignore */ }
+    };
+    const finish = (result) => { if (done) return; done = true; cleanup(); resolve(result); };
+
+    supervisorConnect(socketPath).then((s) => {
+      sock = s;
+      sock.setEncoding('utf8');
+      // Ctrl-C detaches the client only — the daemon keeps draining. (Force
+      // stops don't wait on the operator, so a Ctrl-C there just stops watching.)
+      onSigint = () => {
+        if (force) {
+          logger.info('Detached — supervisor is aborting in the background.');
+        } else {
+          logger.info('Detached — supervisor keeps draining in the background. Rerun `nano supervisor stop` to watch again, or `nano supervisor stop --force` to abort in-flight work.');
+        }
+        finish('detached');
+      };
+      process.on('SIGINT', onSigint);
+
+      sock.on('data', (chunk) => {
+        buf += chunk;
+        const { frames, rest } = decodeFrames(buf);
+        buf = rest;
+        for (const frame of frames) {
+          if (frame && frame.type === 'stopping') {
+            logger.info('Aborting in-flight work — killing harnesses and yielding jobs for retry...');
+          } else if (frame && frame.event === 'draining-escalated') {
+            logger.info('Escalating to --force — aborting in-flight work...');
+          } else if (frame && (frame.type === 'status' || frame.event === 'draining')) {
+            const n = countSupervisorInFlight(frame.workers);
+            if (!force && n !== lastCount) {
+              lastCount = n;
+              if (n > 0) logger.info(`Draining — waiting on ${n} in-flight job(s) to finish. Press Ctrl-C to detach, or rerun with --force to abort.`);
+              else logger.info('No jobs in flight — stopping.');
+            }
+          } else if (frame && frame.event === 'daemon-stop') {
+            finish('stopped');
+          } else if (frame && (frame.type === 'stopped' || frame.final)) {
+            finish('stopped');
+          }
+        }
+      });
+      sock.on('error', () => finish('closed'));
+      sock.on('close', () => finish('closed'));
+      sock.write(encodeFrame({ op: 'stop', force: !!force }));
+    }).catch(() => finish('unreachable'));
+  });
+
+  if (outcome === 'detached') return;
+
+  if (outcome === 'unreachable') {
+    if (force) {
+      // Socket unreachable — fall back to signalling the daemon pid directly.
+      try { process.kill(daemonPid, 'SIGTERM'); } catch { /* already gone */ }
+    } else {
+      logger.warn('Could not reach the supervisor control socket to drain it. Rerun with --force to abort, or stop it manually.');
+      return;
+    }
   }
-  if (isPidAlive(running.pid)) {
-    logger.warn(`Supervisor (pid ${running.pid}) did not stop gracefully — sending SIGKILL.`);
-    // The daemon is spawned detached (a process-group leader) and its workers
-    // are children in that group, so SIGKILL the whole group to avoid orphaning
-    // `nano work` processes. Fall back to the bare pid (e.g. on Windows, or if
-    // the daemon isn't a group leader).
-    let killedGroup = false;
-    if (osPlatform() !== 'win32') {
-      try { process.kill(-running.pid, 'SIGKILL'); killedGroup = true; } catch { /* fall back below */ }
+
+  // Wait for the daemon process itself to exit. A drain waits INDEFINITELY (the
+  // operator opted to wait); a force stop applies the grace window + a SIGKILL
+  // backstop so a wedged daemon/worker group can't linger.
+  if (force) {
+    const deadline = Date.now() + STOP_GRACE_MS + 2_000;
+    while (Date.now() < deadline) {
+      if (!isPidAlive(daemonPid)) break;
+      await new Promise((r) => setTimeout(r, 150));
+    }
+    if (isPidAlive(daemonPid)) {
+      logger.warn(`Supervisor (pid ${daemonPid}) did not stop gracefully — sending SIGKILL.`);
+      let killedGroup = false;
+      if (osPlatform() !== 'win32') {
+        try { process.kill(-daemonPid, 'SIGKILL'); killedGroup = true; } catch { /* fall back below */ }
+      }
+      if (!killedGroup) { try { process.kill(daemonPid, 'SIGKILL'); } catch { /* ignore */ } }
+      clearSupervisorState();
+    }
+  } else {
+    while (isPidAlive(daemonPid)) {
+      await new Promise((r) => setTimeout(r, 150));
     }
-    if (!killedGroup) { try { process.kill(running.pid, 'SIGKILL'); } catch { /* ignore */ } }
-    clearSupervisorState();
   }
   logger.info('Supervisor stopped.');
 }
@@ -10335,7 +11062,7 @@ async function supervisorCommand(req, flags) {
       await supervisorRestartCmd(req);
       return;
     case 'stop':
-      await supervisorStopCmd();
+      await supervisorStopCmd(coerceBool(flags?.force, false));
       return;
     case 'logs':
     case 'log':
@@ -11234,14 +11961,38 @@ async function workforceStopCmd(req, flags, manifestName) {
     })
     .map((w) => w.id);
   let hadError = false;
+  const force = coerceBool(flags?.force, false);
   if (owned.length === 0) {
     logger.info(`No workers from workforce "${manifestName}" are running.`);
-  } else {
+  } else if (force) {
     for (const id of owned) {
-      const res = await supervisorRequest({ op: 'remove', target: id });
-      if (res && res.ok) logger.info(`Removed worker "${id}".`);
+      const res = await supervisorRequest({ op: 'remove', target: id, force: true });
+      if (res && res.ok) logger.info(`Removed worker "${id}" (aborted in-flight work).`);
       else { logger.error(`Could not remove "${id}": ${(res && res.error) || 'unknown error'}`); hadError = true; }
     }
+  } else {
+    // #202 graceful drain: ask the daemon to quiesce each owned worker (finish
+    // in-flight jobs, then exit) and poll status until they've all drained away.
+    for (const id of owned) {
+      const res = await supervisorRequest({ op: 'remove', target: id, force: false });
+      if (res && res.ok) logger.info(`Draining worker "${id}"...`);
+      else { logger.error(`Could not drain "${id}": ${(res && res.error) || 'unknown error'}`); hadError = true; }
+    }
+    const ownedSet = new Set(owned);
+    let lastInFlight = null;
+    for (;;) {
+      const snap = await fetchSupervisorWorkers();
+      if (!snap.running || !snap.reachable) break;
+      const remaining = (snap.workers || []).filter((w) => w && ownedSet.has(w.id));
+      if (remaining.length === 0) break;
+      const inFlight = countSupervisorInFlight(remaining);
+      if (inFlight !== lastInFlight) {
+        lastInFlight = inFlight;
+        logger.info(`Draining "${manifestName}" — ${remaining.length} worker(s) still finishing ${inFlight} in-flight job(s). Rerun with --force to abort.`);
+      }
+      await new Promise((r) => setTimeout(r, 300));
+    }
+    if (lastInFlight !== null) logger.info(`Workforce "${manifestName}" drained.`);
   }
   // If no supervised workers remain, stop the daemon too — but only when the
   // status socket actually answered. A `{ workers: [] }` from an *unreachable*
@@ -11257,7 +12008,7 @@ async function workforceStopCmd(req, flags, manifestName) {
     logger.warn('Supervisor status socket became unreachable; leaving the daemon running.');
   } else if (remaining.length === 0) {
     logger.info('No supervised workers remain — stopping the supervisor daemon.');
-    await supervisorStopCmd();
+    await supervisorStopCmd(force);
   } else {
     logger.info(`${remaining.length} other supervised worker(s) remain; leaving the daemon running.`);
   }
@@ -13054,6 +13805,20 @@ export {
   isPlaceholderEmail,
   postAgentAttribution,
   reapAgentRunDirs,
+  reapChildRunDirs,
+  reapOwnedNamespace,
+  reclaimOrphanNamespaces,
+  allocateWorkerNamespace,
+  workerNamespaceDir,
+  readOwnerRecord,
+  incarnationLiveness,
+  pidStartToken,
+  newIncarnationId,
+  writeJobMarker,
+  recordHarnessPid,
+  removeJobMarker,
+  readJobMarkers,
+  namespaceHasLiveHarness,
   authUrl,
   githubCloneToken,
   ghAuthTokenFromCli,
@@ -13281,7 +14046,7 @@ export const commands = {
       console: { type: 'string', description: 'start: runtime console profile off|observe|studio (NANOBPMN_CONSOLE; default studio)' },
       follow: { type: 'boolean', description: 'logs: stream output (tail -F)', short: 'f' },
       purge: { type: 'boolean', description: 'stop/restart: also delete per-node engine data' },
-      force: { type: 'boolean', description: 'start: stop any existing cluster first' },
+      force: { type: 'boolean', description: 'start: stop any existing cluster first; supervisor/workforce stop: abort in-flight work (kill harness, yield jobs for retry) instead of the default graceful drain' },
       workspace: { type: 'boolean', description: 'clean: also delete the workspace (models + workers)' },
       check: { type: 'boolean', description: 'update: report whether a new release is available (with the changelog since the installed version); do not install' },
       binary: { type: 'string', description: 'Path to the nanobpmn server binary' },