create-walle 0.10.1 → 0.10.2

package/template/coding-task-manager/lib/host-handoff.js

@@ -33,6 +33,22 @@ function modelProcessGenerationMetadata(value) {
   return validModelProcessGeneration(value) ? { modelProcessGeneration: value } : {};
 }
 const MODEL_STORAGE_ROOT_KEYS = Object.freeze(['HOME', 'CODEX_HOME', 'CTM_CODEX_SESSIONS_DIR', 'CLAUDE_CONFIG_DIR', 'GEMINI_CLI_HOME', 'OPENCODE_DATA_DIR', 'XDG_DATA_HOME']);
+// The keys a lead-model switch re-projects into the launch context (lib/lead-model-restore-reconciliation.js
+// projectDeferredRestoreLaunch: HOME / CODEX_HOME from the launch context, TMPDIR alongside them). They are
+// PROJECTED launch context, not per-session private overrides: a spawn still receives them, but their presence
+// must never turn an otherwise empty override set into a private environment that only a surviving owner can
+// reproduce. Anything else in the env is private and is captured by value.
+const PROJECTED_LAUNCH_CONTEXT_KEYS = Object.freeze(['HOME', 'CODEX_HOME', 'TMPDIR']);
+// Returns a NEW object holding only the non-projected entries of `env`; never mutates its input and maps a
+// non-object to `{}`.
+function privateOverridesFromEnv(env) {
+  if (!env || typeof env !== 'object' || Array.isArray(env)) return {};
+  const out = {};
+  for (const [key, value] of Object.entries(env)) {
+    if (!PROJECTED_LAUNCH_CONTEXT_KEYS.includes(key)) out[key] = value;
+  }
+  return out;
+}
 function modelStorageRootsFromEnv(env) {
   const roots = {};
   for (const key of MODEL_STORAGE_ROOT_KEYS) {
@@ -111,6 +127,10 @@ function isAlive(pid) {
 // listSessions() -> [{ sessionId, ptyProcess, ptyClosed, cols, rows, seq, cmd, cwd, name, outputEpoch,
 //   workerDisplayContext, suppressPtyHeadlessUntilInput, mirrorFeed }]; serialize(sessionId) -> {data,cols,rows}|null.
 const REPAINT_NUDGE_MS = 80;
+// D-1: the old host's painted-seal await is bounded. A seal runs on its session's own op chain; a wedged chain
+// (a dead libuv pool) never comes back, and no deadline is long enough to wait it out. Losing the race costs a
+// restart break in that session's painted rows (an unsealed stretch cannot be joined), never the whole handoff.
+const HANDOFF_SEAL_TIMEOUT_MS = 1000;
 
 function createHandoff({ hostId, workerPath, controlSocketPath, listSessions, serialize, mirrorStatus, headless, send, log = () => {}, setTimer = setTimeout, spawnSuccessor = spawnSuccessorHost }) {
   let active = null;
@@ -140,6 +160,26 @@ function createHandoff({ hostId, workerPath, controlSocketPath, listSessions, se
     (reply || send)({ type: 'handoff-aborted', handoffId: cur.handoffId, reason });
   }
 
+  // One session's painted-seal, raced against HANDOFF_SEAL_TIMEOUT_MS: resolves when the seal completes (its
+  // timer then cleared) or when the bound expires, in which case the handoff proceeds and the loss is reported.
+  // A rejecting seal still rejects (the begin() catch aborts the handoff, unchanged).
+  function raceSeal(sessionId) {
+    return new Promise((resolve, reject) => {
+      let settled = false;
+      const timer = setTimer(() => {
+        if (settled) return;
+        settled = true;
+        diagnostic(sessionId, 'host-handoff-seal-timeout', { timeoutMs: HANDOFF_SEAL_TIMEOUT_MS, oldHostId: hostId });
+        resolve();
+      }, HANDOFF_SEAL_TIMEOUT_MS);
+      timer?.unref?.();
+      Promise.resolve(headless.handleMessage({ type: 'painted-seal', sessionId })).then(
+        () => { if (settled) return; settled = true; clearTimeout(timer); resolve(); },
+        (err) => { if (settled) return; settled = true; clearTimeout(timer); reject(err); },
+      );
+    });
+  }
+
   async function begin(msg, reply) {
     const handoffId = String(msg.handoffId || '');
     if (active) { reply({ type: 'handoff-aborted', handoffId, reason: 'already-in-progress' }); return; }
@@ -153,11 +193,11 @@ function createHandoff({ hostId, workerPath, controlSocketPath, listSessions, se
         s.ptyProcess.pause();
         cur.paused.push(s);
       }
-      const seeds = new Map();
-      for (const s of cur.paused) {
+      // D-1: every seed is issued concurrently -- each `serialize` keeps its own 2 000 ms cap, so N sessions cost
+      // one cap, not N. The mirrorFeed flush still lands immediately before its own session's serialize.
+      const seeds = new Map(await Promise.all(cur.paused.map(async (s) => {
         s.mirrorFeed?.flush?.({ force: true });
         const snap = await serialize(s.sessionId);
-        seeds.set(s.sessionId, snap);
         // Probe (observe only): what the seed carries vs what the mirror believed at the same instant.
         const seedData = String((snap && snap.data) || '');
         diagnostic(s.sessionId, 'host-handoff-mirror-seed', {
@@ -168,18 +208,16 @@ function createHandoff({ hostId, workerPath, controlSocketPath, listSessions, se
           seedBytes: Buffer.byteLength(seedData),
           oldHostId: hostId,
         });
-      }
+        return [s.sessionId, snap];
+      })));
       if (active !== cur) return; // aborted (deadline) while serializing
       // This host exits without its headless close(): seal each painted-row stretch (with its last screen) and flush the
       // log now, before the successor exists to load it -- an unsealed stretch cannot be joined (a restart break).
-      // Each id is listed BEFORE its seal is queued, so an abort that lands mid-seal still reopens it (the per-session op
-      // queue runs that reopen after the seal).
-      cur.sealed = [];
-      for (const s of cur.paused) {
-        if (active !== cur) return; // aborted (deadline) while sealing
-        cur.sealed.push(s.sessionId);
-        await headless.handleMessage({ type: 'painted-seal', sessionId: s.sessionId });
-      }
+      // D-1: every id is listed BEFORE any seal is queued (so an abort that lands mid-seal still reopens all of them,
+      // the per-session op queue running each reopen after its seal), and every seal is raced against
+      // HANDOFF_SEAL_TIMEOUT_MS, so one wedged op chain cannot hold the handoff past its bound.
+      cur.sealed = cur.paused.map((s) => s.sessionId);
+      await Promise.all(cur.paused.map((s) => raceSeal(s.sessionId)));
       if (active !== cur) return; // aborted (deadline) while sealing
       const running = cur.paused.filter((s) => !s.ptyClosed);
       const entries = running.map((s, i) => {
@@ -306,4 +344,4 @@ function createHandoff({ hostId, workerPath, controlSocketPath, listSessions, se
   return { intercept, restore, resumeRestored, isActive: () => !!active };
 }
 
-module.exports = { handoffManifestFromArgv, writeHandoffManifest, readHandoffManifest, spawnSuccessorHost, createHandoff, validModelProcessGeneration, modelProcessGenerationFromEnv, modelProcessGenerationMetadata, modelStorageRootsFromEnv, modelStorageRootsMetadata };
+module.exports = { HANDOFF_SEAL_TIMEOUT_MS, handoffManifestFromArgv, writeHandoffManifest, readHandoffManifest, spawnSuccessorHost, createHandoff, validModelProcessGeneration, modelProcessGenerationFromEnv, modelProcessGenerationMetadata, modelStorageRootsFromEnv, modelStorageRootsMetadata, PROJECTED_LAUNCH_CONTEXT_KEYS, privateOverridesFromEnv };

package/template/coding-task-manager/lib/host-health-ledger.js

@@ -11,6 +11,12 @@
 // module's), no further RPCs go to that host; markCapped freezes the verdict
 // until forgetHost. Pure/in-memory except the injectable readStall hook.
 //
+// WP-3 (session-stops-mid-job-2026-10-24.html, D-10..D-12): a hard serialize
+// miss on a host that sent ANY frame after that request was issued (the
+// injectable lastFrameAt hook) is readiness, not liveness -- it latches
+// 'degraded' (reported once, cleared by a reply, never a kill). Only a silent
+// host still reaches 'unresponsive'; the stall-file shortcut is unchanged.
+//
 // No deps beyond node:fs/node:path for the default readStall -- kept free of
 // process globals (server.js binds dataDir into the readStall it injects).
 
@@ -31,23 +37,77 @@ function readStallFromDataDir(dataDir, hostId) {
   }
 }
 
+// WP-3 (stale-pty-host-evacuation design doc, D-2, §6): the evacuation deadline
+// for a continuously stalled libuv threadpool. Value provenance (§4 D-2): every
+// recovered stall in the log evidence lasted <= 1.7 s, all 6 unrecovered ones
+// longer; 30 s equals the existing hard-miss floor below (line 69).
+const THREADPOOL_EVACUATE_MS = 30000;
+
 function createHostHealthLedger({
   missThreshold = 2,
   stallThresholdMs = 30000,
   readStall = null,
+  // WP-3 (D-10): lastFrameAt(hostId) -> number, the manager's per-host frame
+  // clock ("the host sent any frame", any transport). Optional: without it the
+  // ledger behaves exactly as it did before WP-3.
+  lastFrameAt = null,
   now = Date.now,
+  // WP-3 (D-2): the stall-evacuation episode's clock/timers/action seam. All
+  // three default to the real thing, so every existing caller is unaffected.
+  setTimer = setTimeout,
+  clearTimer = clearTimeout,
+  onEvacuate = null,
+  log = console.warn,
 } = {}) {
-  const hosts = new Map(); // hostId -> { misses, lastMissAt, lastReplyAt, verdict }
+  // hostId -> { misses, lastMissAt, lastReplyAt, verdict, degraded, ...threadpool episode }
+  const hosts = new Map();
 
   function entryFor(hostId) {
     let entry = hosts.get(hostId);
     if (!entry) {
-      entry = { misses: 0, lastMissAt: 0, lastReplyAt: 0, verdict: 'ok', cappedUntil: 0 };
+      entry = {
+        misses: 0, lastMissAt: 0, lastReplyAt: 0, verdict: 'ok', cappedUntil: 0, degraded: false,
+        // D-2 (WP-3) per-host stall episode: 0 = no episode in progress.
+        threadpoolStalledSince: 0, threadpoolOldestPendingMs: 0, evacuateFired: false, threadpoolTimer: null,
+      };
       hosts.set(hostId, entry);
     }
     return entry;
   }
 
+  function _cancelThreadpoolTimer(entry) {
+    if (!entry || !entry.threadpoolTimer) return;
+    clearTimer(entry.threadpoolTimer);
+    entry.threadpoolTimer = null;
+  }
+
+  function _threadpoolResult(entry, id) {
+    return {
+      action: 'evacuate',
+      hostId: id,
+      oldestPendingMs: Number(entry.threadpoolOldestPendingMs) || 0,
+      stalledForMs: now() - entry.threadpoolStalledSince,
+    };
+  }
+
+  // Fires an episode exactly once: marks it fired BEFORE calling out, so a
+  // re-entrant recordThreadpool from onEvacuate cannot double-fire.
+  function _markFired(entry, id) {
+    entry.evacuateFired = true;
+    entry.threadpoolTimer = null;
+    return _threadpoolResult(entry, id);
+  }
+
+  // The armed-timer path is the ONLY one that calls out: an immediate verdict is
+  // returned to the caller of recordThreadpool instead (§6 -- one action per
+  // episode, never both).
+  function _fireEvacuate(entry, id) {
+    const result = _markFired(entry, id);
+    if (typeof onEvacuate !== 'function') return; // no action bound -- the episode is still marked fired
+    try { onEvacuate(result); }
+    catch (err) { log(`[host-health] host=${id} onEvacuate threw: ${err?.message || err}`); } // REPORT: a timer callback must never throw into the event loop
+  }
+
   // Final-review fix C4: 'unresponsive-capped' used to be permanent -- markCapped
   // set it forever and recordMiss short-circuited on any non-'ok' verdict, so
   // EVERY snapshot for EVERY session on that host was answered timedOut for the
@@ -63,7 +123,7 @@ function createHostHealthLedger({
     entry.cappedUntil = 0;
   }
 
-  function recordMiss(hostId, { sessionId, kind, deadlineMs } = {}) {
+  function recordMiss(hostId, { sessionId, kind, deadlineMs, issuedAt } = {}) {
     const id = String(hostId || '');
     if (!id) return null;
     // Soft (short-deadline, or explicitly non-'hard') misses never count -- D-2
@@ -74,14 +134,37 @@ function createHostHealthLedger({
     // Once a host already carries a verdict, further misses are idempotent --
     // no state change, verdict() keeps reporting it until forgetHost.
     if (entry.verdict !== 'ok') return null;
-    entry.misses += 1;
-    entry.lastMissAt = now();
     let stall = null;
     if (typeof readStall === 'function') {
       try { stall = readStall(id); } catch { stall = null; }
     }
-    const stalled = stall && Number(stall.durationMs) >= stallThresholdMs;
-    if (stalled || entry.misses >= missThreshold) {
+    // D-2(b), unchanged: the host's own stall report is proof it is wedged, and
+    // it still wins over any frame (a stalled host can relay PTY bytes).
+    if (stall && Number(stall.durationMs) >= stallThresholdMs) {
+      entry.misses += 1;
+      entry.lastMissAt = now();
+      entry.verdict = 'unresponsive';
+      return { verdict: 'unresponsive', misses: entry.misses, stall };
+    }
+    // D-11 (WP-3): the host sent a frame at or after this request was issued --
+    // a deadlined request family timing out on a TALKING host is readiness, not
+    // liveness. Report 'degraded' once per host since the last reply; never
+    // count it toward the miss threshold, never verdict the host unresponsive.
+    if (typeof lastFrameAt === 'function' && Number.isFinite(issuedAt)) {
+      const frameAt = Number(lastFrameAt(id));
+      if (Number.isFinite(frameAt) && frameAt >= issuedAt) {
+        if (entry.degraded !== true) {
+          entry.degraded = true;
+          return { verdict: 'degraded', misses: entry.misses, lastFrameAgoMs: now() - frameAt };
+        }
+        return null;
+      }
+    }
+    // D-12: no frame of any kind since the first missed request was issued --
+    // today's path. Two such misses (missThreshold) is a silent host.
+    entry.misses += 1;
+    entry.lastMissAt = now();
+    if (entry.misses >= missThreshold) {
       entry.verdict = 'unresponsive';
       return { verdict: 'unresponsive', misses: entry.misses, stall: stall || null };
     }
@@ -94,6 +177,9 @@ function createHostHealthLedger({
     const entry = entryFor(id);
     entry.misses = 0;
     entry.lastReplyAt = now();
+    // D-11: the host answered, so whatever readiness conclusion was latched is
+    // stale evidence -- the next talking miss reports once again.
+    entry.degraded = false;
     // C4: the host answered -- whatever the cap concluded is stale evidence.
     if (entry.verdict === 'unresponsive-capped') {
       entry.verdict = 'ok';
@@ -116,17 +202,78 @@ function createHostHealthLedger({
     entry.cappedUntil = now() + CRASH_CAP_WINDOW_MS;
   }
 
+  // WP-3 (D-2, §6): the host's own threadpool snapshot as a verdict, not a
+  // banner. Returns { action:'evacuate', hostId, oldestPendingMs, stalledForMs }
+  // exactly ONCE per stall episode, and only for a host whose pool is either
+  // reported stalled with an already-overdue op (oldestPendingMs >= 30 s) or
+  // has been continuously stalled for >= 30 s with no 'ok' frame in between.
+  function recordThreadpool(hostId, snapshot) {
+    const id = String(hostId || '');
+    const state = snapshot && snapshot.state;
+    if (!id || (state !== 'ok' && state !== 'stalled')) return null; // unknown shape -- no state change
+    if (state === 'ok') {
+      const entry = hosts.get(id);
+      if (!entry) return null; // nothing was in progress -- no entry to create
+      _cancelThreadpoolTimer(entry);
+      entry.threadpoolStalledSince = 0;
+      entry.threadpoolOldestPendingMs = 0;
+      entry.evacuateFired = false;
+      return null;
+    }
+    const entry = entryFor(id);
+    if (!entry.threadpoolStalledSince) entry.threadpoolStalledSince = now(); // first stalled frame starts the episode
+    if (entry.evacuateFired) return null; // already acted on this episode
+    entry.threadpoolOldestPendingMs = Number(snapshot.oldestPendingMs) || 0;
+    const stalledForMs = now() - entry.threadpoolStalledSince;
+    if (entry.threadpoolOldestPendingMs >= THREADPOOL_EVACUATE_MS || stalledForMs >= THREADPOOL_EVACUATE_MS) {
+      _cancelThreadpoolTimer(entry);
+      return _markFired(entry, id);
+    }
+    // Not yet due: arm ONE timer for the remainder of the window (never stack --
+    // an existing timer already targets the same deadline and its own `hosts.get`
+    // liveness check keeps it honest across forget/move).
+    if (!entry.threadpoolTimer) {
+      const timer = setTimer(() => {
+        if (hosts.get(id) !== entry || entry.threadpoolStalledSince <= 0 || entry.evacuateFired) return;
+        _fireEvacuate(entry, id);
+      }, Math.max(1, THREADPOOL_EVACUATE_MS - stalledForMs));
+      timer?.unref?.();
+      entry.threadpoolTimer = timer || null;
+    }
+    return null;
+  }
+
+  // WP-3: re-arm RE-EVALUATION without forgetting the episode (used when an
+  // evacuation was deferred -- owner recovery pending): the next stalled frame
+  // re-decides immediately, keeping the episode's original stalledSince.
+  function resetThreadpoolEpisode(hostId) {
+    const entry = hosts.get(String(hostId || ''));
+    if (!entry) return;
+    _cancelThreadpoolTimer(entry);
+    entry.evacuateFired = false;
+  }
+
   function forgetHost(hostId) {
-    hosts.delete(String(hostId || ''));
+    const id = String(hostId || '');
+    _cancelThreadpoolTimer(hosts.get(id));
+    hosts.delete(id);
   }
 
-  // WP-4: a handoff replaces the host process, not its health history -- carry the entry over.
+  // WP-4: a handoff replaces the host process, not its health history -- carry
+  // the entry over. WP-3 (D-2): the successor is a fresh process with a fresh
+  // libuv pool, so its stall episode is dropped -- its own frames decide.
   function moveHost(fromId, toId) {
     const from = String(fromId || ''); const to = String(toId || '');
     if (!from || !to || from === to) return;
     const entry = hosts.get(from);
     hosts.delete(from);
-    if (entry) hosts.set(to, entry);
+    if (!entry) return;
+    _cancelThreadpoolTimer(entry);
+    entry.threadpoolStalledSince = 0;
+    entry.threadpoolOldestPendingMs = 0;
+    entry.evacuateFired = false;
+    _cancelThreadpoolTimer(hosts.get(to)); // a superseded entry's armed timer must never fire for its successor
+    hosts.set(to, entry);
   }
 
   function snapshot() {
@@ -138,12 +285,17 @@ function createHostHealthLedger({
         lastReplyAt: entry.lastReplyAt,
         verdict: entry.verdict,
         cappedUntil: entry.cappedUntil || 0,
+        threadpoolStalledSince: entry.threadpoolStalledSince || 0,
+        evacuateFired: entry.evacuateFired === true,
+        // D-11 (WP-3): a talking host that missed a hard deadline since its last
+        // reply -- readiness, reported once, never a kill.
+        degraded: entry.degraded === true,
       };
     }
     return { hosts: out };
   }
 
-  return { recordMiss, recordReply, verdict, markCapped, forgetHost, moveHost, snapshot };
+  return { recordMiss, recordReply, recordThreadpool, verdict, markCapped, forgetHost, moveHost, resetThreadpoolEpisode, snapshot };
 }
 
-module.exports = { createHostHealthLedger, readStallFromDataDir };
+module.exports = { createHostHealthLedger, readStallFromDataDir, THREADPOOL_EVACUATE_MS };

package/template/coding-task-manager/lib/host-self-truth-watchdog.js

@@ -34,6 +34,10 @@ const HOST_WATCHDOG_OP_TABLE = [
   'getViewportText', 'composer-probe', 'getRows', 'reset', 'flush-archive',
   'mirror-status', 'mirrors-report', 'trim-mirrors', 'flush-tail', 'destroy',
   'painted-read', 'drop-painted',
+  // WP-0 (D-6/S-13): appended ONLY -- an entry's index IS its opCode, so
+  // reordering or removing any of the rows above would relabel every stamp a
+  // still-running host writes. These three were missing and stamped as 'none'.
+  'serialize-bootstrap', 'painted-seal', 'painted-reopen',
 ];
 
 function _hostCpuUsageMs() {