@hellcoder/companion 0.110.1 → 0.110.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hellcoder/companion",
-  "version": "0.110.1",
+  "version": "0.110.2",
   "type": "module",
   "description": "Web UI for launching and interacting with Claude Code agents — Moritz Edition (fork of the-companion)",
   "license": "MIT",

package/server/claude-adapter.test.ts

@@ -1574,8 +1574,17 @@ describe("stdio transport disconnect propagation", () => {
       await vi.advanceTimersByTimeAsync(0);
       expect(proc.kill).not.toHaveBeenCalled();
 
-      // Past the grace window with no self-exit → treated as wedged and killed.
+      // The old flat 2s grace used to kill here. It no longer does: measured
+      // over 22 production kills that timer never caught a genuinely hung
+      // process — all 22 were healthy CLIs that had finished reaping and simply
+      // had not exited yet, and each kill destroyed the user's in-flight turn.
+      // The window is now 10s (COMPANION_STDOUT_CLOSE_GRACE_MS).
       await vi.advanceTimersByTimeAsync(2100);
+      expect(proc.kill).not.toHaveBeenCalled();
+
+      // Past the stall window with no self-exit → treated as wedged and killed,
+      // so the relaunch path's PID-liveness guard is still cleared.
+      await vi.advanceTimersByTimeAsync(9000);
       expect(proc.kill).toHaveBeenCalledTimes(1);
       expect(adapter.isConnected()).toBe(false);
       expect(disconnectCb).toHaveBeenCalledTimes(1);

package/server/claude-adapter.ts

@@ -46,7 +46,7 @@ import type {
 import type { SocketData } from "./ws-bridge-types.js";
 import type { PendingControlRequest } from "./ws-bridge-types.js";
 import type { RecorderManager } from "./recorder.js";
-import { captureProcState, hasLiveDescendants } from "./proc-diagnostics.js";
+import { captureProcState, hasLiveDescendants, countDescendants } from "./proc-diagnostics.js";
 import { parseNDJSON, isDuplicateCLIMessage } from "./ws-bridge-cli-ingest.js";
 import type { CLIDedupState } from "./ws-bridge-cli-ingest.js";
 import { reportProtocolDrift } from "./protocol-monitor.js";
@@ -57,14 +57,19 @@ import { reportProtocolDrift } from "./protocol-monitor.js";
 const CLI_DEDUP_WINDOW = 2000;
 
 /**
- * Grace period after stdout closes before we kill a still-alive CLI process.
- * The CLI's stdout often EOFs a beat *before* the process exits cleanly (e.g.
- * right after emitting a `result`). Killing immediately turns that code-0 exit
- * into a SIGTERM (143) and forces a needless relaunch. We wait this long for the
- * process to exit on its own; only a process that is still alive afterwards is
- * treated as genuinely wedged and killed. Overridable for tests/tuning.
+ * How long a process with no remaining descendants may sit without exiting
+ * before we treat it as wedged and kill it.
+ *
+ * This was a flat 2s. Measured over 22 kills that timer never once caught a
+ * genuinely hung process — all 22 were healthy (21 sleeping, 1 running),
+ * typically S/ep_poll with 12 threads: CLIs that had finished reaping and had
+ * simply not exited yet. Each kill destroyed the user's in-flight turn.
+ *
+ * Raised to 10s to match the descendants path. A real wedge is still caught,
+ * just 8s later — a trade that costs recovery latency in the rare case and
+ * saves a user's answer in the common one.
  */
-const STDOUT_CLOSE_GRACE_MS = Number(process.env.COMPANION_STDOUT_CLOSE_GRACE_MS) || 2000;
+const STDOUT_CLOSE_GRACE_MS = Number(process.env.COMPANION_STDOUT_CLOSE_GRACE_MS) || 10_000;
 
 /**
  * Longer grace used when stdout EOFs right after a terminal `result` (a clean
@@ -84,6 +89,21 @@ const STDOUT_CLOSE_RESULT_GRACE_MS = Number(process.env.COMPANION_STDOUT_CLOSE_R
  */
 const SIGKILL_ESCALATION_MS = Number(process.env.COMPANION_SIGKILL_ESCALATION_MS) || 2000;
 
+/**
+ * Absolute ceiling on waiting for MCP teardown, however well it is progressing.
+ *
+ * The flat 10s grace was not enough: 8 of 12 observed kills were processes
+ * correctly identified as still reaping children, waited the full 10s for, and
+ * killed anyway while perfectly healthy (S sleeping, 311-351MB). `npm exec`
+ * -wrapped MCP servers plus headless chromium routinely exceed 10s, especially
+ * with ~70 CLIs contending for I/O.
+ */
+const TEARDOWN_MAX_MS = Number(process.env.COMPANION_TEARDOWN_MAX_MS) || 120_000;
+
+/** How often to re-check the descendant count while waiting for teardown. */
+const TEARDOWN_POLL_MS = Number(process.env.COMPANION_TEARDOWN_POLL_MS) || 500;
+
+
 // --- Claude Code Adapter ------------------------------------------------------
 
 export class ClaudeAdapter implements IBackendAdapter {
@@ -348,15 +368,15 @@ export class ClaudeAdapter implements IBackendAdapter {
         // mid-exit.
         const teardownInProgress = hasLiveDescendants(proc.pid);
         if (this.lastInboundWasResult || teardownInProgress) {
-          // Clean end-of-turn shutdown: let the process flush teardown (MCP
-          // servers, etc.) and exit code-0 instead of SIGTERM-ing a code-0 exit.
-          // Use a generous grace so we don't clobber a slow-but-clean exit, but
-          // keep it bounded — a process that wedges *after* a result must still
-          // be killed so recovery isn't blocked indefinitely.
-          const exitedOnOwn = await Promise.race([
-            proc.exited.then(() => true),
-            new Promise<boolean>((resolve) => setTimeout(() => resolve(false), STDOUT_CLOSE_RESULT_GRACE_MS)),
-          ]);
+          // Wait for teardown ADAPTIVELY rather than on a flat timer. A fixed
+          // 10s grace still killed 8 of 12 healthy processes: it cannot know
+          // how long `npm exec`-wrapped MCP servers plus headless chromium will
+          // take, especially with many CLIs contending for I/O.
+          //
+          // Instead, keep waiting as long as the descendant count is dropping —
+          // teardown is demonstrably working, so killing would be wrong at any
+          // deadline. Give up only when progress stalls, or at a hard ceiling.
+          const exitedOnOwn = await this.awaitTeardown(proc, STDOUT_CLOSE_RESULT_GRACE_MS);
           if (!exitedOnOwn && proc.exitCode === null && !proc.killed) {
             // Capture kernel state BEFORE the kill: once we SIGTERM, the
             // evidence is gone. A wedged CLI writes nothing to stderr, so this
@@ -364,17 +384,25 @@ export class ClaudeAdapter implements IBackendAdapter {
             log.warn("claude-adapter", "stdout closed after result but process did not exit within grace; killing wedged process", {
               sessionId: this.sessionId,
               pid: proc.pid,
-              graceMs: STDOUT_CLOSE_RESULT_GRACE_MS,
               graceReason: this.lastInboundWasResult ? "result" : "descendants_alive",
+              teardownOutcome: this.lastTeardownOutcome,
               proc: captureProcState(proc.pid),
             });
             await this.killWithEscalation(proc);
           }
         } else {
-          const exitedOnOwn = await Promise.race([
-            proc.exited.then(() => true),
-            new Promise<boolean>((resolve) => setTimeout(() => resolve(false), STDOUT_CLOSE_GRACE_MS)),
-          ]);
+          // No descendants left to reap. This used to take a flat 2s timer, but
+          // measured over 22 kills that timer never once caught a genuinely
+          // hung process: every single one was healthy (21 sleeping, 1 running,
+          // 0 hung), typically S/ep_poll with 12 threads — a CLI that had
+          // finished reaping and simply had not exited yet.
+          //
+          // A CLI in that state is indistinguishable from a real wedge by any
+          // signal available here, so the tie is broken on cost instead:
+          // killing a healthy process destroys the user's in-flight turn, while
+          // waiting longer on a truly wedged one only delays recovery. So wait
+          // the same way, bounded by the stall window rather than a 2s guess.
+          const exitedOnOwn = await this.awaitTeardown(proc, STDOUT_CLOSE_GRACE_MS);
           if (!exitedOnOwn && proc.exitCode === null && !proc.killed) {
             // Reaching here now means a genuine wedge: stdout closed, no live
             // descendants to reap, and still not exited. Snapshot kernel state
@@ -394,6 +422,60 @@ export class ClaudeAdapter implements IBackendAdapter {
     }
   }
 
+  /** Why the last teardown wait ended, for the kill warning. */
+  private lastTeardownOutcome: string | undefined;
+
+  /**
+   * Wait for a process to exit while its descendants are still being reaped.
+   *
+   * Returns true if it exited on its own. The wait is adaptive: as long as the
+   * descendant count keeps dropping, teardown is demonstrably working and we
+   * keep waiting regardless of elapsed time. We give up only when the count
+   * sits unchanged for TEARDOWN_STALL_MS (nothing is happening) or the hard
+   * TEARDOWN_MAX_MS ceiling is hit.
+   *
+   * This replaces a flat 10s grace that killed 8 of 12 healthy processes: a
+   * fixed deadline cannot know how long `npm exec`-wrapped MCP servers plus
+   * headless chromium need, particularly under contention.
+   */
+  private async awaitTeardown(proc: Subprocess, stallMs: number): Promise<boolean> {
+    const started = Date.now();
+    let lastCount = countDescendants(proc.pid);
+    let lastProgressAt = started;
+    this.lastTeardownOutcome = undefined;
+
+    while (true) {
+      const exited = await Promise.race([
+        proc.exited.then(() => true),
+        new Promise<boolean>((resolve) => setTimeout(() => resolve(false), TEARDOWN_POLL_MS)),
+      ]);
+      if (exited || proc.exitCode !== null) {
+        this.lastTeardownOutcome = `exited_after_${Date.now() - started}ms`;
+        return true;
+      }
+
+      const now = Date.now();
+      const count = countDescendants(proc.pid);
+      if (count < lastCount) {
+        // Progress: children are being reaped. Reset the stall window — a slow
+        // but advancing teardown must never be killed.
+        lastCount = count;
+        lastProgressAt = now;
+      }
+
+      if (now - lastProgressAt >= stallMs) {
+        this.lastTeardownOutcome =
+          `stalled_at_${count}_descendants_after_${now - started}ms`;
+        return false;
+      }
+      if (now - started >= TEARDOWN_MAX_MS) {
+        this.lastTeardownOutcome =
+          `ceiling_${TEARDOWN_MAX_MS}ms_with_${count}_descendants`;
+        return false;
+      }
+    }
+  }
+
   /**
    * SIGTERM a process, then SIGKILL it if it has not exited.
    *

package/server/proc-diagnostics.test.ts

@@ -5,6 +5,7 @@ import {
   isProcAvailable,
   getDescendants,
   hasLiveDescendants,
+  countDescendants,
 } from "./proc-diagnostics.js";
 
 /**
@@ -129,6 +130,48 @@ describe("getDescendants / hasLiveDescendants", () => {
     expect(getDescendants(1, 2).length).toBeLessThanOrEqual(2);
   });
 
+  /**
+   * countDescendants drives the adaptive teardown wait in claude-adapter: a
+   * falling count means teardown is progressing and the process must NOT be
+   * killed, however long it takes. A count that stops falling is what ends the
+   * wait. So the critical property is that the count actually tracks reality.
+   */
+  it("countDescendants returns 0 for undefined pid and non-Linux", () => {
+    expect(countDescendants(undefined)).toBe(0);
+    vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
+    expect(countDescendants(1)).toBe(0);
+  });
+
+  it.runIf(isLinux)("countDescendants drops as children exit", async () => {
+    // This is the exact signal the adaptive wait keys on. If the count did not
+    // fall as children are reaped, a healthy teardown would look like a stall
+    // and get killed — the bug this replaces.
+    const a = spawn("sleep", ["30"], { stdio: "ignore" });
+    const b = spawn("sleep", ["30"], { stdio: "ignore" });
+    await new Promise((r) => setTimeout(r, 150));
+
+    const withBoth = countDescendants(process.pid);
+    expect(withBoth).toBeGreaterThanOrEqual(2);
+
+    a.kill("SIGKILL");
+    await new Promise((r) => a.on("exit", r));
+    await new Promise((r) => setTimeout(r, 150));
+
+    const withOne = countDescendants(process.pid);
+    expect(withOne).toBeLessThan(withBoth);
+
+    b.kill("SIGKILL");
+    await new Promise((r) => b.on("exit", r));
+    await new Promise((r) => setTimeout(r, 150));
+
+    expect(countDescendants(process.pid)).toBeLessThan(withOne);
+  });
+
+  it.runIf(isLinux)("countDescendants is bounded by maxNodes", () => {
+    // Bounds the poll-loop cost: this runs every 500ms during teardown.
+    expect(countDescendants(1, 3)).toBeLessThanOrEqual(3);
+  });
+
   it.runIf(isLinux)("includes descendants in the captured snapshot", async () => {
     const child = spawn("sleep", ["30"], { stdio: "ignore" });
     await new Promise((r) => setTimeout(r, 150));

package/server/proc-diagnostics.ts

@@ -125,6 +125,38 @@ export function hasLiveDescendants(pid: number | undefined): boolean {
   return readChildren(pid).length > 0;
 }
 
+/**
+ * Count live descendants without reading per-process detail.
+ *
+ * Cheaper than getDescendants() because it skips the comm/status reads, so it
+ * is safe to call on a poll loop. Used to tell *progress* (the count is
+ * dropping, teardown is working) from a *stall* (the count is stuck, nothing is
+ * happening) — which is what decides whether waiting longer is worthwhile.
+ */
+export function countDescendants(
+  pid: number | undefined,
+  maxNodes = 256,
+): number {
+  if (pid === undefined || !isProcAvailable()) return 0;
+
+  let count = 0;
+  const seen = new Set<number>([pid]);
+  let frontier = readChildren(pid);
+
+  while (frontier.length > 0 && count < maxNodes) {
+    const next: number[] = [];
+    for (const child of frontier) {
+      if (seen.has(child) || count >= maxNodes) continue;
+      seen.add(child);
+      count++;
+      next.push(...readChildren(child));
+    }
+    frontier = next;
+  }
+
+  return count;
+}
+
 /** True when /proc-based introspection is available (Linux only). */
 export function isProcAvailable(): boolean {
   return process.platform === "linux";

package/server/ws-bridge.test.ts

@@ -1923,6 +1923,46 @@ describe("CLI message routing", () => {
     expect(permBroadcast.request.request_id).toBe("req-reattach");
   });
 
+  /**
+   * Regression: typing into a session whose CLI has already been killed used to
+   * queue the message with no process to drain it. The transition
+   * terminated -> streaming is (correctly) illegal, so the turn never started;
+   * the message only moved when some unrelated event happened to relaunch the
+   * session. From the user's side the session looked alive but silently
+   * swallowed input, then replayed the turn later from the top.
+   *
+   * Typing into a terminated session is an explicit request to resume it, so it
+   * must request the relaunch itself. terminated -> starting is legal, and the
+   * adapter queue flushes on attach.
+   */
+  it("user_message in a terminated session requests a relaunch", async () => {
+    const relaunchCb = vi.fn();
+    companionBus.on("session:relaunch-needed", ({ sessionId }) => relaunchCb(sessionId));
+
+    const cli = makeCliSocket("s1");
+    bridge.handleCLIOpen(cli, "s1");
+    await bridge.handleCLIMessage(cli, makeInitMsg());
+    const session = bridge.getSession("s1")!;
+
+    // Drive the session to terminated, as a wedge kill would.
+    session.stateMachine.transition("terminated", "test_kill");
+    expect(session.stateMachine.phase).toBe("terminated");
+
+    const browser = makeBrowserSocket("s1");
+    bridge.handleBrowserOpen(browser, "s1");
+    relaunchCb.mockClear();
+
+    await bridge.handleBrowserMessage(browser, JSON.stringify({
+      type: "user_message",
+      content: "still working?",
+    }));
+
+    expect(relaunchCb).toHaveBeenCalledWith("s1");
+    // The message must still be preserved for replay, not dropped in favour of
+    // the relaunch.
+    expect(session.inFlightUserTurn).toBeTruthy();
+  });
+
   it("tool_progress: broadcasts", async () => {
     const msg = JSON.stringify({
       type: "tool_progress",

package/server/ws-bridge.ts

@@ -1216,6 +1216,18 @@ export class WsBridge {
           sessionId: session.id,
           phase: session.stateMachine.phase,
         });
+        // A terminated session has no process to drain that queue, so the
+        // message would sit there until some unrelated event (proactive
+        // keepalive) happened to relaunch — the user sees a dead session that
+        // silently swallows input. Typing into a terminated session is an
+        // explicit request to resume it, so ask for the relaunch here:
+        // terminated -> starting is legal, and the queue flushes on attach.
+        if (session.stateMachine.phase === "terminated") {
+          log.info("ws-bridge", "User message in terminated session; requesting relaunch", {
+            sessionId: session.id,
+          });
+          companionBus.emit("session:relaunch-needed", { sessionId: session.id });
+        }
       }
       this.persistSession(session);
       this.broadcastToBrowsers(session, userMessage);