@hellcoder/companion 0.111.2-preview.20260728013354.d1fdce0 → 0.111.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hellcoder/companion",
3
- "version": "0.111.2-preview.20260728013354.d1fdce0",
3
+ "version": "0.111.2",
4
4
  "type": "module",
5
5
  "description": "Web UI for launching and interacting with Claude Code agents — Moritz Edition (fork of the-companion)",
6
6
  "license": "MIT",
@@ -28,9 +28,15 @@ vi.mock("node:crypto", () => ({
28
28
  // Default 0 (childless) preserves the existing tests' behaviour; individual
29
29
  // tests override it.
30
30
  const mockCountDescendants = vi.hoisted(() => vi.fn(() => 0));
31
+ // getDescendants is stubbed so the orphan-reaping tests can present a known tree
32
+ // without spawning real children. Default empty, matching a childless CLI.
33
+ const mockGetDescendants = vi.hoisted(() =>
34
+ vi.fn((): { pid: number; comm?: string; state?: string }[] => []),
35
+ );
31
36
  vi.mock("./proc-diagnostics.js", async (importOriginal) => ({
32
37
  ...(await importOriginal<typeof import("./proc-diagnostics.js")>()),
33
38
  countDescendants: mockCountDescendants,
39
+ getDescendants: mockGetDescendants,
34
40
  }));
35
41
 
36
42
  // Settings are stubbed so the wedge-kill and silence-probe switches can be
@@ -1870,3 +1876,296 @@ describe("stdio silence probe", () => {
1870
1876
  expect(adapter.isConnected()).toBe(true);
1871
1877
  });
1872
1878
  });
1879
+
1880
+ /**
1881
+ * Turn-stall — the variant the silence probe structurally cannot catch.
1882
+ *
1883
+ * Reported from production on 0.111.1 (session aa37e7de): the turn died at
1884
+ * 02:10:56 mid-`stream_event` while the CLI kept answering probes. Each reply
1885
+ * refreshes `lastInboundAt` and pushes escalation out another interval — five
1886
+ * probes were answered before the process went silent outright and the existing
1887
+ * probe could fire, landing recovery at 02:29:33. An 18.6-minute hang, nearly
1888
+ * all of it after the turn was already dead. Turn progress has to be tracked
1889
+ * independently of transport liveness.
1890
+ */
1891
+ describe("stdio turn-stall detection", () => {
1892
+ let adapter: ClaudeAdapter;
1893
+ let disconnectCb: ReturnType<typeof vi.fn>;
1894
+
1895
+ beforeEach(() => {
1896
+ adapter = new ClaudeAdapter("turn-stall-session");
1897
+ disconnectCb = vi.fn();
1898
+ adapter.onDisconnect(disconnectCb as unknown as () => void);
1899
+ mockSettings.silenceProbeEnabled = true;
1900
+ });
1901
+
1902
+ afterEach(() => {
1903
+ mockSettings.silenceProbeEnabled = true;
1904
+ vi.useRealTimers();
1905
+ });
1906
+
1907
+ const controlResponse = (i: number) =>
1908
+ JSON.stringify({
1909
+ type: "control_response",
1910
+ response: { subtype: "mcp_status", request_id: `r-${i}` },
1911
+ }) + "\n";
1912
+
1913
+ const streamEvent = () =>
1914
+ JSON.stringify({
1915
+ type: "stream_event",
1916
+ event: { type: "content_block_delta", index: 0, delta: { type: "thinking_delta", thinking: "" } },
1917
+ session_id: "s",
1918
+ }) + "\n";
1919
+
1920
+ const startTurn = () => adapter.send({ type: "user_message", content: "go" } as never);
1921
+
1922
+ it("fires when the CLI answers control requests but the turn is dead", async () => {
1923
+ vi.useFakeTimers();
1924
+ const { proc, pushStdout } = createMockProc();
1925
+ adapter.attachStdio(proc);
1926
+ startTurn();
1927
+
1928
+ // The turn streams briefly, then dies — exactly the aa37e7de shape.
1929
+ pushStdout(streamEvent());
1930
+ await vi.advanceTimersByTimeAsync(1_000);
1931
+
1932
+ // ...while the transport stays chatty. This is what pins the silence probe
1933
+ // permanently disarmed, so anything that fires here is the turn-stall check.
1934
+ for (let i = 0; i < 12; i++) {
1935
+ await vi.advanceTimersByTimeAsync(30_000);
1936
+ pushStdout(controlResponse(i));
1937
+ }
1938
+
1939
+ expect(disconnectCb).toHaveBeenCalledTimes(1);
1940
+ expect(adapter.isConnected()).toBe(false);
1941
+ });
1942
+
1943
+ it("does NOT fire while a tool call is outstanding", async () => {
1944
+ // The critical false-positive guard: a long `Bash` run legitimately emits
1945
+ // nothing for many minutes and must never be torn down.
1946
+ vi.useFakeTimers();
1947
+ const { proc, pushStdout } = createMockProc();
1948
+ adapter.attachStdio(proc);
1949
+ startTurn();
1950
+
1951
+ pushStdout(
1952
+ JSON.stringify({
1953
+ type: "assistant",
1954
+ message: {
1955
+ role: "assistant",
1956
+ model: "m",
1957
+ content: [{ type: "tool_use", id: "tu-1", name: "Bash", input: {} }],
1958
+ },
1959
+ session_id: "s",
1960
+ }) + "\n",
1961
+ );
1962
+
1963
+ // Keep the transport answering so the (separate) silence probe stays out of
1964
+ // it — this test is only about the turn-stall check.
1965
+ for (let i = 0; i < 20; i++) {
1966
+ await vi.advanceTimersByTimeAsync(30_000);
1967
+ pushStdout(controlResponse(i));
1968
+ }
1969
+
1970
+ expect(disconnectCb).not.toHaveBeenCalled();
1971
+ expect(adapter.isConnected()).toBe(true);
1972
+ });
1973
+
1974
+ it("resumes watching once the tool result comes back", async () => {
1975
+ vi.useFakeTimers();
1976
+ const { proc, pushStdout } = createMockProc();
1977
+ adapter.attachStdio(proc);
1978
+ startTurn();
1979
+
1980
+ pushStdout(
1981
+ JSON.stringify({
1982
+ type: "assistant",
1983
+ message: {
1984
+ role: "assistant",
1985
+ model: "m",
1986
+ content: [{ type: "tool_use", id: "tu-1", name: "Bash", input: {} }],
1987
+ },
1988
+ session_id: "s",
1989
+ }) + "\n",
1990
+ );
1991
+ for (let i = 0; i < 14; i++) {
1992
+ await vi.advanceTimersByTimeAsync(30_000);
1993
+ pushStdout(controlResponse(i));
1994
+ }
1995
+ expect(disconnectCb).not.toHaveBeenCalled();
1996
+
1997
+ // Tool finished; the model should now be producing output again. It does
1998
+ // not, so the exemption lifts and the stall is caught.
1999
+ pushStdout(
2000
+ JSON.stringify({
2001
+ type: "user",
2002
+ message: { role: "user", content: [{ type: "tool_result", tool_use_id: "tu-1", content: "ok" }] },
2003
+ session_id: "s",
2004
+ }) + "\n",
2005
+ );
2006
+ for (let i = 0; i < 12; i++) {
2007
+ await vi.advanceTimersByTimeAsync(30_000);
2008
+ pushStdout(controlResponse(100 + i));
2009
+ }
2010
+
2011
+ expect(disconnectCb).toHaveBeenCalledTimes(1);
2012
+ });
2013
+
2014
+ it("does NOT fire on an idle session with no turn outstanding", async () => {
2015
+ // A session simply waiting for the user emits nothing for hours.
2016
+ vi.useFakeTimers();
2017
+ const { proc, pushStdout } = createMockProc();
2018
+ adapter.attachStdio(proc);
2019
+
2020
+ for (let i = 0; i < 12; i++) {
2021
+ await vi.advanceTimersByTimeAsync(30_000);
2022
+ pushStdout(controlResponse(i));
2023
+ }
2024
+
2025
+ expect(disconnectCb).not.toHaveBeenCalled();
2026
+ expect(adapter.isConnected()).toBe(true);
2027
+ });
2028
+
2029
+ it("does NOT fire while the turn keeps producing output", async () => {
2030
+ vi.useFakeTimers();
2031
+ const { proc, pushStdout } = createMockProc();
2032
+ adapter.attachStdio(proc);
2033
+ startTurn();
2034
+
2035
+ for (let i = 0; i < 12; i++) {
2036
+ await vi.advanceTimersByTimeAsync(30_000);
2037
+ pushStdout(streamEvent());
2038
+ }
2039
+
2040
+ expect(disconnectCb).not.toHaveBeenCalled();
2041
+ expect(adapter.isConnected()).toBe(true);
2042
+ });
2043
+
2044
+ it("does NOT fire after the turn ends with a result", async () => {
2045
+ vi.useFakeTimers();
2046
+ const { proc, pushStdout } = createMockProc();
2047
+ adapter.attachStdio(proc);
2048
+ startTurn();
2049
+
2050
+ pushStdout(
2051
+ JSON.stringify({ type: "result", subtype: "success", is_error: false, session_id: "s" }) + "\n",
2052
+ );
2053
+ for (let i = 0; i < 12; i++) {
2054
+ await vi.advanceTimersByTimeAsync(30_000);
2055
+ pushStdout(controlResponse(i));
2056
+ }
2057
+
2058
+ expect(disconnectCb).not.toHaveBeenCalled();
2059
+ expect(adapter.isConnected()).toBe(true);
2060
+ });
2061
+ });
2062
+
2063
+ /**
2064
+ * Orphaned MCP descendants.
2065
+ *
2066
+ * Signals go to a pid, not a tree, so killing a wedged CLI leaves its 2-3 stdio
2067
+ * MCP servers (and, for @playwright/mcp, a headless chromium) re-parented to
2068
+ * init with no client that can ever reach them. Under kill/relaunch churn they
2069
+ * accumulate: a production box was found holding 123 orphans across 11.9 GB of
2070
+ * 23 GB total, starving the surviving CLIs until they stopped answering — which
2071
+ * caused more kills, and more orphans.
2072
+ */
2073
+ describe("orphaned descendant reaping", () => {
2074
+ let adapter: ClaudeAdapter;
2075
+ let killSpy: ReturnType<typeof vi.spyOn>;
2076
+ let signalled: { pid: number; sig: unknown }[];
2077
+
2078
+ beforeEach(() => {
2079
+ adapter = new ClaudeAdapter("orphan-session");
2080
+ signalled = [];
2081
+ // Simulate a live process table: signal 0 succeeds, real signals recorded.
2082
+ killSpy = vi.spyOn(process, "kill").mockImplementation(((pid: number, sig?: unknown) => {
2083
+ if (sig === 0) return true;
2084
+ signalled.push({ pid, sig });
2085
+ return true;
2086
+ }) as unknown as typeof process.kill);
2087
+ });
2088
+
2089
+ afterEach(() => {
2090
+ killSpy.mockRestore();
2091
+ mockGetDescendants.mockReturnValue([]);
2092
+ vi.useRealTimers();
2093
+ });
2094
+
2095
+ /** Drive the wedge path far enough to trigger killWithEscalation. */
2096
+ const wedgeAndKill = async (endStdout: () => void) => {
2097
+ endStdout();
2098
+ await vi.advanceTimersByTimeAsync(11_000); // past the stdout-close grace
2099
+ };
2100
+
2101
+ it("terminates descendants that outlived the CLI", async () => {
2102
+ vi.useFakeTimers();
2103
+ mockGetDescendants.mockReturnValue([{ pid: 99001 }, { pid: 99002 }]);
2104
+ const { proc, endStdout } = createMockProc();
2105
+ adapter.attachStdio(proc);
2106
+
2107
+ await wedgeAndKill(endStdout);
2108
+
2109
+ expect(proc.kill).toHaveBeenCalled();
2110
+ expect(signalled.map((s) => s.pid).sort()).toEqual([99001, 99002]);
2111
+ expect(signalled.every((s) => s.sig === "SIGTERM")).toBe(true);
2112
+ });
2113
+
2114
+ it("escalates to SIGKILL for descendants that ignore SIGTERM", async () => {
2115
+ // Chromium routinely ignores SIGTERM.
2116
+ vi.useFakeTimers();
2117
+ mockGetDescendants.mockReturnValue([{ pid: 99003 }]);
2118
+ const { proc, endStdout } = createMockProc();
2119
+ adapter.attachStdio(proc);
2120
+
2121
+ await wedgeAndKill(endStdout);
2122
+ expect(signalled).toEqual([{ pid: 99003, sig: "SIGTERM" }]);
2123
+
2124
+ await vi.advanceTimersByTimeAsync(2100);
2125
+ expect(signalled).toContainEqual({ pid: 99003, sig: "SIGKILL" });
2126
+ });
2127
+
2128
+ it("does NOT signal a pid whose identity can no longer be confirmed", async () => {
2129
+ // The recycled-pid guard. A snapshotted descendant carrying a `comm` is only
2130
+ // signalled if /proc still reports that same comm — otherwise the pid has
2131
+ // been reused by an unrelated process and must be left alone. Here /proc has
2132
+ // no such entry at all, so it must be skipped.
2133
+ vi.useFakeTimers();
2134
+ mockGetDescendants.mockReturnValue([{ pid: 99004, comm: "npm exec" }]);
2135
+ const { proc, endStdout } = createMockProc();
2136
+ adapter.attachStdio(proc);
2137
+
2138
+ await wedgeAndKill(endStdout);
2139
+
2140
+ expect(proc.kill).toHaveBeenCalled();
2141
+ expect(signalled).toEqual([]);
2142
+ });
2143
+
2144
+ it("skips descendants that already exited with their parent", async () => {
2145
+ vi.useFakeTimers();
2146
+ killSpy.mockImplementation(((pid: number, sig?: unknown) => {
2147
+ if (sig === 0) throw new Error("ESRCH"); // nothing alive
2148
+ signalled.push({ pid, sig });
2149
+ return true;
2150
+ }) as unknown as typeof process.kill);
2151
+ mockGetDescendants.mockReturnValue([{ pid: 99005 }, { pid: 99006 }]);
2152
+ const { proc, endStdout } = createMockProc();
2153
+ adapter.attachStdio(proc);
2154
+
2155
+ await wedgeAndKill(endStdout);
2156
+
2157
+ expect(signalled).toEqual([]);
2158
+ });
2159
+
2160
+ it("is a no-op for a childless CLI", async () => {
2161
+ vi.useFakeTimers();
2162
+ mockGetDescendants.mockReturnValue([]);
2163
+ const { proc, endStdout } = createMockProc();
2164
+ adapter.attachStdio(proc);
2165
+
2166
+ await wedgeAndKill(endStdout);
2167
+
2168
+ expect(proc.kill).toHaveBeenCalled();
2169
+ expect(signalled).toEqual([]);
2170
+ });
2171
+ });
@@ -10,7 +10,7 @@
10
10
  */
11
11
 
12
12
  import { randomUUID } from "node:crypto";
13
- import { mkdirSync, writeFileSync } from "node:fs";
13
+ import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
14
14
  import { join, basename } from "node:path";
15
15
  import { log } from "./logger.js";
16
16
  import type { ServerWebSocket, Subprocess } from "bun";
@@ -46,7 +46,7 @@ import type {
46
46
  import type { SocketData } from "./ws-bridge-types.js";
47
47
  import type { PendingControlRequest } from "./ws-bridge-types.js";
48
48
  import type { RecorderManager } from "./recorder.js";
49
- import { captureProcState, hasLiveDescendants, countDescendants } from "./proc-diagnostics.js";
49
+ import { captureProcState, hasLiveDescendants, countDescendants, getDescendants, type ProcDescendant } from "./proc-diagnostics.js";
50
50
  import { getSettings } from "./settings-manager.js";
51
51
  import { parseNDJSON, isDuplicateCLIMessage } from "./ws-bridge-cli-ingest.js";
52
52
  import type { CLIDedupState } from "./ws-bridge-cli-ingest.js";
@@ -129,9 +129,71 @@ const SILENCE_PROBE_AFTER_MS = Number(process.env.COMPANION_SILENCE_PROBE_AFTER_
129
129
  const SILENCE_PROBE_TIMEOUT_MS = Number(process.env.COMPANION_SILENCE_PROBE_TIMEOUT_MS) || 60_000;
130
130
  const SILENCE_CHECK_INTERVAL_MS = Number(process.env.COMPANION_SILENCE_CHECK_INTERVAL_MS) || 15_000;
131
131
 
132
+ /**
133
+ * Turn-stall threshold.
134
+ *
135
+ * The silence probe above keys on *total* inbound silence, which recovers a
136
+ * dead turn far later than it needs to — and only if the CLI eventually stops
137
+ * answering control requests as well.
138
+ *
139
+ * Observed on 0.111.1 (session aa37e7de): turn output stopped at 02:10:56
140
+ * mid-`stream_event`, but the CLI kept replying to probes. Every reply refreshes
141
+ * `lastInboundAt`, so escalation is pushed out another full interval each time —
142
+ * five probes were answered before the process finally went silent outright and
143
+ * the probe could fire. Recovery landed at 02:29:33: an **18.6-minute** hang for
144
+ * the user, nearly all of it spent watching a turn that was already dead.
145
+ *
146
+ * The probe is doing its job; it is just asking the wrong question. Transport
147
+ * liveness and turn progress are independent, and a CLI that answers pings
148
+ * forever while its turn is dead would postpone escalation forever.
149
+ *
150
+ * So track turn progress separately from transport liveness. Only substantive
151
+ * turn frames (`stream_event`, `assistant`, `user`, `result`, `tool_progress`)
152
+ * count — control and keepalive traffic does not.
153
+ *
154
+ * False positives are the thing to fear (every previous timer in this file was
155
+ * too aggressive), so this only applies while a turn is actually outstanding,
156
+ * and never while a tool call is in flight: a long `Bash` run legitimately
157
+ * emits nothing for many minutes. The default is deliberately well beyond any
158
+ * plausible model think-time.
159
+ */
160
+ const TURN_STALL_AFTER_MS = Number(process.env.COMPANION_TURN_STALL_AFTER_MS) || 300_000;
161
+
132
162
  /** How often to re-check the descendant count while waiting for teardown. */
133
163
  const TEARDOWN_POLL_MS = Number(process.env.COMPANION_TEARDOWN_POLL_MS) || 500;
134
164
 
165
+ /**
166
+ * Does this frame type represent the turn actually making progress?
167
+ *
168
+ * `control_response` and `keep_alive` are excluded on purpose — they are the
169
+ * frames a stalled-but-responsive CLI keeps sending. `system` is excluded for
170
+ * the same reason it is excluded from `lastInboundWasResult`: it carries status
171
+ * and keepalive chatter, not turn work.
172
+ */
173
+ function isTurnOutput(type: string): boolean {
174
+ return (
175
+ type === "stream_event" ||
176
+ type === "assistant" ||
177
+ type === "user" ||
178
+ type === "result" ||
179
+ type === "tool_progress"
180
+ );
181
+ }
182
+
183
+ /** Extract `tool_use` / `tool_result` ids from a message's content blocks. */
184
+ function toolBlockIds(content: unknown): { uses: string[]; results: string[] } {
185
+ const uses: string[] = [];
186
+ const results: string[] = [];
187
+ if (!Array.isArray(content)) return { uses, results };
188
+ for (const block of content) {
189
+ if (!block || typeof block !== "object") continue;
190
+ const b = block as { type?: string; id?: string; tool_use_id?: string };
191
+ if (b.type === "tool_use" && typeof b.id === "string") uses.push(b.id);
192
+ if (b.type === "tool_result" && typeof b.tool_use_id === "string") results.push(b.tool_use_id);
193
+ }
194
+ return { uses, results };
195
+ }
196
+
135
197
 
136
198
  // --- Claude Code Adapter ------------------------------------------------------
137
199
 
@@ -173,6 +235,20 @@ export class ClaudeAdapter implements IBackendAdapter {
173
235
  /** When the outstanding silence probe was sent, or null if none is in flight. */
174
236
  private probeSentAt: number | null = null;
175
237
  private silenceTimer: ReturnType<typeof setInterval> | null = null;
238
+ /**
239
+ * Timestamp of the last substantive *turn* frame, for the turn-stall check.
240
+ * Deliberately narrower than `lastInboundAt`: control_response and keep_alive
241
+ * prove the transport is alive but say nothing about the turn making progress.
242
+ */
243
+ private lastTurnOutputAt = Date.now();
244
+ /** True while a user turn is outstanding — sent to the CLI, no `result` yet. */
245
+ private turnInFlight = false;
246
+ /**
247
+ * `tool_use` ids emitted by the model that have no matching `tool_result` yet.
248
+ * A long-running tool produces no turn output for minutes at a time and must
249
+ * never be mistaken for a stall.
250
+ */
251
+ private pendingToolUses = new Set<string>();
176
252
 
177
253
  // Callbacks registered by the bridge via on*() methods
178
254
  private browserMessageCb: ((msg: BrowserIncomingMessage) => void) | null = null;
@@ -488,6 +564,11 @@ export class ClaudeAdapter implements IBackendAdapter {
488
564
  this.stopSilenceProbe();
489
565
  this.lastInboundAt = Date.now();
490
566
  this.probeSentAt = null;
567
+ // A relaunched transport starts with a clean slate: the bridge replays the
568
+ // in-flight turn, which re-arms turnInFlight via handleOutgoingUserMessage.
569
+ this.lastTurnOutputAt = Date.now();
570
+ this.turnInFlight = false;
571
+ this.pendingToolUses.clear();
491
572
  this.silenceTimer = setInterval(() => this.checkSilence(), SILENCE_CHECK_INTERVAL_MS);
492
573
  // Never hold the process open for a diagnostic timer.
493
574
  (this.silenceTimer as unknown as { unref?: () => void }).unref?.();
@@ -501,12 +582,47 @@ export class ClaudeAdapter implements IBackendAdapter {
501
582
  this.probeSentAt = null;
502
583
  }
503
584
 
585
+ /**
586
+ * Maintain the outstanding-tool set so a long tool call is never mistaken for
587
+ * a stalled turn. The model announces work with `tool_use` blocks on an
588
+ * `assistant` frame; the CLI echoes the matching `tool_result` blocks back on
589
+ * a `user` frame when it finishes.
590
+ */
591
+ private trackToolCalls(msg: CLIMessage): void {
592
+ if (msg.type !== "assistant" && msg.type !== "user") return;
593
+ const content = (msg as { message?: { content?: unknown } }).message?.content;
594
+ const { uses, results } = toolBlockIds(content);
595
+ for (const id of uses) this.pendingToolUses.add(id);
596
+ for (const id of results) this.pendingToolUses.delete(id);
597
+ }
598
+
504
599
  private checkSilence(): void {
505
600
  if (this.transportKind !== "stdio" || !this.stdioConnected) return;
506
601
  if (getSettings().silenceProbeEnabled === false) return;
507
602
 
508
603
  const now = Date.now();
509
604
 
605
+ // Turn-stall: the CLI is answering control requests but its turn is dead,
606
+ // so `lastInboundAt` never ages and the probe below can never escalate.
607
+ // Only applies while a turn is outstanding and no tool is running.
608
+ if (
609
+ this.turnInFlight &&
610
+ this.pendingToolUses.size === 0 &&
611
+ now - this.lastTurnOutputAt >= TURN_STALL_AFTER_MS
612
+ ) {
613
+ log.warn("claude-adapter", "turn stalled while transport still responsive; treating transport as dead", {
614
+ sessionId: this.sessionId,
615
+ pid: this.stdioProc?.pid,
616
+ turnSilentForMs: now - this.lastTurnOutputAt,
617
+ transportSilentForMs: now - this.lastInboundAt,
618
+ proc: captureProcState(this.stdioProc?.pid),
619
+ });
620
+ this.stopSilenceProbe();
621
+ // Same handoff as the silence probe: relaunch replays the in-flight turn.
622
+ this.notifyStdioDisconnect();
623
+ return;
624
+ }
625
+
510
626
  // An unanswered probe is the actual failure signal: a healthy CLI replies
511
627
  // to control requests promptly even while working.
512
628
  if (this.probeSentAt !== null) {
@@ -609,19 +725,35 @@ export class ClaudeAdapter implements IBackendAdapter {
609
725
  * and satisfying `handleAutoRelaunch`'s PID-liveness guard, which blocks
610
726
  * recovery indefinitely — the exact failure the kill was meant to prevent.
611
727
  * Never throws; the process may exit between any two steps here.
728
+ *
729
+ * Killing the CLI alone leaks its MCP servers. Signals go to the pid, not the
730
+ * tree, so each CLI's 2-3 stdio MCP children (plus, for @playwright/mcp, a
731
+ * headless chromium) are re-parented to init and live on with no client that
732
+ * could ever reach them. Under kill/relaunch churn they accumulate: a
733
+ * production box was found holding 123 such orphans across 11.9 GB, on 23 GB
734
+ * total — which starves the surviving CLIs until they stop answering, causing
735
+ * more kills and more orphans.
612
736
  */
613
737
  private async killWithEscalation(proc: Subprocess): Promise<void> {
738
+ // Snapshot the tree *before* the parent dies. Once it exits its children are
739
+ // re-parented to init and the link back to this session is gone for good.
740
+ const spawned = getDescendants(proc.pid);
741
+
614
742
  try {
615
743
  proc.kill();
616
744
  } catch {
617
- return; // Already gone.
745
+ this.reapOrphans(spawned); // Parent already gone; children may not be.
746
+ return;
618
747
  }
619
748
 
620
749
  const exited = await Promise.race([
621
750
  proc.exited.then(() => true),
622
751
  new Promise<boolean>((resolve) => setTimeout(() => resolve(false), SIGKILL_ESCALATION_MS)),
623
752
  ]);
624
- if (exited || proc.exitCode !== null) return;
753
+ if (exited || proc.exitCode !== null) {
754
+ this.reapOrphans(spawned);
755
+ return;
756
+ }
625
757
 
626
758
  log.warn("claude-adapter", "process ignored SIGTERM; escalating to SIGKILL", {
627
759
  sessionId: this.sessionId,
@@ -633,6 +765,60 @@ export class ClaudeAdapter implements IBackendAdapter {
633
765
  } catch {
634
766
  // Exited between the check and the escalation.
635
767
  }
768
+ this.reapOrphans(spawned);
769
+ }
770
+
771
+ /**
772
+ * Terminate descendants that outlived the CLI they belonged to.
773
+ *
774
+ * Only pids snapshotted as descendants of *this* session's process are
775
+ * touched. A pid could in principle be recycled between the snapshot and the
776
+ * signal, so each one is re-checked against the `comm` recorded at snapshot
777
+ * time and skipped if it no longer matches — a recycled pid is a different
778
+ * process and must not be killed.
779
+ */
780
+ private reapOrphans(spawned: ProcDescendant[]): void {
781
+ if (spawned.length === 0) return;
782
+
783
+ const stillAlive = spawned.filter((d) => {
784
+ try {
785
+ process.kill(d.pid, 0); // Signal 0 = liveness check only.
786
+ } catch {
787
+ return false; // Already reaped with its parent.
788
+ }
789
+ if (!d.comm) return true;
790
+ try {
791
+ return readFileSync(`/proc/${d.pid}/comm`, "utf8").trim() === d.comm;
792
+ } catch {
793
+ return false;
794
+ }
795
+ });
796
+ if (stillAlive.length === 0) return;
797
+
798
+ for (const d of stillAlive) {
799
+ try {
800
+ process.kill(d.pid, "SIGTERM");
801
+ } catch {
802
+ // Exited in the meantime.
803
+ }
804
+ }
805
+ log.info("claude-adapter", "reaped orphaned CLI descendants", {
806
+ sessionId: this.sessionId,
807
+ count: stillAlive.length,
808
+ comms: [...new Set(stillAlive.map((d) => d.comm).filter(Boolean))].slice(0, 5),
809
+ });
810
+
811
+ // Chromium (and npm wrappers mid-install) routinely ignore SIGTERM.
812
+ setTimeout(() => {
813
+ for (const d of stillAlive) {
814
+ try {
815
+ process.kill(d.pid, 0);
816
+ process.kill(d.pid, "SIGKILL");
817
+ } catch {
818
+ // Gone, which is the desired outcome.
819
+ }
820
+ }
821
+ }, SIGKILL_ESCALATION_MS).unref?.();
636
822
  }
637
823
 
638
824
  /**
@@ -878,7 +1064,16 @@ export class ClaudeAdapter implements IBackendAdapter {
878
1064
  });
879
1065
  // Propagate delivery success so the bridge re-queues an undelivered prompt
880
1066
  // (write to a dying stdio pipe) rather than treating send() as delivery.
881
- return this.sendToBackend(ndjson);
1067
+ const delivered = this.sendToBackend(ndjson);
1068
+ if (delivered) {
1069
+ // A turn is now outstanding: arm the turn-stall check. Only armed on
1070
+ // actual delivery, so an undelivered prompt can't start the clock against
1071
+ // a CLI that was never asked to do anything.
1072
+ this.turnInFlight = true;
1073
+ this.lastTurnOutputAt = Date.now();
1074
+ this.pendingToolUses.clear();
1075
+ }
1076
+ return delivered;
882
1077
  }
883
1078
 
884
1079
  /**
@@ -974,6 +1169,9 @@ export class ClaudeAdapter implements IBackendAdapter {
974
1169
  request: { subtype: "interrupt" },
975
1170
  });
976
1171
  this.sendToBackend(ndjson);
1172
+ // The user abandoned the turn, so stop holding it against the CLI.
1173
+ this.turnInFlight = false;
1174
+ this.pendingToolUses.clear();
977
1175
  return true;
978
1176
  }
979
1177
 
@@ -1061,6 +1259,18 @@ export class ClaudeAdapter implements IBackendAdapter {
1061
1259
  this.lastInboundAt = Date.now();
1062
1260
  this.probeSentAt = null;
1063
1261
 
1262
+ // Turn progress is tracked separately: a CLI can keep answering control
1263
+ // requests long after its turn has died (see TURN_STALL_AFTER_MS), so only
1264
+ // frames that represent actual turn work count here.
1265
+ if (isTurnOutput(msg.type)) {
1266
+ this.lastTurnOutputAt = Date.now();
1267
+ }
1268
+ this.trackToolCalls(msg);
1269
+ if (msg.type === "result") {
1270
+ this.turnInFlight = false;
1271
+ this.pendingToolUses.clear();
1272
+ }
1273
+
1064
1274
  // Track activity for idle detection (skip keepalives -- they don't indicate real work)
1065
1275
  if (msg.type !== "keep_alive") {
1066
1276
  this.onActivityUpdate?.();
package/server/index.ts CHANGED
@@ -18,6 +18,7 @@ import { WsBridge } from "./ws-bridge.js";
18
18
  import { SessionStore } from "./session-store.js";
19
19
  import { WorktreeTracker } from "./worktree-tracker.js";
20
20
  import { containerManager } from "./container-manager.js";
21
+ import { startOrphanSweeper } from "./orphan-sweeper.js";
21
22
  import { join } from "node:path";
22
23
  import { COMPANION_HOME } from "./paths.js";
23
24
  import { TerminalManager } from "./terminal-manager.js";
@@ -102,6 +103,14 @@ containerManager.restoreState(CONTAINER_STATE_PATH);
102
103
  // ── Session orchestrator — centralizes lifecycle event wiring ────────────────
103
104
  orchestrator.initialize();
104
105
 
106
+ // ── Reap orphaned MCP servers: once now, then periodically ───────────────────
107
+ // Per-kill reaping cannot cover kills this process did not perform. A previous
108
+ // run that crashed or was OOM-killed orphans every CLI it owned along with those
109
+ // CLIs' MCP children, and a CLI the kernel OOM-kills mid-run does the same. With
110
+ // nothing holding the reference needed to clean them up they survive for the
111
+ // life of the host, holding memory nobody can reclaim.
112
+ startOrphanSweeper();
113
+
105
114
  console.log(`[server] Session persistence: ${sessionStore.directory}`);
106
115
  if (recorder.isGloballyEnabled()) {
107
116
  console.log(`[server] Recording enabled (dir: ${recorder.getRecordingsDir()}, max: ${recorder.getMaxLines()} lines)`);
@@ -0,0 +1,156 @@
1
+ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
2
+
3
+ const mockFindOrphans = vi.hoisted(() =>
4
+ vi.fn((): { pid: number; comm?: string }[] => []),
5
+ );
6
+ vi.mock("./proc-diagnostics.js", async (importOriginal) => ({
7
+ ...(await importOriginal<typeof import("./proc-diagnostics.js")>()),
8
+ findOrphanedMcpProcesses: mockFindOrphans,
9
+ }));
10
+
11
+ import {
12
+ sweepOrphanedMcpProcesses,
13
+ startOrphanSweeper,
14
+ stopOrphanSweeper,
15
+ } from "./orphan-sweeper.js";
16
+
17
+ /**
18
+ * Orphans left behind when companion itself dies. Per-kill reaping cannot cover
19
+ * this: the CLIs and their MCP children are all orphaned at once and nothing
20
+ * remains holding the reference needed to clean them up.
21
+ */
22
+ describe("sweepOrphanedMcpProcesses", () => {
23
+ let killSpy: ReturnType<typeof vi.spyOn>;
24
+ let signalled: { pid: number; sig: unknown }[];
25
+
26
+ beforeEach(() => {
27
+ signalled = [];
28
+ killSpy = vi.spyOn(process, "kill").mockImplementation(((pid: number, sig?: unknown) => {
29
+ if (sig === 0) return true;
30
+ signalled.push({ pid, sig });
31
+ return true;
32
+ }) as unknown as typeof process.kill);
33
+ });
34
+
35
+ afterEach(() => {
36
+ killSpy.mockRestore();
37
+ mockFindOrphans.mockReturnValue([]);
38
+ vi.useRealTimers();
39
+ });
40
+
41
+ it("terminates every orphaned MCP server it finds", () => {
42
+ mockFindOrphans.mockReturnValue([
43
+ { pid: 501, comm: "npm exec" },
44
+ { pid: 502, comm: "chrome" },
45
+ ]);
46
+
47
+ expect(sweepOrphanedMcpProcesses()).toBe(2);
48
+ expect(signalled).toEqual([
49
+ { pid: 501, sig: "SIGTERM" },
50
+ { pid: 502, sig: "SIGTERM" },
51
+ ]);
52
+ });
53
+
54
+ it("escalates to SIGKILL for processes that ignore SIGTERM", () => {
55
+ vi.useFakeTimers();
56
+ mockFindOrphans.mockReturnValue([{ pid: 503 }]);
57
+
58
+ sweepOrphanedMcpProcesses();
59
+ expect(signalled).toEqual([{ pid: 503, sig: "SIGTERM" }]);
60
+
61
+ vi.advanceTimersByTime(5100);
62
+ expect(signalled).toContainEqual({ pid: 503, sig: "SIGKILL" });
63
+ });
64
+
65
+ it("is a no-op when nothing is orphaned", () => {
66
+ mockFindOrphans.mockReturnValue([]);
67
+
68
+ expect(sweepOrphanedMcpProcesses()).toBe(0);
69
+ expect(signalled).toEqual([]);
70
+ });
71
+
72
+ it("never throws when the scan fails, so startup is not blocked", () => {
73
+ mockFindOrphans.mockImplementation(() => {
74
+ throw new Error("/proc unreadable");
75
+ });
76
+
77
+ expect(() => sweepOrphanedMcpProcesses()).not.toThrow();
78
+ expect(sweepOrphanedMcpProcesses()).toBe(0);
79
+ });
80
+
81
+ it("tolerates a process that exits between the scan and the signal", () => {
82
+ killSpy.mockImplementation(((pid: number, sig?: unknown) => {
83
+ if (sig === "SIGTERM" && pid === 504) throw new Error("ESRCH");
84
+ if (sig === 0) return true;
85
+ signalled.push({ pid, sig });
86
+ return true;
87
+ }) as unknown as typeof process.kill);
88
+ mockFindOrphans.mockReturnValue([{ pid: 504 }, { pid: 505 }]);
89
+
90
+ expect(sweepOrphanedMcpProcesses()).toBe(1);
91
+ expect(signalled).toEqual([{ pid: 505, sig: "SIGTERM" }]);
92
+ });
93
+ });
94
+
95
+ /**
96
+ * The periodic schedule. A startup-only sweep leaves a gap: a CLI the kernel
97
+ * OOM-kills mid-run orphans its MCP children with nobody to reap them, and they
98
+ * then accumulate for the rest of the server's uptime — which is how a
99
+ * production box reached 123 orphans across 11.9 GB inside a single run.
100
+ */
101
+ describe("startOrphanSweeper", () => {
102
+ let killSpy: ReturnType<typeof vi.spyOn>;
103
+ let signalled: number[];
104
+
105
+ beforeEach(() => {
106
+ vi.useFakeTimers();
107
+ signalled = [];
108
+ killSpy = vi.spyOn(process, "kill").mockImplementation(((pid: number, sig?: unknown) => {
109
+ if (sig === 0) return true;
110
+ if (sig === "SIGTERM") signalled.push(pid);
111
+ return true;
112
+ }) as unknown as typeof process.kill);
113
+ });
114
+
115
+ afterEach(() => {
116
+ stopOrphanSweeper();
117
+ killSpy.mockRestore();
118
+ mockFindOrphans.mockReturnValue([]);
119
+ vi.useRealTimers();
120
+ });
121
+
122
+ it("sweeps immediately and then on each interval", () => {
123
+ mockFindOrphans.mockReturnValue([{ pid: 601 }]);
124
+
125
+ startOrphanSweeper();
126
+ expect(signalled).toEqual([601]); // startup sweep
127
+
128
+ vi.advanceTimersByTime(600_000);
129
+ expect(signalled).toEqual([601, 601]); // first periodic run
130
+
131
+ vi.advanceTimersByTime(600_000);
132
+ expect(signalled).toEqual([601, 601, 601]);
133
+ });
134
+
135
+ it("stops sweeping once cancelled", () => {
136
+ mockFindOrphans.mockReturnValue([{ pid: 602 }]);
137
+
138
+ const stop = startOrphanSweeper();
139
+ expect(signalled).toEqual([602]);
140
+
141
+ stop();
142
+ vi.advanceTimersByTime(1_800_000);
143
+ expect(signalled).toEqual([602]); // no further sweeps
144
+ });
145
+
146
+ it("replaces the schedule rather than stacking a second one", () => {
147
+ mockFindOrphans.mockReturnValue([{ pid: 603 }]);
148
+
149
+ startOrphanSweeper();
150
+ startOrphanSweeper();
151
+ signalled.length = 0; // ignore the two startup sweeps
152
+
153
+ vi.advanceTimersByTime(600_000);
154
+ expect(signalled).toEqual([603]); // one interval fired, not two
155
+ });
156
+ });
@@ -0,0 +1,113 @@
1
+ /**
2
+ * Startup sweep for MCP servers orphaned by a previous companion run.
3
+ *
4
+ * Each `claude --print` CLI spawns 2-3 stdio MCP children (e.g. @shopify/dev-mcp,
5
+ * @playwright/mcp plus its headless chromium). Signals go to a pid rather than a
6
+ * tree, so when a CLI dies its children are re-parented to init. `ClaudeAdapter`
7
+ * now reaps them on its own kill path, but that only covers kills companion
8
+ * performs while it is running. If companion itself is restarted, crashes or is
9
+ * OOM-killed, every CLI it owned is orphaned together with those children and
10
+ * nothing remains that could clean them up.
11
+ *
12
+ * Left alone they persist for the life of the host. A production box was found
13
+ * holding 123 such processes across 11.9 GB of 23 GB total, which starved the
14
+ * live CLIs of CPU and memory until they stopped answering — producing exactly
15
+ * the stall and wedge symptoms that cause more kills, and so more orphans.
16
+ *
17
+ * Running this once at startup breaks that cycle. An MCP stdio server whose
18
+ * parent is not a `claude` process has no client and can never acquire one, so
19
+ * terminating it is unambiguously safe.
20
+ */
21
+ import { findOrphanedMcpProcesses } from "./proc-diagnostics.js";
22
+ import { log } from "./logger.js";
23
+
24
+ /** Grace before escalating; chromium and npm wrappers often ignore SIGTERM. */
25
+ const SWEEP_SIGKILL_AFTER_MS = Number(process.env.COMPANION_SWEEP_SIGKILL_AFTER_MS) || 5000;
26
+
27
+ /**
28
+ * How often to re-run the sweep while the server is up.
29
+ *
30
+ * A startup sweep alone leaves a gap: a CLI killed by something other than this
31
+ * process — the kernel OOM killer is the common one on a loaded host — orphans
32
+ * its MCP children with nobody to reap them, and they then accumulate for the
33
+ * rest of the server's uptime. That is exactly how a production box reached 123
34
+ * orphans across 11.9 GB inside a single run. The scan is a cheap /proc walk, so
35
+ * running it periodically costs nothing measurable.
36
+ */
37
+ const SWEEP_INTERVAL_MS = Number(process.env.COMPANION_SWEEP_INTERVAL_MS) || 600_000;
38
+
39
+ /**
40
+ * Terminate every MCP server process with no live CLI parent.
41
+ *
42
+ * Never throws: a failure to tidy up must not prevent the server from starting.
43
+ * Returns the number of processes signalled, for logging and tests.
44
+ */
45
+ export function sweepOrphanedMcpProcesses(): number {
46
+ let orphans: ReturnType<typeof findOrphanedMcpProcesses>;
47
+ try {
48
+ orphans = findOrphanedMcpProcesses();
49
+ } catch (err) {
50
+ log.warn("orphan-sweeper", "scan failed; skipping sweep", {
51
+ error: err instanceof Error ? err.message : String(err),
52
+ });
53
+ return 0;
54
+ }
55
+
56
+ if (orphans.length === 0) return 0;
57
+
58
+ const signalled: number[] = [];
59
+ for (const o of orphans) {
60
+ try {
61
+ process.kill(o.pid, "SIGTERM");
62
+ signalled.push(o.pid);
63
+ } catch {
64
+ // Exited between the scan and the signal.
65
+ }
66
+ }
67
+
68
+ if (signalled.length === 0) return 0;
69
+
70
+ log.info("orphan-sweeper", "reaped MCP servers orphaned by a previous run", {
71
+ count: signalled.length,
72
+ comms: [...new Set(orphans.map((o) => o.comm).filter(Boolean))].slice(0, 5),
73
+ });
74
+
75
+ setTimeout(() => {
76
+ for (const pid of signalled) {
77
+ try {
78
+ process.kill(pid, 0);
79
+ process.kill(pid, "SIGKILL");
80
+ } catch {
81
+ // Gone, which is the desired outcome.
82
+ }
83
+ }
84
+ }, SWEEP_SIGKILL_AFTER_MS).unref?.();
85
+
86
+ return signalled.length;
87
+ }
88
+
89
+ let sweepTimer: ReturnType<typeof setInterval> | null = null;
90
+
91
+ /**
92
+ * Sweep at startup, then periodically for as long as the server runs.
93
+ *
94
+ * Returns a stop function, so tests and shutdown paths can clear the timer.
95
+ * Calling this twice replaces the existing schedule rather than stacking a
96
+ * second one.
97
+ */
98
+ export function startOrphanSweeper(): () => void {
99
+ stopOrphanSweeper();
100
+ sweepOrphanedMcpProcesses();
101
+ sweepTimer = setInterval(() => sweepOrphanedMcpProcesses(), SWEEP_INTERVAL_MS);
102
+ // Housekeeping must never hold the process open.
103
+ (sweepTimer as unknown as { unref?: () => void }).unref?.();
104
+ return stopOrphanSweeper;
105
+ }
106
+
107
+ /** Cancel the periodic sweep, if one is scheduled. */
108
+ export function stopOrphanSweeper(): void {
109
+ if (sweepTimer) {
110
+ clearInterval(sweepTimer);
111
+ sweepTimer = null;
112
+ }
113
+ }
@@ -6,6 +6,7 @@ import {
6
6
  getDescendants,
7
7
  hasLiveDescendants,
8
8
  countDescendants,
9
+ findOrphanedMcpProcesses,
9
10
  } from "./proc-diagnostics.js";
10
11
 
11
12
  /**
@@ -208,3 +209,81 @@ describe("getDescendants / hasLiveDescendants", () => {
208
209
  await new Promise((r) => child.on("exit", r));
209
210
  });
210
211
  });
212
+
213
+ /**
214
+ * Orphaned-MCP detection. This drives a sweeper that SIGTERMs whatever it
215
+ * returns, so both directions matter: missing an orphan leaks memory (a
216
+ * production host reached 123 orphans across 11.9 GB), but a false positive
217
+ * kills a live MCP server out from under a working session.
218
+ */
219
+ describe("findOrphanedMcpProcesses", () => {
220
+ const spawned: ReturnType<typeof spawn>[] = [];
221
+
222
+ afterEach(async () => {
223
+ for (const p of spawned.splice(0)) {
224
+ try {
225
+ p.kill("SIGKILL");
226
+ await new Promise((r) => p.on("exit", r));
227
+ } catch {
228
+ // Already gone.
229
+ }
230
+ }
231
+ });
232
+
233
+ it("returns empty on platforms without /proc", () => {
234
+ vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
235
+ expect(findOrphanedMcpProcesses()).toEqual([]);
236
+ });
237
+
238
+ it.runIf(isLinux)("never throws and returns a well-formed list", () => {
239
+ // Runs against the real /proc of whatever host this is, so it must tolerate
240
+ // processes exiting mid-scan — the scan races every other process on the box.
241
+ const orphans = findOrphanedMcpProcesses();
242
+ expect(Array.isArray(orphans)).toBe(true);
243
+ for (const o of orphans) {
244
+ expect(typeof o.pid).toBe("number");
245
+ expect(o.pid).toBeGreaterThan(0);
246
+ }
247
+ });
248
+
249
+ it.runIf(isLinux)("detects a process whose cmdline marks it an MCP server with no CLI parent", async () => {
250
+ // `exec -a` sets argv[0], reproducing the cmdline shape of a CLI-spawned
251
+ // stdio MCP server. Its parent here is the test runner, not a `claude`
252
+ // process, so it is exactly the orphan the sweeper targets.
253
+ const child = spawn("/bin/bash", [
254
+ "-c",
255
+ "exec -a 'npm exec @modelcontextprotocol/server-test' sleep 30",
256
+ ], { stdio: "ignore" });
257
+ spawned.push(child);
258
+ await new Promise((r) => setTimeout(r, 300));
259
+
260
+ const orphans = findOrphanedMcpProcesses();
261
+ expect(orphans.some((o) => o.pid === child.pid)).toBe(true);
262
+ });
263
+
264
+ it.runIf(isLinux)("ignores processes with no MCP marker in their cmdline", async () => {
265
+ // The false-positive guard: an ordinary child must never be swept, or the
266
+ // sweeper would kill unrelated processes on the host.
267
+ const child = spawn("sleep", ["30"], { stdio: "ignore" });
268
+ spawned.push(child);
269
+ await new Promise((r) => setTimeout(r, 300));
270
+
271
+ const orphans = findOrphanedMcpProcesses();
272
+ expect(orphans.some((o) => o.pid === child.pid)).toBe(false);
273
+ });
274
+
275
+ it.runIf(isLinux)("labels findings with comm where readable", async () => {
276
+ const child = spawn("/bin/bash", [
277
+ "-c",
278
+ "exec -a 'mcp-server-label-test' sleep 30",
279
+ ], { stdio: "ignore" });
280
+ spawned.push(child);
281
+ await new Promise((r) => setTimeout(r, 300));
282
+
283
+ const found = findOrphanedMcpProcesses().find((o) => o.pid === child.pid);
284
+ expect(found).toBeDefined();
285
+ // comm is the executable name (truncated to 15 chars by the kernel), not
286
+ // argv[0] — it is a label for the log line, not the match key.
287
+ expect(typeof found?.comm).toBe("string");
288
+ });
289
+ });
@@ -162,6 +162,73 @@ export function isProcAvailable(): boolean {
162
162
  return process.platform === "linux";
163
163
  }
164
164
 
165
+ /** Command-line markers identifying a CLI-spawned stdio MCP server. */
166
+ const MCP_CMDLINE_MARKERS = ["dev-mcp", "playwright/mcp", "mcp-server", "@modelcontextprotocol"];
167
+
168
+ /**
169
+ * Find MCP server processes that no longer belong to a live CLI.
170
+ *
171
+ * Per-kill reaping (see `ClaudeAdapter.reapOrphans`) handles the steady state,
172
+ * but it cannot clean up after the server itself dies: if companion is killed,
173
+ * restarted or OOM-killed, every CLI it owned is orphaned along with that CLI's
174
+ * MCP children, and nothing is left holding the reference needed to reap them.
175
+ * They then sit there for the lifetime of the host, holding memory no one can
176
+ * reclaim — 123 such processes across 11.9 GB were found on a 23 GB production
177
+ * box.
178
+ *
179
+ * An MCP stdio server whose parent is not a `claude` process has no client and
180
+ * can never acquire one, which makes it unambiguously safe to terminate.
181
+ */
182
+ export function findOrphanedMcpProcesses(): ProcDescendant[] {
183
+ if (!isProcAvailable()) return [];
184
+
185
+ const orphans: ProcDescendant[] = [];
186
+ let pids: string[];
187
+ try {
188
+ pids = readdirSync("/proc").filter((n) => /^\d+$/.test(n));
189
+ } catch {
190
+ return [];
191
+ }
192
+
193
+ for (const entry of pids) {
194
+ const pid = Number(entry);
195
+ if (pid === process.pid) continue;
196
+
197
+ let cmdline: string;
198
+ try {
199
+ cmdline = readFileSync(`/proc/${pid}/cmdline`, "utf8").replace(/\0/g, " ");
200
+ } catch {
201
+ continue; // Exited mid-scan.
202
+ }
203
+ if (!MCP_CMDLINE_MARKERS.some((m) => cmdline.includes(m))) continue;
204
+
205
+ // Parent still a live claude process → it has a client; leave it alone.
206
+ let ppid: number | undefined;
207
+ try {
208
+ ppid = Number(readFileSync(`/proc/${pid}/stat`, "utf8").split(" ")[3]);
209
+ } catch {
210
+ continue;
211
+ }
212
+ if (ppid && ppid !== 1) {
213
+ try {
214
+ if (readFileSync(`/proc/${ppid}/cmdline`, "utf8").includes("claude")) continue;
215
+ } catch {
216
+ // Parent vanished between reads — treat as orphaned.
217
+ }
218
+ }
219
+
220
+ const orphan: ProcDescendant = { pid };
221
+ try {
222
+ orphan.comm = readFileSync(`/proc/${pid}/comm`, "utf8").trim();
223
+ } catch {
224
+ // Best-effort label only.
225
+ }
226
+ orphans.push(orphan);
227
+ }
228
+
229
+ return orphans;
230
+ }
231
+
165
232
  /**
166
233
  * Capture kernel state for a pid. Returns `{ error }` rather than throwing when
167
234
  * the platform has no /proc or the process has already exited (the common race:
@@ -8,6 +8,7 @@ import type { Session } from "./ws-bridge-types.js";
8
8
  import { appendHistory } from "./ws-bridge-persist.js";
9
9
  import { validatePermission } from "./ai-validator.js";
10
10
  import { getEffectiveAiValidation } from "./ai-validation-settings.js";
11
+ import { log } from "./logger.js";
11
12
  import { companionBus } from "./event-bus.js";
12
13
 
13
14
  /**
@@ -176,7 +177,15 @@ export function attachCodexAdapterHandlers(
176
177
  session.pendingPermissions.clear();
177
178
  session.backendAdapter = null;
178
179
  deps.persistSession(session);
179
- console.log(`[ws-bridge] Codex adapter disconnected for session ${sessionId}`);
180
+ // Mirrors WsBridge.broadcastCliDisconnected so every path that raises the
181
+ // "CLI disconnected / Reconnect" banner is greppable under one message.
182
+ log.warn("ws-bridge", "UI: CLI disconnected banner shown", {
183
+ sessionId,
184
+ reason: "codex_adapter_disconnected",
185
+ backendType: session.backendType,
186
+ phase: session.stateMachine.phase,
187
+ browsers: session.browserSockets.size,
188
+ });
180
189
  deps.broadcastToBrowsers(session, { type: "cli_disconnected" });
181
190
 
182
191
  // Auto-relaunch if browsers are still connected (don't leave users staring
@@ -1963,6 +1963,39 @@ describe("CLI message routing", () => {
1963
1963
  expect(session.inFlightUserTurn).toBeTruthy();
1964
1964
  });
1965
1965
 
1966
+ /**
1967
+ * The "CLI disconnected / Reconnect" banner is the single most-reported
1968
+ * symptom, so every path that raises it must leave a greppable log line.
1969
+ * Previously the four broadcast sites logged inconsistently — two via
1970
+ * console.log, one describing the cause rather than the broadcast, one
1971
+ * silent — which made post-hoc log analysis miss occurrences entirely.
1972
+ */
1973
+ it("logs a greppable line whenever the CLI-disconnected banner is broadcast", async () => {
1974
+ vi.useFakeTimers();
1975
+ try {
1976
+ const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
1977
+ const cli = makeCliSocket("s1");
1978
+ bridge.handleCLIOpen(cli, "s1");
1979
+ await bridge.handleCLIMessage(cli, makeInitMsg());
1980
+
1981
+ const browser = makeBrowserSocket("s1");
1982
+ bridge.handleBrowserOpen(browser, "s1");
1983
+
1984
+ bridge.handleCLIClose(cli);
1985
+ // Past the disconnect debounce, the banner goes out.
1986
+ await vi.advanceTimersByTimeAsync(20_000);
1987
+
1988
+ const calls = warnSpy.mock.calls.map(([a]) => String(a));
1989
+ const banner = calls.find((l) => l.includes("UI: CLI disconnected banner shown"));
1990
+ expect(banner).toBeDefined();
1991
+ // The reason is what makes a log searchable for *why*, not just *that*.
1992
+ expect(banner).toContain("cli_disconnect_confirmed");
1993
+ expect(banner).toContain("s1");
1994
+ } finally {
1995
+ vi.useRealTimers();
1996
+ }
1997
+ });
1998
+
1966
1999
  it("tool_progress: broadcasts", async () => {
1967
2000
  const msg = JSON.stringify({
1968
2001
  type: "tool_progress",
@@ -687,7 +687,7 @@ export class WsBridge {
687
687
  session.pendingPermissions.clear();
688
688
  session.stateMachine.transition("terminated", "disconnect_confirmed");
689
689
  this.persistSession(session);
690
- this.broadcastToBrowsers(session, { type: "cli_disconnected" });
690
+ this.broadcastCliDisconnected(session, "codex_disconnect_confirmed");
691
691
 
692
692
  // Request auto-relaunch regardless of browser state — proactive
693
693
  // keepalive in the orchestrator ensures headless sessions stay alive.
@@ -931,7 +931,10 @@ export class WsBridge {
931
931
  interruptedMidTurn,
932
932
  });
933
933
  session.stateMachine.transition("terminated", "disconnect_confirmed");
934
- this.broadcastToBrowsers(session, { type: "cli_disconnected" });
934
+ this.broadcastCliDisconnected(session, "cli_disconnect_confirmed", {
935
+ phaseAtClose,
936
+ interruptedMidTurn,
937
+ });
935
938
  for (const [reqId] of session.pendingPermissions) {
936
939
  this.broadcastToBrowsers(session, { type: "permission_cancelled", request_id: reqId });
937
940
  }
@@ -990,8 +993,17 @@ export class WsBridge {
990
993
  if (!backendConnected && !this.disconnectTimers.has(sessionId)) {
991
994
  // Only signal disconnection if we're not within the debounce window
992
995
  // (CLI may be mid-reconnect — avoid UI flap and spurious relaunch)
996
+ // Same banner, different trigger: a browser attached to a session whose
997
+ // backend is already gone. Logged in the same shape so both paths surface
998
+ // under one grep.
999
+ log.warn("ws-bridge", "UI: CLI disconnected banner shown", {
1000
+ sessionId,
1001
+ reason: "backend_dead_on_browser_open",
1002
+ backendType: session.backendType,
1003
+ phase: session.stateMachine.phase,
1004
+ browsers: session.browserSockets.size,
1005
+ });
993
1006
  this.sendToBrowser(ws, { type: "cli_disconnected" });
994
- console.log(`[ws-bridge] Browser connected but backend is dead for session ${sessionId}, requesting relaunch`);
995
1007
  companionBus.emit("session:relaunch-needed", { sessionId });
996
1008
  }
997
1009
  }
@@ -1304,6 +1316,34 @@ export class WsBridge {
1304
1316
  this.broadcastToBrowsers(session, { type: "session_name_update", name });
1305
1317
  }
1306
1318
 
1319
+ /**
1320
+ * Broadcast `cli_disconnected` and record it, in that order, from one place.
1321
+ *
1322
+ * `cli_disconnected` is what puts the "CLI disconnected / Reconnect" banner in
1323
+ * front of the user, so every occurrence must be greppable afterwards — this
1324
+ * is the single most reported symptom and was previously logged inconsistently
1325
+ * across the four call sites (two `console.log`, one `log.warn` describing the
1326
+ * cause rather than the broadcast, one silent).
1327
+ *
1328
+ * Emitting it here means the log line and the banner cannot drift apart: if
1329
+ * the user saw the banner, this line exists.
1330
+ */
1331
+ private broadcastCliDisconnected(
1332
+ session: Session,
1333
+ reason: string,
1334
+ detail: Record<string, unknown> = {},
1335
+ ) {
1336
+ log.warn("ws-bridge", "UI: CLI disconnected banner shown", {
1337
+ sessionId: session.id,
1338
+ reason,
1339
+ backendType: session.backendType,
1340
+ phase: session.stateMachine.phase,
1341
+ browsers: session.browserSockets.size,
1342
+ ...detail,
1343
+ });
1344
+ this.broadcastToBrowsers(session, { type: "cli_disconnected" });
1345
+ }
1346
+
1307
1347
  private broadcastToBrowsers(session: Session, msg: BrowserIncomingMessage) {
1308
1348
  broadcastToBrowsersFn(session, msg, {
1309
1349
  eventBufferLimit: EVENT_BUFFER_LIMIT,