@hellcoder/companion 0.111.2-preview.20260728013354.d1fdce0 → 0.111.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server/claude-adapter.test.ts +299 -0
- package/server/claude-adapter.ts +215 -5
- package/server/index.ts +9 -0
- package/server/orphan-sweeper.test.ts +156 -0
- package/server/orphan-sweeper.ts +113 -0
- package/server/proc-diagnostics.test.ts +79 -0
- package/server/proc-diagnostics.ts +67 -0
- package/server/ws-bridge-codex.ts +10 -1
- package/server/ws-bridge.test.ts +33 -0
- package/server/ws-bridge.ts +43 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hellcoder/companion",
|
|
3
|
-
"version": "0.111.2
|
|
3
|
+
"version": "0.111.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Web UI for launching and interacting with Claude Code agents — Moritz Edition (fork of the-companion)",
|
|
6
6
|
"license": "MIT",
|
|
@@ -28,9 +28,15 @@ vi.mock("node:crypto", () => ({
|
|
|
28
28
|
// Default 0 (childless) preserves the existing tests' behaviour; individual
|
|
29
29
|
// tests override it.
|
|
30
30
|
const mockCountDescendants = vi.hoisted(() => vi.fn(() => 0));
|
|
31
|
+
// getDescendants is stubbed so the orphan-reaping tests can present a known tree
|
|
32
|
+
// without spawning real children. Default empty, matching a childless CLI.
|
|
33
|
+
const mockGetDescendants = vi.hoisted(() =>
|
|
34
|
+
vi.fn((): { pid: number; comm?: string; state?: string }[] => []),
|
|
35
|
+
);
|
|
31
36
|
vi.mock("./proc-diagnostics.js", async (importOriginal) => ({
|
|
32
37
|
...(await importOriginal<typeof import("./proc-diagnostics.js")>()),
|
|
33
38
|
countDescendants: mockCountDescendants,
|
|
39
|
+
getDescendants: mockGetDescendants,
|
|
34
40
|
}));
|
|
35
41
|
|
|
36
42
|
// Settings are stubbed so the wedge-kill and silence-probe switches can be
|
|
@@ -1870,3 +1876,296 @@ describe("stdio silence probe", () => {
|
|
|
1870
1876
|
expect(adapter.isConnected()).toBe(true);
|
|
1871
1877
|
});
|
|
1872
1878
|
});
|
|
1879
|
+
|
|
1880
|
+
/**
|
|
1881
|
+
* Turn-stall — the variant the silence probe structurally cannot catch.
|
|
1882
|
+
*
|
|
1883
|
+
* Reported from production on 0.111.1 (session aa37e7de): the turn died at
|
|
1884
|
+
* 02:10:56 mid-`stream_event` while the CLI kept answering probes. Each reply
|
|
1885
|
+
* refreshes `lastInboundAt` and pushes escalation out another interval — five
|
|
1886
|
+
* probes were answered before the process went silent outright and the existing
|
|
1887
|
+
* probe could fire, landing recovery at 02:29:33. An 18.6-minute hang, nearly
|
|
1888
|
+
* all of it after the turn was already dead. Turn progress has to be tracked
|
|
1889
|
+
* independently of transport liveness.
|
|
1890
|
+
*/
|
|
1891
|
+
describe("stdio turn-stall detection", () => {
|
|
1892
|
+
let adapter: ClaudeAdapter;
|
|
1893
|
+
let disconnectCb: ReturnType<typeof vi.fn>;
|
|
1894
|
+
|
|
1895
|
+
beforeEach(() => {
|
|
1896
|
+
adapter = new ClaudeAdapter("turn-stall-session");
|
|
1897
|
+
disconnectCb = vi.fn();
|
|
1898
|
+
adapter.onDisconnect(disconnectCb as unknown as () => void);
|
|
1899
|
+
mockSettings.silenceProbeEnabled = true;
|
|
1900
|
+
});
|
|
1901
|
+
|
|
1902
|
+
afterEach(() => {
|
|
1903
|
+
mockSettings.silenceProbeEnabled = true;
|
|
1904
|
+
vi.useRealTimers();
|
|
1905
|
+
});
|
|
1906
|
+
|
|
1907
|
+
const controlResponse = (i: number) =>
|
|
1908
|
+
JSON.stringify({
|
|
1909
|
+
type: "control_response",
|
|
1910
|
+
response: { subtype: "mcp_status", request_id: `r-${i}` },
|
|
1911
|
+
}) + "\n";
|
|
1912
|
+
|
|
1913
|
+
const streamEvent = () =>
|
|
1914
|
+
JSON.stringify({
|
|
1915
|
+
type: "stream_event",
|
|
1916
|
+
event: { type: "content_block_delta", index: 0, delta: { type: "thinking_delta", thinking: "" } },
|
|
1917
|
+
session_id: "s",
|
|
1918
|
+
}) + "\n";
|
|
1919
|
+
|
|
1920
|
+
const startTurn = () => adapter.send({ type: "user_message", content: "go" } as never);
|
|
1921
|
+
|
|
1922
|
+
it("fires when the CLI answers control requests but the turn is dead", async () => {
|
|
1923
|
+
vi.useFakeTimers();
|
|
1924
|
+
const { proc, pushStdout } = createMockProc();
|
|
1925
|
+
adapter.attachStdio(proc);
|
|
1926
|
+
startTurn();
|
|
1927
|
+
|
|
1928
|
+
// The turn streams briefly, then dies — exactly the aa37e7de shape.
|
|
1929
|
+
pushStdout(streamEvent());
|
|
1930
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
1931
|
+
|
|
1932
|
+
// ...while the transport stays chatty. This is what pins the silence probe
|
|
1933
|
+
// permanently disarmed, so anything that fires here is the turn-stall check.
|
|
1934
|
+
for (let i = 0; i < 12; i++) {
|
|
1935
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
1936
|
+
pushStdout(controlResponse(i));
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
expect(disconnectCb).toHaveBeenCalledTimes(1);
|
|
1940
|
+
expect(adapter.isConnected()).toBe(false);
|
|
1941
|
+
});
|
|
1942
|
+
|
|
1943
|
+
it("does NOT fire while a tool call is outstanding", async () => {
|
|
1944
|
+
// The critical false-positive guard: a long `Bash` run legitimately emits
|
|
1945
|
+
// nothing for many minutes and must never be torn down.
|
|
1946
|
+
vi.useFakeTimers();
|
|
1947
|
+
const { proc, pushStdout } = createMockProc();
|
|
1948
|
+
adapter.attachStdio(proc);
|
|
1949
|
+
startTurn();
|
|
1950
|
+
|
|
1951
|
+
pushStdout(
|
|
1952
|
+
JSON.stringify({
|
|
1953
|
+
type: "assistant",
|
|
1954
|
+
message: {
|
|
1955
|
+
role: "assistant",
|
|
1956
|
+
model: "m",
|
|
1957
|
+
content: [{ type: "tool_use", id: "tu-1", name: "Bash", input: {} }],
|
|
1958
|
+
},
|
|
1959
|
+
session_id: "s",
|
|
1960
|
+
}) + "\n",
|
|
1961
|
+
);
|
|
1962
|
+
|
|
1963
|
+
// Keep the transport answering so the (separate) silence probe stays out of
|
|
1964
|
+
// it — this test is only about the turn-stall check.
|
|
1965
|
+
for (let i = 0; i < 20; i++) {
|
|
1966
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
1967
|
+
pushStdout(controlResponse(i));
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1970
|
+
expect(disconnectCb).not.toHaveBeenCalled();
|
|
1971
|
+
expect(adapter.isConnected()).toBe(true);
|
|
1972
|
+
});
|
|
1973
|
+
|
|
1974
|
+
it("resumes watching once the tool result comes back", async () => {
|
|
1975
|
+
vi.useFakeTimers();
|
|
1976
|
+
const { proc, pushStdout } = createMockProc();
|
|
1977
|
+
adapter.attachStdio(proc);
|
|
1978
|
+
startTurn();
|
|
1979
|
+
|
|
1980
|
+
pushStdout(
|
|
1981
|
+
JSON.stringify({
|
|
1982
|
+
type: "assistant",
|
|
1983
|
+
message: {
|
|
1984
|
+
role: "assistant",
|
|
1985
|
+
model: "m",
|
|
1986
|
+
content: [{ type: "tool_use", id: "tu-1", name: "Bash", input: {} }],
|
|
1987
|
+
},
|
|
1988
|
+
session_id: "s",
|
|
1989
|
+
}) + "\n",
|
|
1990
|
+
);
|
|
1991
|
+
for (let i = 0; i < 14; i++) {
|
|
1992
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
1993
|
+
pushStdout(controlResponse(i));
|
|
1994
|
+
}
|
|
1995
|
+
expect(disconnectCb).not.toHaveBeenCalled();
|
|
1996
|
+
|
|
1997
|
+
// Tool finished; the model should now be producing output again. It does
|
|
1998
|
+
// not, so the exemption lifts and the stall is caught.
|
|
1999
|
+
pushStdout(
|
|
2000
|
+
JSON.stringify({
|
|
2001
|
+
type: "user",
|
|
2002
|
+
message: { role: "user", content: [{ type: "tool_result", tool_use_id: "tu-1", content: "ok" }] },
|
|
2003
|
+
session_id: "s",
|
|
2004
|
+
}) + "\n",
|
|
2005
|
+
);
|
|
2006
|
+
for (let i = 0; i < 12; i++) {
|
|
2007
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
2008
|
+
pushStdout(controlResponse(100 + i));
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
expect(disconnectCb).toHaveBeenCalledTimes(1);
|
|
2012
|
+
});
|
|
2013
|
+
|
|
2014
|
+
it("does NOT fire on an idle session with no turn outstanding", async () => {
|
|
2015
|
+
// A session simply waiting for the user emits nothing for hours.
|
|
2016
|
+
vi.useFakeTimers();
|
|
2017
|
+
const { proc, pushStdout } = createMockProc();
|
|
2018
|
+
adapter.attachStdio(proc);
|
|
2019
|
+
|
|
2020
|
+
for (let i = 0; i < 12; i++) {
|
|
2021
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
2022
|
+
pushStdout(controlResponse(i));
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
expect(disconnectCb).not.toHaveBeenCalled();
|
|
2026
|
+
expect(adapter.isConnected()).toBe(true);
|
|
2027
|
+
});
|
|
2028
|
+
|
|
2029
|
+
it("does NOT fire while the turn keeps producing output", async () => {
|
|
2030
|
+
vi.useFakeTimers();
|
|
2031
|
+
const { proc, pushStdout } = createMockProc();
|
|
2032
|
+
adapter.attachStdio(proc);
|
|
2033
|
+
startTurn();
|
|
2034
|
+
|
|
2035
|
+
for (let i = 0; i < 12; i++) {
|
|
2036
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
2037
|
+
pushStdout(streamEvent());
|
|
2038
|
+
}
|
|
2039
|
+
|
|
2040
|
+
expect(disconnectCb).not.toHaveBeenCalled();
|
|
2041
|
+
expect(adapter.isConnected()).toBe(true);
|
|
2042
|
+
});
|
|
2043
|
+
|
|
2044
|
+
it("does NOT fire after the turn ends with a result", async () => {
|
|
2045
|
+
vi.useFakeTimers();
|
|
2046
|
+
const { proc, pushStdout } = createMockProc();
|
|
2047
|
+
adapter.attachStdio(proc);
|
|
2048
|
+
startTurn();
|
|
2049
|
+
|
|
2050
|
+
pushStdout(
|
|
2051
|
+
JSON.stringify({ type: "result", subtype: "success", is_error: false, session_id: "s" }) + "\n",
|
|
2052
|
+
);
|
|
2053
|
+
for (let i = 0; i < 12; i++) {
|
|
2054
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
2055
|
+
pushStdout(controlResponse(i));
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
expect(disconnectCb).not.toHaveBeenCalled();
|
|
2059
|
+
expect(adapter.isConnected()).toBe(true);
|
|
2060
|
+
});
|
|
2061
|
+
});
|
|
2062
|
+
|
|
2063
|
+
/**
|
|
2064
|
+
* Orphaned MCP descendants.
|
|
2065
|
+
*
|
|
2066
|
+
* Signals go to a pid, not a tree, so killing a wedged CLI leaves its 2-3 stdio
|
|
2067
|
+
* MCP servers (and, for @playwright/mcp, a headless chromium) re-parented to
|
|
2068
|
+
* init with no client that can ever reach them. Under kill/relaunch churn they
|
|
2069
|
+
* accumulate: a production box was found holding 123 orphans across 11.9 GB of
|
|
2070
|
+
* 23 GB total, starving the surviving CLIs until they stopped answering — which
|
|
2071
|
+
* caused more kills, and more orphans.
|
|
2072
|
+
*/
|
|
2073
|
+
describe("orphaned descendant reaping", () => {
|
|
2074
|
+
let adapter: ClaudeAdapter;
|
|
2075
|
+
let killSpy: ReturnType<typeof vi.spyOn>;
|
|
2076
|
+
let signalled: { pid: number; sig: unknown }[];
|
|
2077
|
+
|
|
2078
|
+
beforeEach(() => {
|
|
2079
|
+
adapter = new ClaudeAdapter("orphan-session");
|
|
2080
|
+
signalled = [];
|
|
2081
|
+
// Simulate a live process table: signal 0 succeeds, real signals recorded.
|
|
2082
|
+
killSpy = vi.spyOn(process, "kill").mockImplementation(((pid: number, sig?: unknown) => {
|
|
2083
|
+
if (sig === 0) return true;
|
|
2084
|
+
signalled.push({ pid, sig });
|
|
2085
|
+
return true;
|
|
2086
|
+
}) as unknown as typeof process.kill);
|
|
2087
|
+
});
|
|
2088
|
+
|
|
2089
|
+
afterEach(() => {
|
|
2090
|
+
killSpy.mockRestore();
|
|
2091
|
+
mockGetDescendants.mockReturnValue([]);
|
|
2092
|
+
vi.useRealTimers();
|
|
2093
|
+
});
|
|
2094
|
+
|
|
2095
|
+
/** Drive the wedge path far enough to trigger killWithEscalation. */
|
|
2096
|
+
const wedgeAndKill = async (endStdout: () => void) => {
|
|
2097
|
+
endStdout();
|
|
2098
|
+
await vi.advanceTimersByTimeAsync(11_000); // past the stdout-close grace
|
|
2099
|
+
};
|
|
2100
|
+
|
|
2101
|
+
it("terminates descendants that outlived the CLI", async () => {
|
|
2102
|
+
vi.useFakeTimers();
|
|
2103
|
+
mockGetDescendants.mockReturnValue([{ pid: 99001 }, { pid: 99002 }]);
|
|
2104
|
+
const { proc, endStdout } = createMockProc();
|
|
2105
|
+
adapter.attachStdio(proc);
|
|
2106
|
+
|
|
2107
|
+
await wedgeAndKill(endStdout);
|
|
2108
|
+
|
|
2109
|
+
expect(proc.kill).toHaveBeenCalled();
|
|
2110
|
+
expect(signalled.map((s) => s.pid).sort()).toEqual([99001, 99002]);
|
|
2111
|
+
expect(signalled.every((s) => s.sig === "SIGTERM")).toBe(true);
|
|
2112
|
+
});
|
|
2113
|
+
|
|
2114
|
+
it("escalates to SIGKILL for descendants that ignore SIGTERM", async () => {
|
|
2115
|
+
// Chromium routinely ignores SIGTERM.
|
|
2116
|
+
vi.useFakeTimers();
|
|
2117
|
+
mockGetDescendants.mockReturnValue([{ pid: 99003 }]);
|
|
2118
|
+
const { proc, endStdout } = createMockProc();
|
|
2119
|
+
adapter.attachStdio(proc);
|
|
2120
|
+
|
|
2121
|
+
await wedgeAndKill(endStdout);
|
|
2122
|
+
expect(signalled).toEqual([{ pid: 99003, sig: "SIGTERM" }]);
|
|
2123
|
+
|
|
2124
|
+
await vi.advanceTimersByTimeAsync(2100);
|
|
2125
|
+
expect(signalled).toContainEqual({ pid: 99003, sig: "SIGKILL" });
|
|
2126
|
+
});
|
|
2127
|
+
|
|
2128
|
+
it("does NOT signal a pid whose identity can no longer be confirmed", async () => {
|
|
2129
|
+
// The recycled-pid guard. A snapshotted descendant carrying a `comm` is only
|
|
2130
|
+
// signalled if /proc still reports that same comm — otherwise the pid has
|
|
2131
|
+
// been reused by an unrelated process and must be left alone. Here /proc has
|
|
2132
|
+
// no such entry at all, so it must be skipped.
|
|
2133
|
+
vi.useFakeTimers();
|
|
2134
|
+
mockGetDescendants.mockReturnValue([{ pid: 99004, comm: "npm exec" }]);
|
|
2135
|
+
const { proc, endStdout } = createMockProc();
|
|
2136
|
+
adapter.attachStdio(proc);
|
|
2137
|
+
|
|
2138
|
+
await wedgeAndKill(endStdout);
|
|
2139
|
+
|
|
2140
|
+
expect(proc.kill).toHaveBeenCalled();
|
|
2141
|
+
expect(signalled).toEqual([]);
|
|
2142
|
+
});
|
|
2143
|
+
|
|
2144
|
+
it("skips descendants that already exited with their parent", async () => {
|
|
2145
|
+
vi.useFakeTimers();
|
|
2146
|
+
killSpy.mockImplementation(((pid: number, sig?: unknown) => {
|
|
2147
|
+
if (sig === 0) throw new Error("ESRCH"); // nothing alive
|
|
2148
|
+
signalled.push({ pid, sig });
|
|
2149
|
+
return true;
|
|
2150
|
+
}) as unknown as typeof process.kill);
|
|
2151
|
+
mockGetDescendants.mockReturnValue([{ pid: 99005 }, { pid: 99006 }]);
|
|
2152
|
+
const { proc, endStdout } = createMockProc();
|
|
2153
|
+
adapter.attachStdio(proc);
|
|
2154
|
+
|
|
2155
|
+
await wedgeAndKill(endStdout);
|
|
2156
|
+
|
|
2157
|
+
expect(signalled).toEqual([]);
|
|
2158
|
+
});
|
|
2159
|
+
|
|
2160
|
+
it("is a no-op for a childless CLI", async () => {
|
|
2161
|
+
vi.useFakeTimers();
|
|
2162
|
+
mockGetDescendants.mockReturnValue([]);
|
|
2163
|
+
const { proc, endStdout } = createMockProc();
|
|
2164
|
+
adapter.attachStdio(proc);
|
|
2165
|
+
|
|
2166
|
+
await wedgeAndKill(endStdout);
|
|
2167
|
+
|
|
2168
|
+
expect(proc.kill).toHaveBeenCalled();
|
|
2169
|
+
expect(signalled).toEqual([]);
|
|
2170
|
+
});
|
|
2171
|
+
});
|
package/server/claude-adapter.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { randomUUID } from "node:crypto";
|
|
13
|
-
import { mkdirSync, writeFileSync } from "node:fs";
|
|
13
|
+
import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
|
|
14
14
|
import { join, basename } from "node:path";
|
|
15
15
|
import { log } from "./logger.js";
|
|
16
16
|
import type { ServerWebSocket, Subprocess } from "bun";
|
|
@@ -46,7 +46,7 @@ import type {
|
|
|
46
46
|
import type { SocketData } from "./ws-bridge-types.js";
|
|
47
47
|
import type { PendingControlRequest } from "./ws-bridge-types.js";
|
|
48
48
|
import type { RecorderManager } from "./recorder.js";
|
|
49
|
-
import { captureProcState, hasLiveDescendants, countDescendants } from "./proc-diagnostics.js";
|
|
49
|
+
import { captureProcState, hasLiveDescendants, countDescendants, getDescendants, type ProcDescendant } from "./proc-diagnostics.js";
|
|
50
50
|
import { getSettings } from "./settings-manager.js";
|
|
51
51
|
import { parseNDJSON, isDuplicateCLIMessage } from "./ws-bridge-cli-ingest.js";
|
|
52
52
|
import type { CLIDedupState } from "./ws-bridge-cli-ingest.js";
|
|
@@ -129,9 +129,71 @@ const SILENCE_PROBE_AFTER_MS = Number(process.env.COMPANION_SILENCE_PROBE_AFTER_
|
|
|
129
129
|
const SILENCE_PROBE_TIMEOUT_MS = Number(process.env.COMPANION_SILENCE_PROBE_TIMEOUT_MS) || 60_000;
|
|
130
130
|
const SILENCE_CHECK_INTERVAL_MS = Number(process.env.COMPANION_SILENCE_CHECK_INTERVAL_MS) || 15_000;
|
|
131
131
|
|
|
132
|
+
/**
|
|
133
|
+
* Turn-stall threshold.
|
|
134
|
+
*
|
|
135
|
+
* The silence probe above keys on *total* inbound silence, which recovers a
|
|
136
|
+
* dead turn far later than it needs to — and only if the CLI eventually stops
|
|
137
|
+
* answering control requests as well.
|
|
138
|
+
*
|
|
139
|
+
* Observed on 0.111.1 (session aa37e7de): turn output stopped at 02:10:56
|
|
140
|
+
* mid-`stream_event`, but the CLI kept replying to probes. Every reply refreshes
|
|
141
|
+
* `lastInboundAt`, so escalation is pushed out another full interval each time —
|
|
142
|
+
* five probes were answered before the process finally went silent outright and
|
|
143
|
+
* the probe could fire. Recovery landed at 02:29:33: an **18.6-minute** hang for
|
|
144
|
+
* the user, nearly all of it spent watching a turn that was already dead.
|
|
145
|
+
*
|
|
146
|
+
* The probe is doing its job; it is just asking the wrong question. Transport
|
|
147
|
+
* liveness and turn progress are independent, and a CLI that answers pings
|
|
148
|
+
* forever while its turn is dead would postpone escalation forever.
|
|
149
|
+
*
|
|
150
|
+
* So track turn progress separately from transport liveness. Only substantive
|
|
151
|
+
* turn frames (`stream_event`, `assistant`, `user`, `result`, `tool_progress`)
|
|
152
|
+
* count — control and keepalive traffic does not.
|
|
153
|
+
*
|
|
154
|
+
* False positives are the thing to fear (every previous timer in this file was
|
|
155
|
+
* too aggressive), so this only applies while a turn is actually outstanding,
|
|
156
|
+
* and never while a tool call is in flight: a long `Bash` run legitimately
|
|
157
|
+
* emits nothing for many minutes. The default is deliberately well beyond any
|
|
158
|
+
* plausible model think-time.
|
|
159
|
+
*/
|
|
160
|
+
const TURN_STALL_AFTER_MS = Number(process.env.COMPANION_TURN_STALL_AFTER_MS) || 300_000;
|
|
161
|
+
|
|
132
162
|
/** How often to re-check the descendant count while waiting for teardown. */
|
|
133
163
|
const TEARDOWN_POLL_MS = Number(process.env.COMPANION_TEARDOWN_POLL_MS) || 500;
|
|
134
164
|
|
|
165
|
+
/**
|
|
166
|
+
* Does this frame type represent the turn actually making progress?
|
|
167
|
+
*
|
|
168
|
+
* `control_response` and `keep_alive` are excluded on purpose — they are the
|
|
169
|
+
* frames a stalled-but-responsive CLI keeps sending. `system` is excluded for
|
|
170
|
+
* the same reason it is excluded from `lastInboundWasResult`: it carries status
|
|
171
|
+
* and keepalive chatter, not turn work.
|
|
172
|
+
*/
|
|
173
|
+
function isTurnOutput(type: string): boolean {
|
|
174
|
+
return (
|
|
175
|
+
type === "stream_event" ||
|
|
176
|
+
type === "assistant" ||
|
|
177
|
+
type === "user" ||
|
|
178
|
+
type === "result" ||
|
|
179
|
+
type === "tool_progress"
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Extract `tool_use` / `tool_result` ids from a message's content blocks. */
|
|
184
|
+
function toolBlockIds(content: unknown): { uses: string[]; results: string[] } {
|
|
185
|
+
const uses: string[] = [];
|
|
186
|
+
const results: string[] = [];
|
|
187
|
+
if (!Array.isArray(content)) return { uses, results };
|
|
188
|
+
for (const block of content) {
|
|
189
|
+
if (!block || typeof block !== "object") continue;
|
|
190
|
+
const b = block as { type?: string; id?: string; tool_use_id?: string };
|
|
191
|
+
if (b.type === "tool_use" && typeof b.id === "string") uses.push(b.id);
|
|
192
|
+
if (b.type === "tool_result" && typeof b.tool_use_id === "string") results.push(b.tool_use_id);
|
|
193
|
+
}
|
|
194
|
+
return { uses, results };
|
|
195
|
+
}
|
|
196
|
+
|
|
135
197
|
|
|
136
198
|
// --- Claude Code Adapter ------------------------------------------------------
|
|
137
199
|
|
|
@@ -173,6 +235,20 @@ export class ClaudeAdapter implements IBackendAdapter {
|
|
|
173
235
|
/** When the outstanding silence probe was sent, or null if none is in flight. */
|
|
174
236
|
private probeSentAt: number | null = null;
|
|
175
237
|
private silenceTimer: ReturnType<typeof setInterval> | null = null;
|
|
238
|
+
/**
|
|
239
|
+
* Timestamp of the last substantive *turn* frame, for the turn-stall check.
|
|
240
|
+
* Deliberately narrower than `lastInboundAt`: control_response and keep_alive
|
|
241
|
+
* prove the transport is alive but say nothing about the turn making progress.
|
|
242
|
+
*/
|
|
243
|
+
private lastTurnOutputAt = Date.now();
|
|
244
|
+
/** True while a user turn is outstanding — sent to the CLI, no `result` yet. */
|
|
245
|
+
private turnInFlight = false;
|
|
246
|
+
/**
|
|
247
|
+
* `tool_use` ids emitted by the model that have no matching `tool_result` yet.
|
|
248
|
+
* A long-running tool produces no turn output for minutes at a time and must
|
|
249
|
+
* never be mistaken for a stall.
|
|
250
|
+
*/
|
|
251
|
+
private pendingToolUses = new Set<string>();
|
|
176
252
|
|
|
177
253
|
// Callbacks registered by the bridge via on*() methods
|
|
178
254
|
private browserMessageCb: ((msg: BrowserIncomingMessage) => void) | null = null;
|
|
@@ -488,6 +564,11 @@ export class ClaudeAdapter implements IBackendAdapter {
|
|
|
488
564
|
this.stopSilenceProbe();
|
|
489
565
|
this.lastInboundAt = Date.now();
|
|
490
566
|
this.probeSentAt = null;
|
|
567
|
+
// A relaunched transport starts with a clean slate: the bridge replays the
|
|
568
|
+
// in-flight turn, which re-arms turnInFlight via handleOutgoingUserMessage.
|
|
569
|
+
this.lastTurnOutputAt = Date.now();
|
|
570
|
+
this.turnInFlight = false;
|
|
571
|
+
this.pendingToolUses.clear();
|
|
491
572
|
this.silenceTimer = setInterval(() => this.checkSilence(), SILENCE_CHECK_INTERVAL_MS);
|
|
492
573
|
// Never hold the process open for a diagnostic timer.
|
|
493
574
|
(this.silenceTimer as unknown as { unref?: () => void }).unref?.();
|
|
@@ -501,12 +582,47 @@ export class ClaudeAdapter implements IBackendAdapter {
|
|
|
501
582
|
this.probeSentAt = null;
|
|
502
583
|
}
|
|
503
584
|
|
|
585
|
+
/**
|
|
586
|
+
* Maintain the outstanding-tool set so a long tool call is never mistaken for
|
|
587
|
+
* a stalled turn. The model announces work with `tool_use` blocks on an
|
|
588
|
+
* `assistant` frame; the CLI echoes the matching `tool_result` blocks back on
|
|
589
|
+
* a `user` frame when it finishes.
|
|
590
|
+
*/
|
|
591
|
+
private trackToolCalls(msg: CLIMessage): void {
|
|
592
|
+
if (msg.type !== "assistant" && msg.type !== "user") return;
|
|
593
|
+
const content = (msg as { message?: { content?: unknown } }).message?.content;
|
|
594
|
+
const { uses, results } = toolBlockIds(content);
|
|
595
|
+
for (const id of uses) this.pendingToolUses.add(id);
|
|
596
|
+
for (const id of results) this.pendingToolUses.delete(id);
|
|
597
|
+
}
|
|
598
|
+
|
|
504
599
|
private checkSilence(): void {
|
|
505
600
|
if (this.transportKind !== "stdio" || !this.stdioConnected) return;
|
|
506
601
|
if (getSettings().silenceProbeEnabled === false) return;
|
|
507
602
|
|
|
508
603
|
const now = Date.now();
|
|
509
604
|
|
|
605
|
+
// Turn-stall: the CLI is answering control requests but its turn is dead,
|
|
606
|
+
// so `lastInboundAt` never ages and the probe below can never escalate.
|
|
607
|
+
// Only applies while a turn is outstanding and no tool is running.
|
|
608
|
+
if (
|
|
609
|
+
this.turnInFlight &&
|
|
610
|
+
this.pendingToolUses.size === 0 &&
|
|
611
|
+
now - this.lastTurnOutputAt >= TURN_STALL_AFTER_MS
|
|
612
|
+
) {
|
|
613
|
+
log.warn("claude-adapter", "turn stalled while transport still responsive; treating transport as dead", {
|
|
614
|
+
sessionId: this.sessionId,
|
|
615
|
+
pid: this.stdioProc?.pid,
|
|
616
|
+
turnSilentForMs: now - this.lastTurnOutputAt,
|
|
617
|
+
transportSilentForMs: now - this.lastInboundAt,
|
|
618
|
+
proc: captureProcState(this.stdioProc?.pid),
|
|
619
|
+
});
|
|
620
|
+
this.stopSilenceProbe();
|
|
621
|
+
// Same handoff as the silence probe: relaunch replays the in-flight turn.
|
|
622
|
+
this.notifyStdioDisconnect();
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
|
|
510
626
|
// An unanswered probe is the actual failure signal: a healthy CLI replies
|
|
511
627
|
// to control requests promptly even while working.
|
|
512
628
|
if (this.probeSentAt !== null) {
|
|
@@ -609,19 +725,35 @@ export class ClaudeAdapter implements IBackendAdapter {
|
|
|
609
725
|
* and satisfying `handleAutoRelaunch`'s PID-liveness guard, which blocks
|
|
610
726
|
* recovery indefinitely — the exact failure the kill was meant to prevent.
|
|
611
727
|
* Never throws; the process may exit between any two steps here.
|
|
728
|
+
*
|
|
729
|
+
* Killing the CLI alone leaks its MCP servers. Signals go to the pid, not the
|
|
730
|
+
* tree, so each CLI's 2-3 stdio MCP children (plus, for @playwright/mcp, a
|
|
731
|
+
* headless chromium) are re-parented to init and live on with no client that
|
|
732
|
+
* could ever reach them. Under kill/relaunch churn they accumulate: a
|
|
733
|
+
* production box was found holding 123 such orphans across 11.9 GB, on 23 GB
|
|
734
|
+
* total — which starves the surviving CLIs until they stop answering, causing
|
|
735
|
+
* more kills and more orphans.
|
|
612
736
|
*/
|
|
613
737
|
private async killWithEscalation(proc: Subprocess): Promise<void> {
|
|
738
|
+
// Snapshot the tree *before* the parent dies. Once it exits its children are
|
|
739
|
+
// re-parented to init and the link back to this session is gone for good.
|
|
740
|
+
const spawned = getDescendants(proc.pid);
|
|
741
|
+
|
|
614
742
|
try {
|
|
615
743
|
proc.kill();
|
|
616
744
|
} catch {
|
|
617
|
-
|
|
745
|
+
this.reapOrphans(spawned); // Parent already gone; children may not be.
|
|
746
|
+
return;
|
|
618
747
|
}
|
|
619
748
|
|
|
620
749
|
const exited = await Promise.race([
|
|
621
750
|
proc.exited.then(() => true),
|
|
622
751
|
new Promise<boolean>((resolve) => setTimeout(() => resolve(false), SIGKILL_ESCALATION_MS)),
|
|
623
752
|
]);
|
|
624
|
-
if (exited || proc.exitCode !== null)
|
|
753
|
+
if (exited || proc.exitCode !== null) {
|
|
754
|
+
this.reapOrphans(spawned);
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
625
757
|
|
|
626
758
|
log.warn("claude-adapter", "process ignored SIGTERM; escalating to SIGKILL", {
|
|
627
759
|
sessionId: this.sessionId,
|
|
@@ -633,6 +765,60 @@ export class ClaudeAdapter implements IBackendAdapter {
|
|
|
633
765
|
} catch {
|
|
634
766
|
// Exited between the check and the escalation.
|
|
635
767
|
}
|
|
768
|
+
this.reapOrphans(spawned);
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/**
|
|
772
|
+
* Terminate descendants that outlived the CLI they belonged to.
|
|
773
|
+
*
|
|
774
|
+
* Only pids snapshotted as descendants of *this* session's process are
|
|
775
|
+
* touched. A pid could in principle be recycled between the snapshot and the
|
|
776
|
+
* signal, so each one is re-checked against the `comm` recorded at snapshot
|
|
777
|
+
* time and skipped if it no longer matches — a recycled pid is a different
|
|
778
|
+
* process and must not be killed.
|
|
779
|
+
*/
|
|
780
|
+
private reapOrphans(spawned: ProcDescendant[]): void {
|
|
781
|
+
if (spawned.length === 0) return;
|
|
782
|
+
|
|
783
|
+
const stillAlive = spawned.filter((d) => {
|
|
784
|
+
try {
|
|
785
|
+
process.kill(d.pid, 0); // Signal 0 = liveness check only.
|
|
786
|
+
} catch {
|
|
787
|
+
return false; // Already reaped with its parent.
|
|
788
|
+
}
|
|
789
|
+
if (!d.comm) return true;
|
|
790
|
+
try {
|
|
791
|
+
return readFileSync(`/proc/${d.pid}/comm`, "utf8").trim() === d.comm;
|
|
792
|
+
} catch {
|
|
793
|
+
return false;
|
|
794
|
+
}
|
|
795
|
+
});
|
|
796
|
+
if (stillAlive.length === 0) return;
|
|
797
|
+
|
|
798
|
+
for (const d of stillAlive) {
|
|
799
|
+
try {
|
|
800
|
+
process.kill(d.pid, "SIGTERM");
|
|
801
|
+
} catch {
|
|
802
|
+
// Exited in the meantime.
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
log.info("claude-adapter", "reaped orphaned CLI descendants", {
|
|
806
|
+
sessionId: this.sessionId,
|
|
807
|
+
count: stillAlive.length,
|
|
808
|
+
comms: [...new Set(stillAlive.map((d) => d.comm).filter(Boolean))].slice(0, 5),
|
|
809
|
+
});
|
|
810
|
+
|
|
811
|
+
// Chromium (and npm wrappers mid-install) routinely ignore SIGTERM.
|
|
812
|
+
setTimeout(() => {
|
|
813
|
+
for (const d of stillAlive) {
|
|
814
|
+
try {
|
|
815
|
+
process.kill(d.pid, 0);
|
|
816
|
+
process.kill(d.pid, "SIGKILL");
|
|
817
|
+
} catch {
|
|
818
|
+
// Gone, which is the desired outcome.
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
}, SIGKILL_ESCALATION_MS).unref?.();
|
|
636
822
|
}
|
|
637
823
|
|
|
638
824
|
/**
|
|
@@ -878,7 +1064,16 @@ export class ClaudeAdapter implements IBackendAdapter {
|
|
|
878
1064
|
});
|
|
879
1065
|
// Propagate delivery success so the bridge re-queues an undelivered prompt
|
|
880
1066
|
// (write to a dying stdio pipe) rather than treating send() as delivery.
|
|
881
|
-
|
|
1067
|
+
const delivered = this.sendToBackend(ndjson);
|
|
1068
|
+
if (delivered) {
|
|
1069
|
+
// A turn is now outstanding: arm the turn-stall check. Only armed on
|
|
1070
|
+
// actual delivery, so an undelivered prompt can't start the clock against
|
|
1071
|
+
// a CLI that was never asked to do anything.
|
|
1072
|
+
this.turnInFlight = true;
|
|
1073
|
+
this.lastTurnOutputAt = Date.now();
|
|
1074
|
+
this.pendingToolUses.clear();
|
|
1075
|
+
}
|
|
1076
|
+
return delivered;
|
|
882
1077
|
}
|
|
883
1078
|
|
|
884
1079
|
/**
|
|
@@ -974,6 +1169,9 @@ export class ClaudeAdapter implements IBackendAdapter {
|
|
|
974
1169
|
request: { subtype: "interrupt" },
|
|
975
1170
|
});
|
|
976
1171
|
this.sendToBackend(ndjson);
|
|
1172
|
+
// The user abandoned the turn, so stop holding it against the CLI.
|
|
1173
|
+
this.turnInFlight = false;
|
|
1174
|
+
this.pendingToolUses.clear();
|
|
977
1175
|
return true;
|
|
978
1176
|
}
|
|
979
1177
|
|
|
@@ -1061,6 +1259,18 @@ export class ClaudeAdapter implements IBackendAdapter {
|
|
|
1061
1259
|
this.lastInboundAt = Date.now();
|
|
1062
1260
|
this.probeSentAt = null;
|
|
1063
1261
|
|
|
1262
|
+
// Turn progress is tracked separately: a CLI can keep answering control
|
|
1263
|
+
// requests long after its turn has died (see TURN_STALL_AFTER_MS), so only
|
|
1264
|
+
// frames that represent actual turn work count here.
|
|
1265
|
+
if (isTurnOutput(msg.type)) {
|
|
1266
|
+
this.lastTurnOutputAt = Date.now();
|
|
1267
|
+
}
|
|
1268
|
+
this.trackToolCalls(msg);
|
|
1269
|
+
if (msg.type === "result") {
|
|
1270
|
+
this.turnInFlight = false;
|
|
1271
|
+
this.pendingToolUses.clear();
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1064
1274
|
// Track activity for idle detection (skip keepalives -- they don't indicate real work)
|
|
1065
1275
|
if (msg.type !== "keep_alive") {
|
|
1066
1276
|
this.onActivityUpdate?.();
|
package/server/index.ts
CHANGED
|
@@ -18,6 +18,7 @@ import { WsBridge } from "./ws-bridge.js";
|
|
|
18
18
|
import { SessionStore } from "./session-store.js";
|
|
19
19
|
import { WorktreeTracker } from "./worktree-tracker.js";
|
|
20
20
|
import { containerManager } from "./container-manager.js";
|
|
21
|
+
import { startOrphanSweeper } from "./orphan-sweeper.js";
|
|
21
22
|
import { join } from "node:path";
|
|
22
23
|
import { COMPANION_HOME } from "./paths.js";
|
|
23
24
|
import { TerminalManager } from "./terminal-manager.js";
|
|
@@ -102,6 +103,14 @@ containerManager.restoreState(CONTAINER_STATE_PATH);
|
|
|
102
103
|
// ── Session orchestrator — centralizes lifecycle event wiring ────────────────
|
|
103
104
|
orchestrator.initialize();
|
|
104
105
|
|
|
106
|
+
// ── Reap orphaned MCP servers: once now, then periodically ───────────────────
|
|
107
|
+
// Per-kill reaping cannot cover kills this process did not perform. A previous
|
|
108
|
+
// run that crashed or was OOM-killed orphans every CLI it owned along with those
|
|
109
|
+
// CLIs' MCP children, and a CLI the kernel OOM-kills mid-run does the same. With
|
|
110
|
+
// nothing holding the reference needed to clean them up they survive for the
|
|
111
|
+
// life of the host, holding memory nobody can reclaim.
|
|
112
|
+
startOrphanSweeper();
|
|
113
|
+
|
|
105
114
|
console.log(`[server] Session persistence: ${sessionStore.directory}`);
|
|
106
115
|
if (recorder.isGloballyEnabled()) {
|
|
107
116
|
console.log(`[server] Recording enabled (dir: ${recorder.getRecordingsDir()}, max: ${recorder.getMaxLines()} lines)`);
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
2
|
+
|
|
3
|
+
const mockFindOrphans = vi.hoisted(() =>
|
|
4
|
+
vi.fn((): { pid: number; comm?: string }[] => []),
|
|
5
|
+
);
|
|
6
|
+
vi.mock("./proc-diagnostics.js", async (importOriginal) => ({
|
|
7
|
+
...(await importOriginal<typeof import("./proc-diagnostics.js")>()),
|
|
8
|
+
findOrphanedMcpProcesses: mockFindOrphans,
|
|
9
|
+
}));
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
sweepOrphanedMcpProcesses,
|
|
13
|
+
startOrphanSweeper,
|
|
14
|
+
stopOrphanSweeper,
|
|
15
|
+
} from "./orphan-sweeper.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Orphans left behind when companion itself dies. Per-kill reaping cannot cover
|
|
19
|
+
* this: the CLIs and their MCP children are all orphaned at once and nothing
|
|
20
|
+
* remains holding the reference needed to clean them up.
|
|
21
|
+
*/
|
|
22
|
+
describe("sweepOrphanedMcpProcesses", () => {
|
|
23
|
+
let killSpy: ReturnType<typeof vi.spyOn>;
|
|
24
|
+
let signalled: { pid: number; sig: unknown }[];
|
|
25
|
+
|
|
26
|
+
beforeEach(() => {
|
|
27
|
+
signalled = [];
|
|
28
|
+
killSpy = vi.spyOn(process, "kill").mockImplementation(((pid: number, sig?: unknown) => {
|
|
29
|
+
if (sig === 0) return true;
|
|
30
|
+
signalled.push({ pid, sig });
|
|
31
|
+
return true;
|
|
32
|
+
}) as unknown as typeof process.kill);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
killSpy.mockRestore();
|
|
37
|
+
mockFindOrphans.mockReturnValue([]);
|
|
38
|
+
vi.useRealTimers();
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("terminates every orphaned MCP server it finds", () => {
|
|
42
|
+
mockFindOrphans.mockReturnValue([
|
|
43
|
+
{ pid: 501, comm: "npm exec" },
|
|
44
|
+
{ pid: 502, comm: "chrome" },
|
|
45
|
+
]);
|
|
46
|
+
|
|
47
|
+
expect(sweepOrphanedMcpProcesses()).toBe(2);
|
|
48
|
+
expect(signalled).toEqual([
|
|
49
|
+
{ pid: 501, sig: "SIGTERM" },
|
|
50
|
+
{ pid: 502, sig: "SIGTERM" },
|
|
51
|
+
]);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("escalates to SIGKILL for processes that ignore SIGTERM", () => {
|
|
55
|
+
vi.useFakeTimers();
|
|
56
|
+
mockFindOrphans.mockReturnValue([{ pid: 503 }]);
|
|
57
|
+
|
|
58
|
+
sweepOrphanedMcpProcesses();
|
|
59
|
+
expect(signalled).toEqual([{ pid: 503, sig: "SIGTERM" }]);
|
|
60
|
+
|
|
61
|
+
vi.advanceTimersByTime(5100);
|
|
62
|
+
expect(signalled).toContainEqual({ pid: 503, sig: "SIGKILL" });
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it("is a no-op when nothing is orphaned", () => {
|
|
66
|
+
mockFindOrphans.mockReturnValue([]);
|
|
67
|
+
|
|
68
|
+
expect(sweepOrphanedMcpProcesses()).toBe(0);
|
|
69
|
+
expect(signalled).toEqual([]);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("never throws when the scan fails, so startup is not blocked", () => {
|
|
73
|
+
mockFindOrphans.mockImplementation(() => {
|
|
74
|
+
throw new Error("/proc unreadable");
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
expect(() => sweepOrphanedMcpProcesses()).not.toThrow();
|
|
78
|
+
expect(sweepOrphanedMcpProcesses()).toBe(0);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("tolerates a process that exits between the scan and the signal", () => {
|
|
82
|
+
killSpy.mockImplementation(((pid: number, sig?: unknown) => {
|
|
83
|
+
if (sig === "SIGTERM" && pid === 504) throw new Error("ESRCH");
|
|
84
|
+
if (sig === 0) return true;
|
|
85
|
+
signalled.push({ pid, sig });
|
|
86
|
+
return true;
|
|
87
|
+
}) as unknown as typeof process.kill);
|
|
88
|
+
mockFindOrphans.mockReturnValue([{ pid: 504 }, { pid: 505 }]);
|
|
89
|
+
|
|
90
|
+
expect(sweepOrphanedMcpProcesses()).toBe(1);
|
|
91
|
+
expect(signalled).toEqual([{ pid: 505, sig: "SIGTERM" }]);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The periodic schedule. A startup-only sweep leaves a gap: a CLI the kernel
|
|
97
|
+
* OOM-kills mid-run orphans its MCP children with nobody to reap them, and they
|
|
98
|
+
* then accumulate for the rest of the server's uptime — which is how a
|
|
99
|
+
* production box reached 123 orphans across 11.9 GB inside a single run.
|
|
100
|
+
*/
|
|
101
|
+
describe("startOrphanSweeper", () => {
|
|
102
|
+
let killSpy: ReturnType<typeof vi.spyOn>;
|
|
103
|
+
let signalled: number[];
|
|
104
|
+
|
|
105
|
+
beforeEach(() => {
|
|
106
|
+
vi.useFakeTimers();
|
|
107
|
+
signalled = [];
|
|
108
|
+
killSpy = vi.spyOn(process, "kill").mockImplementation(((pid: number, sig?: unknown) => {
|
|
109
|
+
if (sig === 0) return true;
|
|
110
|
+
if (sig === "SIGTERM") signalled.push(pid);
|
|
111
|
+
return true;
|
|
112
|
+
}) as unknown as typeof process.kill);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
afterEach(() => {
|
|
116
|
+
stopOrphanSweeper();
|
|
117
|
+
killSpy.mockRestore();
|
|
118
|
+
mockFindOrphans.mockReturnValue([]);
|
|
119
|
+
vi.useRealTimers();
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it("sweeps immediately and then on each interval", () => {
|
|
123
|
+
mockFindOrphans.mockReturnValue([{ pid: 601 }]);
|
|
124
|
+
|
|
125
|
+
startOrphanSweeper();
|
|
126
|
+
expect(signalled).toEqual([601]); // startup sweep
|
|
127
|
+
|
|
128
|
+
vi.advanceTimersByTime(600_000);
|
|
129
|
+
expect(signalled).toEqual([601, 601]); // first periodic run
|
|
130
|
+
|
|
131
|
+
vi.advanceTimersByTime(600_000);
|
|
132
|
+
expect(signalled).toEqual([601, 601, 601]);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it("stops sweeping once cancelled", () => {
|
|
136
|
+
mockFindOrphans.mockReturnValue([{ pid: 602 }]);
|
|
137
|
+
|
|
138
|
+
const stop = startOrphanSweeper();
|
|
139
|
+
expect(signalled).toEqual([602]);
|
|
140
|
+
|
|
141
|
+
stop();
|
|
142
|
+
vi.advanceTimersByTime(1_800_000);
|
|
143
|
+
expect(signalled).toEqual([602]); // no further sweeps
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it("replaces the schedule rather than stacking a second one", () => {
|
|
147
|
+
mockFindOrphans.mockReturnValue([{ pid: 603 }]);
|
|
148
|
+
|
|
149
|
+
startOrphanSweeper();
|
|
150
|
+
startOrphanSweeper();
|
|
151
|
+
signalled.length = 0; // ignore the two startup sweeps
|
|
152
|
+
|
|
153
|
+
vi.advanceTimersByTime(600_000);
|
|
154
|
+
expect(signalled).toEqual([603]); // one interval fired, not two
|
|
155
|
+
});
|
|
156
|
+
});
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Startup sweep for MCP servers orphaned by a previous companion run.
|
|
3
|
+
*
|
|
4
|
+
* Each `claude --print` CLI spawns 2-3 stdio MCP children (e.g. @shopify/dev-mcp,
|
|
5
|
+
* @playwright/mcp plus its headless chromium). Signals go to a pid rather than a
|
|
6
|
+
* tree, so when a CLI dies its children are re-parented to init. `ClaudeAdapter`
|
|
7
|
+
* now reaps them on its own kill path, but that only covers kills companion
|
|
8
|
+
* performs while it is running. If companion itself is restarted, crashes or is
|
|
9
|
+
* OOM-killed, every CLI it owned is orphaned together with those children and
|
|
10
|
+
* nothing remains that could clean them up.
|
|
11
|
+
*
|
|
12
|
+
* Left alone they persist for the life of the host. A production box was found
|
|
13
|
+
* holding 123 such processes across 11.9 GB of 23 GB total, which starved the
|
|
14
|
+
* live CLIs of CPU and memory until they stopped answering — producing exactly
|
|
15
|
+
* the stall and wedge symptoms that cause more kills, and so more orphans.
|
|
16
|
+
*
|
|
17
|
+
* Running this once at startup breaks that cycle. An MCP stdio server whose
|
|
18
|
+
* parent is not a `claude` process has no client and can never acquire one, so
|
|
19
|
+
* terminating it is unambiguously safe.
|
|
20
|
+
*/
|
|
21
|
+
import { findOrphanedMcpProcesses } from "./proc-diagnostics.js";
|
|
22
|
+
import { log } from "./logger.js";
|
|
23
|
+
|
|
24
|
+
/** Grace before escalating; chromium and npm wrappers often ignore SIGTERM. */
|
|
25
|
+
const SWEEP_SIGKILL_AFTER_MS = Number(process.env.COMPANION_SWEEP_SIGKILL_AFTER_MS) || 5000;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* How often to re-run the sweep while the server is up.
|
|
29
|
+
*
|
|
30
|
+
* A startup sweep alone leaves a gap: a CLI killed by something other than this
|
|
31
|
+
* process — the kernel OOM killer is the common one on a loaded host — orphans
|
|
32
|
+
* its MCP children with nobody to reap them, and they then accumulate for the
|
|
33
|
+
* rest of the server's uptime. That is exactly how a production box reached 123
|
|
34
|
+
* orphans across 11.9 GB inside a single run. The scan is a cheap /proc walk, so
|
|
35
|
+
* running it periodically costs nothing measurable.
|
|
36
|
+
*/
|
|
37
|
+
const SWEEP_INTERVAL_MS = Number(process.env.COMPANION_SWEEP_INTERVAL_MS) || 600_000;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Terminate every MCP server process with no live CLI parent.
|
|
41
|
+
*
|
|
42
|
+
* Never throws: a failure to tidy up must not prevent the server from starting.
|
|
43
|
+
* Returns the number of processes signalled, for logging and tests.
|
|
44
|
+
*/
|
|
45
|
+
export function sweepOrphanedMcpProcesses(): number {
|
|
46
|
+
let orphans: ReturnType<typeof findOrphanedMcpProcesses>;
|
|
47
|
+
try {
|
|
48
|
+
orphans = findOrphanedMcpProcesses();
|
|
49
|
+
} catch (err) {
|
|
50
|
+
log.warn("orphan-sweeper", "scan failed; skipping sweep", {
|
|
51
|
+
error: err instanceof Error ? err.message : String(err),
|
|
52
|
+
});
|
|
53
|
+
return 0;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (orphans.length === 0) return 0;
|
|
57
|
+
|
|
58
|
+
const signalled: number[] = [];
|
|
59
|
+
for (const o of orphans) {
|
|
60
|
+
try {
|
|
61
|
+
process.kill(o.pid, "SIGTERM");
|
|
62
|
+
signalled.push(o.pid);
|
|
63
|
+
} catch {
|
|
64
|
+
// Exited between the scan and the signal.
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (signalled.length === 0) return 0;
|
|
69
|
+
|
|
70
|
+
log.info("orphan-sweeper", "reaped MCP servers orphaned by a previous run", {
|
|
71
|
+
count: signalled.length,
|
|
72
|
+
comms: [...new Set(orphans.map((o) => o.comm).filter(Boolean))].slice(0, 5),
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
setTimeout(() => {
|
|
76
|
+
for (const pid of signalled) {
|
|
77
|
+
try {
|
|
78
|
+
process.kill(pid, 0);
|
|
79
|
+
process.kill(pid, "SIGKILL");
|
|
80
|
+
} catch {
|
|
81
|
+
// Gone, which is the desired outcome.
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}, SWEEP_SIGKILL_AFTER_MS).unref?.();
|
|
85
|
+
|
|
86
|
+
return signalled.length;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
let sweepTimer: ReturnType<typeof setInterval> | null = null;
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Sweep at startup, then periodically for as long as the server runs.
|
|
93
|
+
*
|
|
94
|
+
* Returns a stop function, so tests and shutdown paths can clear the timer.
|
|
95
|
+
* Calling this twice replaces the existing schedule rather than stacking a
|
|
96
|
+
* second one.
|
|
97
|
+
*/
|
|
98
|
+
export function startOrphanSweeper(): () => void {
|
|
99
|
+
stopOrphanSweeper();
|
|
100
|
+
sweepOrphanedMcpProcesses();
|
|
101
|
+
sweepTimer = setInterval(() => sweepOrphanedMcpProcesses(), SWEEP_INTERVAL_MS);
|
|
102
|
+
// Housekeeping must never hold the process open.
|
|
103
|
+
(sweepTimer as unknown as { unref?: () => void }).unref?.();
|
|
104
|
+
return stopOrphanSweeper;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Cancel the periodic sweep, if one is scheduled. */
|
|
108
|
+
export function stopOrphanSweeper(): void {
|
|
109
|
+
if (sweepTimer) {
|
|
110
|
+
clearInterval(sweepTimer);
|
|
111
|
+
sweepTimer = null;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
getDescendants,
|
|
7
7
|
hasLiveDescendants,
|
|
8
8
|
countDescendants,
|
|
9
|
+
findOrphanedMcpProcesses,
|
|
9
10
|
} from "./proc-diagnostics.js";
|
|
10
11
|
|
|
11
12
|
/**
|
|
@@ -208,3 +209,81 @@ describe("getDescendants / hasLiveDescendants", () => {
|
|
|
208
209
|
await new Promise((r) => child.on("exit", r));
|
|
209
210
|
});
|
|
210
211
|
});
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Orphaned-MCP detection. This drives a sweeper that SIGTERMs whatever it
|
|
215
|
+
* returns, so both directions matter: missing an orphan leaks memory (a
|
|
216
|
+
* production host reached 123 orphans across 11.9 GB), but a false positive
|
|
217
|
+
* kills a live MCP server out from under a working session.
|
|
218
|
+
*/
|
|
219
|
+
describe("findOrphanedMcpProcesses", () => {
|
|
220
|
+
const spawned: ReturnType<typeof spawn>[] = [];
|
|
221
|
+
|
|
222
|
+
afterEach(async () => {
|
|
223
|
+
for (const p of spawned.splice(0)) {
|
|
224
|
+
try {
|
|
225
|
+
p.kill("SIGKILL");
|
|
226
|
+
await new Promise((r) => p.on("exit", r));
|
|
227
|
+
} catch {
|
|
228
|
+
// Already gone.
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it("returns empty on platforms without /proc", () => {
|
|
234
|
+
vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
|
|
235
|
+
expect(findOrphanedMcpProcesses()).toEqual([]);
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
it.runIf(isLinux)("never throws and returns a well-formed list", () => {
|
|
239
|
+
// Runs against the real /proc of whatever host this is, so it must tolerate
|
|
240
|
+
// processes exiting mid-scan — the scan races every other process on the box.
|
|
241
|
+
const orphans = findOrphanedMcpProcesses();
|
|
242
|
+
expect(Array.isArray(orphans)).toBe(true);
|
|
243
|
+
for (const o of orphans) {
|
|
244
|
+
expect(typeof o.pid).toBe("number");
|
|
245
|
+
expect(o.pid).toBeGreaterThan(0);
|
|
246
|
+
}
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
it.runIf(isLinux)("detects a process whose cmdline marks it an MCP server with no CLI parent", async () => {
|
|
250
|
+
// `exec -a` sets argv[0], reproducing the cmdline shape of a CLI-spawned
|
|
251
|
+
// stdio MCP server. Its parent here is the test runner, not a `claude`
|
|
252
|
+
// process, so it is exactly the orphan the sweeper targets.
|
|
253
|
+
const child = spawn("/bin/bash", [
|
|
254
|
+
"-c",
|
|
255
|
+
"exec -a 'npm exec @modelcontextprotocol/server-test' sleep 30",
|
|
256
|
+
], { stdio: "ignore" });
|
|
257
|
+
spawned.push(child);
|
|
258
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
259
|
+
|
|
260
|
+
const orphans = findOrphanedMcpProcesses();
|
|
261
|
+
expect(orphans.some((o) => o.pid === child.pid)).toBe(true);
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
it.runIf(isLinux)("ignores processes with no MCP marker in their cmdline", async () => {
|
|
265
|
+
// The false-positive guard: an ordinary child must never be swept, or the
|
|
266
|
+
// sweeper would kill unrelated processes on the host.
|
|
267
|
+
const child = spawn("sleep", ["30"], { stdio: "ignore" });
|
|
268
|
+
spawned.push(child);
|
|
269
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
270
|
+
|
|
271
|
+
const orphans = findOrphanedMcpProcesses();
|
|
272
|
+
expect(orphans.some((o) => o.pid === child.pid)).toBe(false);
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
it.runIf(isLinux)("labels findings with comm where readable", async () => {
|
|
276
|
+
const child = spawn("/bin/bash", [
|
|
277
|
+
"-c",
|
|
278
|
+
"exec -a 'mcp-server-label-test' sleep 30",
|
|
279
|
+
], { stdio: "ignore" });
|
|
280
|
+
spawned.push(child);
|
|
281
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
282
|
+
|
|
283
|
+
const found = findOrphanedMcpProcesses().find((o) => o.pid === child.pid);
|
|
284
|
+
expect(found).toBeDefined();
|
|
285
|
+
// comm is the executable name (truncated to 15 chars by the kernel), not
|
|
286
|
+
// argv[0] — it is a label for the log line, not the match key.
|
|
287
|
+
expect(typeof found?.comm).toBe("string");
|
|
288
|
+
});
|
|
289
|
+
});
|
|
@@ -162,6 +162,73 @@ export function isProcAvailable(): boolean {
|
|
|
162
162
|
return process.platform === "linux";
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
+
/** Command-line markers identifying a CLI-spawned stdio MCP server. */
|
|
166
|
+
const MCP_CMDLINE_MARKERS = ["dev-mcp", "playwright/mcp", "mcp-server", "@modelcontextprotocol"];
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Find MCP server processes that no longer belong to a live CLI.
|
|
170
|
+
*
|
|
171
|
+
* Per-kill reaping (see `ClaudeAdapter.reapOrphans`) handles the steady state,
|
|
172
|
+
* but it cannot clean up after the server itself dies: if companion is killed,
|
|
173
|
+
* restarted or OOM-killed, every CLI it owned is orphaned along with that CLI's
|
|
174
|
+
* MCP children, and nothing is left holding the reference needed to reap them.
|
|
175
|
+
* They then sit there for the lifetime of the host, holding memory no one can
|
|
176
|
+
* reclaim — 123 such processes across 11.9 GB were found on a 23 GB production
|
|
177
|
+
* box.
|
|
178
|
+
*
|
|
179
|
+
* An MCP stdio server whose parent is not a `claude` process has no client and
|
|
180
|
+
* can never acquire one, which makes it unambiguously safe to terminate.
|
|
181
|
+
*/
|
|
182
|
+
export function findOrphanedMcpProcesses(): ProcDescendant[] {
|
|
183
|
+
if (!isProcAvailable()) return [];
|
|
184
|
+
|
|
185
|
+
const orphans: ProcDescendant[] = [];
|
|
186
|
+
let pids: string[];
|
|
187
|
+
try {
|
|
188
|
+
pids = readdirSync("/proc").filter((n) => /^\d+$/.test(n));
|
|
189
|
+
} catch {
|
|
190
|
+
return [];
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
for (const entry of pids) {
|
|
194
|
+
const pid = Number(entry);
|
|
195
|
+
if (pid === process.pid) continue;
|
|
196
|
+
|
|
197
|
+
let cmdline: string;
|
|
198
|
+
try {
|
|
199
|
+
cmdline = readFileSync(`/proc/${pid}/cmdline`, "utf8").replace(/\0/g, " ");
|
|
200
|
+
} catch {
|
|
201
|
+
continue; // Exited mid-scan.
|
|
202
|
+
}
|
|
203
|
+
if (!MCP_CMDLINE_MARKERS.some((m) => cmdline.includes(m))) continue;
|
|
204
|
+
|
|
205
|
+
// Parent still a live claude process → it has a client; leave it alone.
|
|
206
|
+
let ppid: number | undefined;
|
|
207
|
+
try {
|
|
208
|
+
ppid = Number(readFileSync(`/proc/${pid}/stat`, "utf8").split(" ")[3]);
|
|
209
|
+
} catch {
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
if (ppid && ppid !== 1) {
|
|
213
|
+
try {
|
|
214
|
+
if (readFileSync(`/proc/${ppid}/cmdline`, "utf8").includes("claude")) continue;
|
|
215
|
+
} catch {
|
|
216
|
+
// Parent vanished between reads — treat as orphaned.
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const orphan: ProcDescendant = { pid };
|
|
221
|
+
try {
|
|
222
|
+
orphan.comm = readFileSync(`/proc/${pid}/comm`, "utf8").trim();
|
|
223
|
+
} catch {
|
|
224
|
+
// Best-effort label only.
|
|
225
|
+
}
|
|
226
|
+
orphans.push(orphan);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
return orphans;
|
|
230
|
+
}
|
|
231
|
+
|
|
165
232
|
/**
|
|
166
233
|
* Capture kernel state for a pid. Returns `{ error }` rather than throwing when
|
|
167
234
|
* the platform has no /proc or the process has already exited (the common race:
|
|
@@ -8,6 +8,7 @@ import type { Session } from "./ws-bridge-types.js";
|
|
|
8
8
|
import { appendHistory } from "./ws-bridge-persist.js";
|
|
9
9
|
import { validatePermission } from "./ai-validator.js";
|
|
10
10
|
import { getEffectiveAiValidation } from "./ai-validation-settings.js";
|
|
11
|
+
import { log } from "./logger.js";
|
|
11
12
|
import { companionBus } from "./event-bus.js";
|
|
12
13
|
|
|
13
14
|
/**
|
|
@@ -176,7 +177,15 @@ export function attachCodexAdapterHandlers(
|
|
|
176
177
|
session.pendingPermissions.clear();
|
|
177
178
|
session.backendAdapter = null;
|
|
178
179
|
deps.persistSession(session);
|
|
179
|
-
|
|
180
|
+
// Mirrors WsBridge.broadcastCliDisconnected so every path that raises the
|
|
181
|
+
// "CLI disconnected / Reconnect" banner is greppable under one message.
|
|
182
|
+
log.warn("ws-bridge", "UI: CLI disconnected banner shown", {
|
|
183
|
+
sessionId,
|
|
184
|
+
reason: "codex_adapter_disconnected",
|
|
185
|
+
backendType: session.backendType,
|
|
186
|
+
phase: session.stateMachine.phase,
|
|
187
|
+
browsers: session.browserSockets.size,
|
|
188
|
+
});
|
|
180
189
|
deps.broadcastToBrowsers(session, { type: "cli_disconnected" });
|
|
181
190
|
|
|
182
191
|
// Auto-relaunch if browsers are still connected (don't leave users staring
|
package/server/ws-bridge.test.ts
CHANGED
|
@@ -1963,6 +1963,39 @@ describe("CLI message routing", () => {
|
|
|
1963
1963
|
expect(session.inFlightUserTurn).toBeTruthy();
|
|
1964
1964
|
});
|
|
1965
1965
|
|
|
1966
|
+
/**
|
|
1967
|
+
* The "CLI disconnected / Reconnect" banner is the single most-reported
|
|
1968
|
+
* symptom, so every path that raises it must leave a greppable log line.
|
|
1969
|
+
* Previously the four broadcast sites logged inconsistently — two via
|
|
1970
|
+
* console.log, one describing the cause rather than the broadcast, one
|
|
1971
|
+
* silent — which made post-hoc log analysis miss occurrences entirely.
|
|
1972
|
+
*/
|
|
1973
|
+
it("logs a greppable line whenever the CLI-disconnected banner is broadcast", async () => {
|
|
1974
|
+
vi.useFakeTimers();
|
|
1975
|
+
try {
|
|
1976
|
+
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
|
1977
|
+
const cli = makeCliSocket("s1");
|
|
1978
|
+
bridge.handleCLIOpen(cli, "s1");
|
|
1979
|
+
await bridge.handleCLIMessage(cli, makeInitMsg());
|
|
1980
|
+
|
|
1981
|
+
const browser = makeBrowserSocket("s1");
|
|
1982
|
+
bridge.handleBrowserOpen(browser, "s1");
|
|
1983
|
+
|
|
1984
|
+
bridge.handleCLIClose(cli);
|
|
1985
|
+
// Past the disconnect debounce, the banner goes out.
|
|
1986
|
+
await vi.advanceTimersByTimeAsync(20_000);
|
|
1987
|
+
|
|
1988
|
+
const calls = warnSpy.mock.calls.map(([a]) => String(a));
|
|
1989
|
+
const banner = calls.find((l) => l.includes("UI: CLI disconnected banner shown"));
|
|
1990
|
+
expect(banner).toBeDefined();
|
|
1991
|
+
// The reason is what makes a log searchable for *why*, not just *that*.
|
|
1992
|
+
expect(banner).toContain("cli_disconnect_confirmed");
|
|
1993
|
+
expect(banner).toContain("s1");
|
|
1994
|
+
} finally {
|
|
1995
|
+
vi.useRealTimers();
|
|
1996
|
+
}
|
|
1997
|
+
});
|
|
1998
|
+
|
|
1966
1999
|
it("tool_progress: broadcasts", async () => {
|
|
1967
2000
|
const msg = JSON.stringify({
|
|
1968
2001
|
type: "tool_progress",
|
package/server/ws-bridge.ts
CHANGED
|
@@ -687,7 +687,7 @@ export class WsBridge {
|
|
|
687
687
|
session.pendingPermissions.clear();
|
|
688
688
|
session.stateMachine.transition("terminated", "disconnect_confirmed");
|
|
689
689
|
this.persistSession(session);
|
|
690
|
-
this.
|
|
690
|
+
this.broadcastCliDisconnected(session, "codex_disconnect_confirmed");
|
|
691
691
|
|
|
692
692
|
// Request auto-relaunch regardless of browser state — proactive
|
|
693
693
|
// keepalive in the orchestrator ensures headless sessions stay alive.
|
|
@@ -931,7 +931,10 @@ export class WsBridge {
|
|
|
931
931
|
interruptedMidTurn,
|
|
932
932
|
});
|
|
933
933
|
session.stateMachine.transition("terminated", "disconnect_confirmed");
|
|
934
|
-
this.
|
|
934
|
+
this.broadcastCliDisconnected(session, "cli_disconnect_confirmed", {
|
|
935
|
+
phaseAtClose,
|
|
936
|
+
interruptedMidTurn,
|
|
937
|
+
});
|
|
935
938
|
for (const [reqId] of session.pendingPermissions) {
|
|
936
939
|
this.broadcastToBrowsers(session, { type: "permission_cancelled", request_id: reqId });
|
|
937
940
|
}
|
|
@@ -990,8 +993,17 @@ export class WsBridge {
|
|
|
990
993
|
if (!backendConnected && !this.disconnectTimers.has(sessionId)) {
|
|
991
994
|
// Only signal disconnection if we're not within the debounce window
|
|
992
995
|
// (CLI may be mid-reconnect — avoid UI flap and spurious relaunch)
|
|
996
|
+
// Same banner, different trigger: a browser attached to a session whose
|
|
997
|
+
// backend is already gone. Logged in the same shape so both paths surface
|
|
998
|
+
// under one grep.
|
|
999
|
+
log.warn("ws-bridge", "UI: CLI disconnected banner shown", {
|
|
1000
|
+
sessionId,
|
|
1001
|
+
reason: "backend_dead_on_browser_open",
|
|
1002
|
+
backendType: session.backendType,
|
|
1003
|
+
phase: session.stateMachine.phase,
|
|
1004
|
+
browsers: session.browserSockets.size,
|
|
1005
|
+
});
|
|
993
1006
|
this.sendToBrowser(ws, { type: "cli_disconnected" });
|
|
994
|
-
console.log(`[ws-bridge] Browser connected but backend is dead for session ${sessionId}, requesting relaunch`);
|
|
995
1007
|
companionBus.emit("session:relaunch-needed", { sessionId });
|
|
996
1008
|
}
|
|
997
1009
|
}
|
|
@@ -1304,6 +1316,34 @@ export class WsBridge {
|
|
|
1304
1316
|
this.broadcastToBrowsers(session, { type: "session_name_update", name });
|
|
1305
1317
|
}
|
|
1306
1318
|
|
|
1319
|
+
/**
|
|
1320
|
+
* Broadcast `cli_disconnected` and record it, in that order, from one place.
|
|
1321
|
+
*
|
|
1322
|
+
* `cli_disconnected` is what puts the "CLI disconnected / Reconnect" banner in
|
|
1323
|
+
* front of the user, so every occurrence must be greppable afterwards — this
|
|
1324
|
+
* is the single most reported symptom and was previously logged inconsistently
|
|
1325
|
+
* across the four call sites (two `console.log`, one `log.warn` describing the
|
|
1326
|
+
* cause rather than the broadcast, one silent).
|
|
1327
|
+
*
|
|
1328
|
+
* Emitting it here means the log line and the banner cannot drift apart: if
|
|
1329
|
+
* the user saw the banner, this line exists.
|
|
1330
|
+
*/
|
|
1331
|
+
private broadcastCliDisconnected(
|
|
1332
|
+
session: Session,
|
|
1333
|
+
reason: string,
|
|
1334
|
+
detail: Record<string, unknown> = {},
|
|
1335
|
+
) {
|
|
1336
|
+
log.warn("ws-bridge", "UI: CLI disconnected banner shown", {
|
|
1337
|
+
sessionId: session.id,
|
|
1338
|
+
reason,
|
|
1339
|
+
backendType: session.backendType,
|
|
1340
|
+
phase: session.stateMachine.phase,
|
|
1341
|
+
browsers: session.browserSockets.size,
|
|
1342
|
+
...detail,
|
|
1343
|
+
});
|
|
1344
|
+
this.broadcastToBrowsers(session, { type: "cli_disconnected" });
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1307
1347
|
private broadcastToBrowsers(session: Session, msg: BrowserIncomingMessage) {
|
|
1308
1348
|
broadcastToBrowsersFn(session, msg, {
|
|
1309
1349
|
eventBufferLimit: EVENT_BUFFER_LIMIT,
|