@lensmcp/cluster 1.16.33 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/executors/gateway/runtime/lens-children.d.ts +27 -5
- package/executors/gateway/runtime/lens-children.d.ts.map +1 -1
- package/executors/gateway/runtime/lens-children.js +222 -28
- package/executors/gateway/runtime/lifecycle.d.ts +55 -2
- package/executors/gateway/runtime/lifecycle.d.ts.map +1 -1
- package/executors/gateway/runtime/lifecycle.js +65 -9
- package/executors/gateway/runtime/proxy.d.ts +4 -1
- package/executors/gateway/runtime/proxy.d.ts.map +1 -1
- package/executors/gateway/runtime/proxy.js +91 -14
- package/executors/gateway/runtime/scope.d.ts +12 -0
- package/executors/gateway/runtime/scope.d.ts.map +1 -1
- package/executors/gateway/runtime/scope.js +25 -0
- package/executors/gateway/runtime/server.d.ts +33 -0
- package/executors/gateway/runtime/server.d.ts.map +1 -1
- package/executors/gateway/runtime/server.js +72 -3
- package/executors/gateway/runtime/types.d.ts +55 -0
- package/executors/gateway/runtime/types.d.ts.map +1 -1
- package/executors/gateway/runtime/types.js +56 -1
- package/package.json +3 -3
|
@@ -2,12 +2,34 @@ import { type SpawnChild } from '@lensmcp/nx-plugin/lens-frontend';
|
|
|
2
2
|
import type { WorkspaceFragment } from './route-registry';
|
|
3
3
|
import type { GatewayRuntime, GatewayRuntimeOptions } from './types';
|
|
4
4
|
import { type Observability } from './observability';
|
|
5
|
-
/** True when `port` can actually be BOUND on this machine right now. The in-process `used*Ports` sets only
|
|
6
|
-
* know what THIS daemon claimed — an ORPHANED child of a CRASHED daemon (or any foreign process) still
|
|
7
|
-
* holds its port at the OS level. Routing to such a port serves the WRONG workspace's content: the child
|
|
8
|
-
* we spawn crash-loops on EADDRINUSE while the route proxies into the stale squatter (the "tetros
|
|
9
|
-
* dashboard redirects to foodguard" bug). */
|
|
10
5
|
export declare function probePortFree(port: number): Promise<boolean>;
|
|
6
|
+
/** True when something is LISTENING on `port` on loopback — the liveness signal the exit handler cannot
|
|
7
|
+
* give us ({@link LENS_PROBE_ENABLED}: a vite alive with zero LISTEN sockets).
|
|
8
|
+
*
|
|
9
|
+
* BOTH loopback families are tried before reporting a failure. Vite binds the NAME `localhost`, which can
|
|
10
|
+
* resolve to `::1` on a v6-preferring host — probing only `127.0.0.1` there would report every healthy dev
|
|
11
|
+
* server as wedged and SIGKILL the whole cluster in a loop. The v6 attempt only runs on the v4 failure
|
|
12
|
+
* path, so the healthy case stays one connect. */
|
|
13
|
+
export declare function probePortListening(port: number, timeoutMs?: number): Promise<boolean>;
|
|
14
|
+
/**
|
|
15
|
+
* Poll until `port` is actually BINDABLE, up to `timeoutMs`. Resolves `true` when it frees, `false` on
|
|
16
|
+
* timeout (the caller then spawns anyway and lets the normal heal handle the fallout).
|
|
17
|
+
*
|
|
18
|
+
* WHY a respawn must await this: a dying vite can hold its listener for seconds (open HMR websockets keep
|
|
19
|
+
* the server closing), so the fixed 1.5s heal delay raced it and the fresh child died on
|
|
20
|
+
* `listen EADDRINUSE` — a self-inflicted crash that burned a heal attempt and lengthened the outage
|
|
21
|
+
* (see {@link PORT_FREE_WAIT_MS}). Probes at least ONCE even with `timeoutMs: 0`, so the common already-free
|
|
22
|
+
* case costs a single probe and no delay.
|
|
23
|
+
*/
|
|
24
|
+
export declare function waitForPortFree(port: number, timeoutMs: number, deps?: {
|
|
25
|
+
isPortFree?: (port: number) => Promise<boolean>;
|
|
26
|
+
intervalMs?: number;
|
|
27
|
+
}): Promise<boolean>;
|
|
28
|
+
/** The `--port <n>` / `--port=<n>` a vite child was launched with. `spawnLensFrontend` always PINS the port
|
|
29
|
+
* for a gateway-routed app (`--port <n> --strictPort`), so argv is the authoritative record of which port
|
|
30
|
+
* this child owns — the port the liveness probe checks and the port its respawn must wait to be free.
|
|
31
|
+
* `undefined` for an unpinned child (nothing to probe, nothing to wait for). */
|
|
32
|
+
export declare function viteArgPort(args: readonly string[]): number | undefined;
|
|
11
33
|
export interface LensChildren {
|
|
12
34
|
spawnManagedChild: SpawnChild;
|
|
13
35
|
/** Boot the dashboard + MCP singletons and every `lens:true` app. Pushes the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lens-children.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lens-children.ts"],"names":[],"mappings":"AASA,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,kCAAkC,CAAC;AAI1C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1D,OAAO,KAAK,EAAE,cAAc,EAAE,qBAAqB,EAAuB,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"lens-children.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lens-children.ts"],"names":[],"mappings":"AASA,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,kCAAkC,CAAC;AAI1C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1D,OAAO,KAAK,EAAE,cAAc,EAAE,qBAAqB,EAAuB,MAAM,SAAS,CAAC;AAgB1F,OAAO,EAA0B,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA0B7E,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAalE;AAyBD;;;;;;mDAMmD;AACnD,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,SAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAG1F;AAMD;;;;;;;;;GASG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAClF,OAAO,CAAC,OAAO,CAAC,CAUlB;AAED;;;iFAGiF;AACjF,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAQvE;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB,EAAE,UAAU,CAAC;IAC9B;;gFAE4E;IAC5E,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC;;6GAEyG;IACzG,sBAAsB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACvF;;yGAEqG;IACrG,iBAAiB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACpE;gHAC4G;IAC5G,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,cAAc,EAClB,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,qBAAqB,EAE9B,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,GACzD,YAAY,CAmbd"}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.probePortFree = probePortFree;
|
|
4
|
+
exports.probePortListening = probePortListening;
|
|
5
|
+
exports.waitForPortFree = waitForPortFree;
|
|
6
|
+
exports.viteArgPort = viteArgPort;
|
|
4
7
|
exports.createLensChildren = createLensChildren;
|
|
5
8
|
const tslib_1 = require("tslib");
|
|
6
9
|
/**
|
|
@@ -27,14 +30,110 @@ const HEAL_HEALTHY_MS = 20_000; // ran at least this long ⇒ a FRESH crash ⇒
|
|
|
27
30
|
* holds its port at the OS level. Routing to such a port serves the WRONG workspace's content: the child
|
|
28
31
|
* we spawn crash-loops on EADDRINUSE while the route proxies into the stale squatter (the "tetros
|
|
29
32
|
* dashboard redirects to foodguard" bug). */
|
|
30
|
-
|
|
33
|
+
/** One bind attempt. `in-use` only for the errnos that actually mean "someone has it"; any other failure
|
|
34
|
+
* means that ADDRESS FAMILY is unavailable on this machine (no IPv6, for instance), which is not evidence
|
|
35
|
+
* either way and must never be reported as "taken" — that would make `claimFreePort` step up forever. */
|
|
36
|
+
function tryBind(port, host) {
|
|
31
37
|
return new Promise((resolve) => {
|
|
32
38
|
const probe = net.createServer();
|
|
33
39
|
probe.unref();
|
|
34
|
-
probe.once('error', () =>
|
|
35
|
-
|
|
40
|
+
probe.once('error', (e) => {
|
|
41
|
+
const code = e.code ?? '';
|
|
42
|
+
resolve(code === 'EADDRINUSE' || code === 'EACCES' ? 'in-use' : 'unavailable');
|
|
43
|
+
});
|
|
44
|
+
probe.listen(host === undefined ? { port } : { port, host }, () => probe.close(() => resolve('free')));
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
async function probePortFree(port) {
|
|
48
|
+
// The WILDCARD bind alone is NOT enough, and that hole defeated the very purpose of this probe. MEASURED:
|
|
49
|
+
// with a holder on `127.0.0.1:P`, `listen({ port: P })` (the wildcard) reports **FREE** — a wildcard bind
|
|
50
|
+
// does not collide with a specific-address bind on macOS/BSD. Our children bind the NAME `localhost`
|
|
51
|
+
// (vite `--port N --strictPort`, the dashboard bundles), i.e. a LOOPBACK address — exactly the case the
|
|
52
|
+
// wildcard probe cannot see. So an orphaned vite holding `127.0.0.1:4200` looked free, `claimFreePort`
|
|
53
|
+
// handed 4200 straight back out, and the fresh child died on `listen EADDRINUSE` — the self-inflicted
|
|
54
|
+
// crash this probe exists to prevent (and what makes `waitForPortFree` meaningful rather than a no-op).
|
|
55
|
+
// Free means bindable on the wildcard AND on both loopback addresses; unavailable families are skipped.
|
|
56
|
+
for (const host of [undefined, '127.0.0.1', '::1']) {
|
|
57
|
+
if ((await tryBind(port, host)) === 'in-use')
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
/** A single-address TCP connect: does `host:port` ACCEPT right now? Connect-and-destroy, never an HTTP
|
|
63
|
+
* request — a WEDGED vite may accept then hang forever, so a `GET` would time out even where the listener
|
|
64
|
+
* is fine (and would pollute the dev server's log). Every timer/socket is unref'd: a probe must never hold
|
|
65
|
+
* the gateway process open. */
|
|
66
|
+
function probeConnect(host, port, timeoutMs) {
|
|
67
|
+
return new Promise((resolve) => {
|
|
68
|
+
let done = false;
|
|
69
|
+
const sock = net.createConnection({ host, port });
|
|
70
|
+
sock.unref();
|
|
71
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
72
|
+
timer.unref?.();
|
|
73
|
+
function finish(ok) {
|
|
74
|
+
if (done)
|
|
75
|
+
return;
|
|
76
|
+
done = true;
|
|
77
|
+
clearTimeout(timer);
|
|
78
|
+
sock.destroy();
|
|
79
|
+
resolve(ok);
|
|
80
|
+
}
|
|
81
|
+
sock.once('connect', () => finish(true));
|
|
82
|
+
sock.once('error', () => finish(false));
|
|
36
83
|
});
|
|
37
84
|
}
|
|
85
|
+
/** True when something is LISTENING on `port` on loopback — the liveness signal the exit handler cannot
|
|
86
|
+
* give us ({@link LENS_PROBE_ENABLED}: a vite alive with zero LISTEN sockets).
|
|
87
|
+
*
|
|
88
|
+
* BOTH loopback families are tried before reporting a failure. Vite binds the NAME `localhost`, which can
|
|
89
|
+
* resolve to `::1` on a v6-preferring host — probing only `127.0.0.1` there would report every healthy dev
|
|
90
|
+
* server as wedged and SIGKILL the whole cluster in a loop. The v6 attempt only runs on the v4 failure
|
|
91
|
+
* path, so the healthy case stays one connect. */
|
|
92
|
+
async function probePortListening(port, timeoutMs = 1_000) {
|
|
93
|
+
if (await probeConnect('127.0.0.1', port, timeoutMs))
|
|
94
|
+
return true;
|
|
95
|
+
return probeConnect('::1', port, timeoutMs);
|
|
96
|
+
}
|
|
97
|
+
/** Unref'd sleep — a heal/backoff wait must never keep the gateway alive past `stop()`. */
|
|
98
|
+
const sleepUnref = (ms) => new Promise((r) => { const t = setTimeout(r, ms); t.unref?.(); });
|
|
99
|
+
/**
|
|
100
|
+
* Poll until `port` is actually BINDABLE, up to `timeoutMs`. Resolves `true` when it frees, `false` on
|
|
101
|
+
* timeout (the caller then spawns anyway and lets the normal heal handle the fallout).
|
|
102
|
+
*
|
|
103
|
+
* WHY a respawn must await this: a dying vite can hold its listener for seconds (open HMR websockets keep
|
|
104
|
+
* the server closing), so the fixed 1.5s heal delay raced it and the fresh child died on
|
|
105
|
+
* `listen EADDRINUSE` — a self-inflicted crash that burned a heal attempt and lengthened the outage
|
|
106
|
+
* (see {@link PORT_FREE_WAIT_MS}). Probes at least ONCE even with `timeoutMs: 0`, so the common already-free
|
|
107
|
+
* case costs a single probe and no delay.
|
|
108
|
+
*/
|
|
109
|
+
async function waitForPortFree(port, timeoutMs, deps = {}) {
|
|
110
|
+
const isFree = deps.isPortFree ?? probePortFree;
|
|
111
|
+
const intervalMs = Math.max(1, deps.intervalMs ?? types_1.PORT_FREE_POLL_MS);
|
|
112
|
+
const deadline = Date.now() + Math.max(0, timeoutMs);
|
|
113
|
+
for (;;) {
|
|
114
|
+
if (await isFree(port))
|
|
115
|
+
return true;
|
|
116
|
+
const left = deadline - Date.now();
|
|
117
|
+
if (left <= 0)
|
|
118
|
+
return false;
|
|
119
|
+
await sleepUnref(Math.min(intervalMs, left));
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/** The `--port <n>` / `--port=<n>` a vite child was launched with. `spawnLensFrontend` always PINS the port
|
|
123
|
+
* for a gateway-routed app (`--port <n> --strictPort`), so argv is the authoritative record of which port
|
|
124
|
+
* this child owns — the port the liveness probe checks and the port its respawn must wait to be free.
|
|
125
|
+
* `undefined` for an unpinned child (nothing to probe, nothing to wait for). */
|
|
126
|
+
function viteArgPort(args) {
|
|
127
|
+
const valid = (n) => (Number.isInteger(n) && n > 0 && n < 65_536 ? n : undefined);
|
|
128
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
129
|
+
const a = args[i];
|
|
130
|
+
if (a === '--port')
|
|
131
|
+
return valid(Number(args[i + 1]));
|
|
132
|
+
if (a.startsWith('--port='))
|
|
133
|
+
return valid(Number(a.slice('--port='.length)));
|
|
134
|
+
}
|
|
135
|
+
return undefined;
|
|
136
|
+
}
|
|
38
137
|
function createLensChildren(rt, obs, options,
|
|
39
138
|
// Injectable OS-level port probe (unit tests stub it; real ports are machine-dependent).
|
|
40
139
|
deps) {
|
|
@@ -43,10 +142,14 @@ deps) {
|
|
|
43
142
|
const lensChildren = [];
|
|
44
143
|
// Auto-heal bookkeeping per child label: consecutive restart attempts (drives the exponential
|
|
45
144
|
// backoff) so a boot-loop settles instead of hammering, reset once a child has run healthily.
|
|
145
|
+
// `portConflicts` counts CONSECUTIVE lost bind races (EADDRINUSE) — forgiven rather than charged to
|
|
146
|
+
// `attempts`, but capped so a permanently-squatted port still falls back to backoff.
|
|
46
147
|
const lensHeal = new Map();
|
|
47
|
-
// Zombie recycle: the live `lens:true` app VITE children (label 'vite') + when each
|
|
48
|
-
//
|
|
49
|
-
//
|
|
148
|
+
// Zombie recycle + LIVENESS probe: the live `lens:true` app VITE children (label 'vite') + when each
|
|
149
|
+
// spawned, its pinned `port`, and its run of consecutive probe failures. `port`/`probeFails` drive the
|
|
150
|
+
// wedge probe (alive but NOT listening); `spawnedAt` also drives the stale-source-set recycle. Both reset
|
|
151
|
+
// naturally on a respawn, which registers a FRESH handle. The capture sidecar + the dashboard/MCP
|
|
152
|
+
// singletons are NOT tracked (they don't serve app source).
|
|
50
153
|
const viteChildren = new Map();
|
|
51
154
|
// Per-project last-recycle time — the cooldown survives the kill→auto-heal→respawn (a new child handle).
|
|
52
155
|
const viteRecycleAt = new Map();
|
|
@@ -101,9 +204,14 @@ deps) {
|
|
|
101
204
|
lensChildren.push(child);
|
|
102
205
|
labeledChildren.set(label, child); // the CURRENT handle for this label (a per-workspace dashboard reap targets it)
|
|
103
206
|
// Track a `lens:true` app's vite dev server (label 'vite', or `<ws>:<proj>:vite` for a registered
|
|
104
|
-
// workspace) for the zombie recycle. Its `LENSMCP_PROJECT` names the app
|
|
105
|
-
|
|
106
|
-
|
|
207
|
+
// workspace) for the zombie recycle + the liveness probe. Its `LENSMCP_PROJECT` names the app and its
|
|
208
|
+
// argv pins the port; a fresh respawn re-adds here with a clean probe run.
|
|
209
|
+
const isVite = label === 'vite' || label.endsWith(':vite');
|
|
210
|
+
// The port THIS child owns (pinned `--port <n> --strictPort`): what the liveness probe checks and what
|
|
211
|
+
// the respawn must wait to be free. `undefined` for the dashboard/MCP singletons + an unpinned vite.
|
|
212
|
+
const ownPort = isVite ? viteArgPort(args) : undefined;
|
|
213
|
+
if (isVite)
|
|
214
|
+
viteChildren.set(child, { project: env?.['LENSMCP_PROJECT'] ?? 'app', spawnedAt, port: ownPort, probeFails: 0 });
|
|
107
215
|
child.on('exit', (code, sig) => {
|
|
108
216
|
const idx = lensChildren.indexOf(child);
|
|
109
217
|
if (idx >= 0)
|
|
@@ -111,34 +219,120 @@ deps) {
|
|
|
111
219
|
viteChildren.delete(child); // the auto-heal respawn re-registers a fresh handle with a new spawnedAt
|
|
112
220
|
if (rt.stopped() || removedLabels.has(label))
|
|
113
221
|
return; // shutting down, or this workspace unregistered — do NOT resurrect
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
222
|
+
const ranMs = Date.now() - spawnedAt;
|
|
223
|
+
const heal = lensHeal.get(label) ?? { attempts: 0, portConflicts: 0 };
|
|
224
|
+
// The heal is ASYNC from here: classifying a bind race and waiting for the port both need a probe.
|
|
225
|
+
// Every wait is unref'd and every resume re-checks stopped()/removedLabels, so shutdown still wins.
|
|
226
|
+
void (async () => {
|
|
227
|
+
// BIND RACE vs CRASH. A fast non-zero exit while ANOTHER process still holds our pinned port is
|
|
228
|
+
// vite's `--strictPort` losing an EADDRINUSE race with the child it replaced — an environment
|
|
229
|
+
// condition, not a code fault. Charging it to `attempts` is what pushed the observed heal to
|
|
230
|
+
// "attempt 2" and doubled the outage; forgive it (bounded) and wait the port out instead.
|
|
231
|
+
const portHeld = ownPort !== undefined && !(await isPortFree(ownPort));
|
|
232
|
+
const raceLost = (0, lifecycle_1.isPortConflictExit)({ code, signal: sig, ranMs, portHeld }, { fastFailMs: types_1.CHILD_START_FAIL_MS });
|
|
233
|
+
if (raceLost && heal.portConflicts < types_1.MAX_PORT_CONFLICT_RETRIES) {
|
|
234
|
+
heal.portConflicts += 1; // `attempts` UNTOUCHED — the crash-loop budget is for real crashes
|
|
235
|
+
lensHeal.set(label, heal);
|
|
236
|
+
console.warn(`[gateway] lens child ${label} lost the :${ownPort} bind race (EADDRINUSE, code=${code}); waiting for the port, then retrying (port-retry ${heal.portConflicts}/${types_1.MAX_PORT_CONFLICT_RETRIES}; crash budget untouched).`);
|
|
237
|
+
}
|
|
238
|
+
else {
|
|
239
|
+
heal.portConflicts = 0; // a real exit ends the bind-race run
|
|
240
|
+
if (ranMs >= HEAL_HEALTHY_MS)
|
|
241
|
+
heal.attempts = 0; // it was healthy → fresh budget
|
|
242
|
+
const delay = Math.min(HEAL_MAX_MS, HEAL_BASE_MS * 2 ** heal.attempts);
|
|
243
|
+
heal.attempts += 1;
|
|
244
|
+
lensHeal.set(label, heal);
|
|
245
|
+
console.warn(`[gateway] lens child ${label} exited (code=${code} sig=${sig}); auto-healing in ${Math.round(delay / 1000)}s (attempt ${heal.attempts}).`);
|
|
246
|
+
await sleepUnref(delay);
|
|
247
|
+
}
|
|
248
|
+
if (rt.stopped() || removedLabels.has(label))
|
|
249
|
+
return;
|
|
250
|
+
// NEVER respawn into the dying child's port. A closing vite can hold its listener for seconds (open
|
|
251
|
+
// HMR websockets), so the old fixed delay raced it → `listen EADDRINUSE` → a self-inflicted crash
|
|
252
|
+
// that burned a heal attempt and LENGTHENED the 502. Bounded: past the budget we spawn anyway and
|
|
253
|
+
// let the (now correctly-classified) heal take it from there.
|
|
254
|
+
if (ownPort !== undefined && !(await waitForPortFree(ownPort, types_1.PORT_FREE_WAIT_MS, { isPortFree, intervalMs: types_1.PORT_FREE_POLL_MS }))) {
|
|
255
|
+
console.warn(`[gateway] lens child ${label}: :${ownPort} still held after ${Math.round(types_1.PORT_FREE_WAIT_MS / 1000)}s — spawning anyway.`);
|
|
256
|
+
}
|
|
257
|
+
if (rt.stopped() || removedLabels.has(label))
|
|
258
|
+
return;
|
|
259
|
+
spawnManagedChild(label, bin, args, env, optional, respawn, cwd);
|
|
260
|
+
})().catch(() => undefined); // a heal must never surface as an unhandled rejection
|
|
124
261
|
});
|
|
125
262
|
return child;
|
|
126
263
|
};
|
|
127
|
-
//
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
//
|
|
264
|
+
// LIVENESS probe for lens frontends: kill a vite that is ALIVE but NOT LISTENING (the permanent-502 wedge
|
|
265
|
+
// — see LENS_PROBE_ENABLED). One cheap TCP connect per tracked child per tick; the pure `lensChildWedged`
|
|
266
|
+
// owns the verdict (grace + N consecutive failures + the shared per-project cooldown), this owns the I/O.
|
|
267
|
+
// A confirmed wedge gets SIGKILL — deliberately NOT SIGTERM: graceful shutdown is exactly what wedged (the
|
|
268
|
+
// event loop is pinned by the lens WS server / bridge sockets), so a SIGTERM can hang and leave the app
|
|
269
|
+
// 502ing forever. The kill lands on the proven exit→auto-heal path, which respawns after waiting the port
|
|
270
|
+
// out. `probeInFlight` keeps a slow round from overlapping the next tick.
|
|
271
|
+
let probeInFlight = false;
|
|
272
|
+
const probeLensChildren = async (cfg) => {
|
|
273
|
+
if (probeInFlight)
|
|
274
|
+
return;
|
|
275
|
+
probeInFlight = true;
|
|
276
|
+
try {
|
|
277
|
+
for (const [child, meta] of [...viteChildren]) {
|
|
278
|
+
if (rt.stopped())
|
|
279
|
+
return;
|
|
280
|
+
if (meta.port === undefined)
|
|
281
|
+
continue; // unpinned port ⇒ nothing to probe
|
|
282
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
283
|
+
continue; // already dying — the exit heal owns it
|
|
284
|
+
if (Date.now() - meta.spawnedAt < cfg.graceMs) {
|
|
285
|
+
meta.probeFails = 0;
|
|
286
|
+
continue;
|
|
287
|
+
} // still booting
|
|
288
|
+
if (await probePortListening(meta.port, types_1.LENS_PROBE_TIMEOUT_MS)) {
|
|
289
|
+
meta.probeFails = 0;
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
meta.probeFails += 1;
|
|
293
|
+
const reason = (0, lifecycle_1.lensChildWedged)(meta, viteRecycleAt.get(meta.project) ?? 0, Date.now(), cfg);
|
|
294
|
+
if (!reason)
|
|
295
|
+
continue; // failing, but not yet N-in-a-row / still in cooldown
|
|
296
|
+
viteRecycleAt.set(meta.project, Date.now());
|
|
297
|
+
meta.probeFails = 0;
|
|
298
|
+
console.error(`[gateway] wedge recycle: lens app ${meta.project} is ALIVE but NOT LISTENING on :${meta.port} after ${Math.max(2, cfg.failureThreshold)} probes — SIGKILL; a fresh vite is coming.`);
|
|
299
|
+
emit('warning', `wedge recycle: ${meta.project} (${reason})`, `cluster-lens-app:${meta.project}`, { kind: 'pod-recycle', project: meta.project, reason, port: meta.port });
|
|
300
|
+
try {
|
|
301
|
+
child.kill('SIGKILL');
|
|
302
|
+
}
|
|
303
|
+
catch { /* already gone — the exit auto-heal will respawn */ }
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
finally {
|
|
307
|
+
probeInFlight = false;
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
// The lens-frontend sweeper drives BOTH self-heals over the tracked vite children:
|
|
311
|
+
// 1. the LIVENESS probe above — alive but not listening (a permanent 502, invisible to the exit handler);
|
|
312
|
+
// 2. ZOMBIE recycle — a vite whose fs-watcher went stale: ALIVE + reachable yet MISSING files created
|
|
313
|
+
// after it spawned (500 `Failed to resolve import` for a file that exists; the reported 17.7h
|
|
314
|
+
// dashboard pod). The shared `rt.sourceSetChangedAt` (stamped by the lifecycle sweeper's source-set
|
|
315
|
+
// scan) drives it: a vite that spawned before a settled set change is killed → this manager's own
|
|
316
|
+
// `on('exit')` auto-heal respawns a fresh vite that re-scans the FS. Only on a set CHANGE
|
|
317
|
+
// (add/remove), never a content save, and never a blind max-age (no idle signal for a frontend).
|
|
133
318
|
const startLensSweeper = () => {
|
|
134
|
-
|
|
319
|
+
// Either heal alone justifies the sweeper: a user who disabled the staleness recycle (its churn was a
|
|
320
|
+
// known problem) must still get a wedged 502 healed, so each branch is gated on its OWN switch.
|
|
321
|
+
if (lensSweeper || (!types_1.POD_RECYCLE_ENABLED && !types_1.LENS_PROBE_ENABLED))
|
|
135
322
|
return;
|
|
136
323
|
const cfg = { graceMs: types_1.POD_STALE_GRACE_MS, settleMs: types_1.POD_RECYCLE_SETTLE_MS, cooldownMs: types_1.POD_STALE_RECYCLE_COOLDOWN_MS };
|
|
324
|
+
const probeCfg = { graceMs: types_1.LENS_PROBE_GRACE_MS, failureThreshold: types_1.LENS_PROBE_FAILS, cooldownMs: types_1.POD_STALE_RECYCLE_COOLDOWN_MS };
|
|
137
325
|
lensSweeper = setInterval(() => {
|
|
138
|
-
if (rt.stopped())
|
|
326
|
+
if (rt.stopped() || viteChildren.size === 0)
|
|
327
|
+
return;
|
|
328
|
+
// The liveness probe is INDEPENDENT of the source-set clock — a wedge has nothing to do with the file
|
|
329
|
+
// set, so it must run before (and regardless of) the staleness early-return below.
|
|
330
|
+
if (types_1.LENS_PROBE_ENABLED)
|
|
331
|
+
void probeLensChildren(probeCfg);
|
|
332
|
+
if (!types_1.POD_RECYCLE_ENABLED)
|
|
139
333
|
return;
|
|
140
334
|
const wideChangedAt = rt.sourceSetChangedAt ?? 0;
|
|
141
|
-
if (wideChangedAt <= 0
|
|
335
|
+
if (wideChangedAt <= 0)
|
|
142
336
|
return;
|
|
143
337
|
const now = Date.now();
|
|
144
338
|
for (const [child, meta] of viteChildren) {
|
|
@@ -9,7 +9,7 @@ import type { Observability } from './observability';
|
|
|
9
9
|
* actually resolve, instead of the whole monorepo — see that function for WHY. Omitted ⇒ the whole root
|
|
10
10
|
* (the original workspace-wide behavior, still used when a child's project root is unknown). Root-level
|
|
11
11
|
* loose files are always counted, so a `package.json`/`tsconfig.base.json` add reaches every scope. */
|
|
12
|
-
export declare function sourceSetSignature(root: string, scopeDirs?: readonly string[]): Promise<string>;
|
|
12
|
+
export declare function sourceSetSignature(root: string, scopeDirs?: readonly string[], extraExclude?: ReadonlySet<string>): Promise<string>;
|
|
13
13
|
/**
|
|
14
14
|
* PURE: which subtrees a child of `projectRoot` must watch for a "file set changed" verdict.
|
|
15
15
|
*
|
|
@@ -30,7 +30,7 @@ export declare function sourceSetSignature(root: string, scopeDirs?: readonly st
|
|
|
30
30
|
* Both args are absolute; the result is absolute and deduped. An empty result (the project root IS the
|
|
31
31
|
* workspace root, or it sits outside it) means "no narrowing" → the caller falls back to the whole root.
|
|
32
32
|
*/
|
|
33
|
-
export declare function sourceScopeDirs(workspaceRoot: string, projectRoot: string, managedBuckets: readonly string[], topLevelDirs: readonly string[]): string[];
|
|
33
|
+
export declare function sourceScopeDirs(workspaceRoot: string, projectRoot: string, managedBuckets: readonly string[], topLevelDirs: readonly string[], extraExclude?: ReadonlySet<string>): string[];
|
|
34
34
|
export interface PodRecycleConfig {
|
|
35
35
|
maxAgeMs: number;
|
|
36
36
|
graceMs: number;
|
|
@@ -58,6 +58,59 @@ export declare function lensFrontendStale(spawnedAt: number, lastRecycleAt: numb
|
|
|
58
58
|
settleMs: number;
|
|
59
59
|
cooldownMs: number;
|
|
60
60
|
}): boolean;
|
|
61
|
+
/** Config for the lens-child LIVENESS decision ({@link lensChildWedged}). */
|
|
62
|
+
export interface LensChildProbeConfig {
|
|
63
|
+
/** Spawn grace: no verdict until the child has had this long to BIND its port (a cold vite boot). */
|
|
64
|
+
graceMs: number;
|
|
65
|
+
/** CONSECUTIVE probe failures required. Floored at 2 — one blip must never recycle. */
|
|
66
|
+
failureThreshold: number;
|
|
67
|
+
/** Per-project anti-thrash window after any recycle (shared with {@link lensFrontendStale}). */
|
|
68
|
+
cooldownMs: number;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* PURE decision: is this ALIVE lens-frontend child WEDGED — process running, port NOT listening?
|
|
72
|
+
*
|
|
73
|
+
* The third self-heal blind spot. `lensFrontendStale` catches a vite serving a stale FS view; the exit
|
|
74
|
+
* handler catches a vite that DIED. Neither sees a vite that is alive, has printed `VITE ready`, and has
|
|
75
|
+
* NO LISTEN socket — its event loop held open by the lens WS server / bridge sockets while its HTTP
|
|
76
|
+
* listener is gone. Nothing exits, so nothing auto-heals, and the app 502s until a human kills the pid
|
|
77
|
+
* (foodguard 2026-07-29 — see {@link LENS_PROBE_ENABLED}).
|
|
78
|
+
*
|
|
79
|
+
* The caller owns the socket I/O (a cheap TCP connect per tick) and accumulates `probeFails`; this owns
|
|
80
|
+
* WHEN that evidence justifies a kill. Three guards, each earning its place:
|
|
81
|
+
* - **grace** — a still-booting vite has no listener YET; killing it would be the outage, not the fix.
|
|
82
|
+
* - **N consecutive failures** (≥2, enforced here so even a misconfigured `failureThreshold: 1` can't
|
|
83
|
+
* single-blip recycle) — a momentary refusal is not a wedge.
|
|
84
|
+
* - **cooldown** — the same per-project window the two recycle paths share, so probe + staleness can
|
|
85
|
+
* never thrash one project between them.
|
|
86
|
+
* Returns the recycle reason or `null`. Kept pure (no process, no socket) so all four are unit-tested.
|
|
87
|
+
*/
|
|
88
|
+
export declare function lensChildWedged(child: {
|
|
89
|
+
spawnedAt: number;
|
|
90
|
+
probeFails: number;
|
|
91
|
+
}, lastRecycleAt: number, now: number, cfg: LensChildProbeConfig): 'wedged-no-listener' | null;
|
|
92
|
+
/**
|
|
93
|
+
* PURE decision: was this child's exit a lost BIND RACE (EADDRINUSE) rather than a crash?
|
|
94
|
+
*
|
|
95
|
+
* The auto-heal respawned on a fixed backoff while the child it replaced still held the pinned port, so
|
|
96
|
+
* vite's `--strictPort` exited code=1 on bind — and that self-inflicted exit consumed a heal attempt,
|
|
97
|
+
* pushing the backoff up and lengthening the very 502 it was healing (observed: `listen EADDRINUSE
|
|
98
|
+
* 127.0.0.1:50059` → `exited (code=1 sig=null); auto-healing in 3s (attempt 2)`). A bind race is an
|
|
99
|
+
* ENVIRONMENT condition, not a code fault, so it must not spend the crash-loop budget.
|
|
100
|
+
*
|
|
101
|
+
* The signature, from evidence we already have (stdio is `inherit`, so the child's stderr is unreadable):
|
|
102
|
+
* a non-zero exit, NOT from a signal (we killed it), FAST enough that it cannot have served traffic, while
|
|
103
|
+
* some OTHER process still holds the port. If the port is free by now the next spawn will simply succeed,
|
|
104
|
+
* so it is treated as a normal heal — the conservative direction.
|
|
105
|
+
*/
|
|
106
|
+
export declare function isPortConflictExit(exit: {
|
|
107
|
+
code: number | null;
|
|
108
|
+
signal: NodeJS.Signals | string | null;
|
|
109
|
+
ranMs: number;
|
|
110
|
+
portHeld: boolean;
|
|
111
|
+
}, cfg: {
|
|
112
|
+
fastFailMs: number;
|
|
113
|
+
}): boolean;
|
|
61
114
|
export interface ServiceLayer {
|
|
62
115
|
scanNow(pool: SockPool): number;
|
|
63
116
|
spawnService(svc: ServiceCtl): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAapE,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,iBAAiB,CAAC;AAyE/D;;;;;;;;wGAQwG;AACxG,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,YAAY,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAqCzI;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,eAAe,CAC7B,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,SAAS,MAAM,EAAE,EACjC,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,YAAY,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GACjC,MAAM,EAAE,CAcV;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,oBAAoB,CAAC,EACpG,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,MAAM,EAC1B,GAAG,EAAE,gBAAgB,GACpB,kBAAkB,GAAG,SAAS,GAAG,IAAI,CAQvC;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,kBAAkB,EAAE,MAAM,EAC1B,GAAG,EAAE,MAAM,EACX,GAAG,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC7D,OAAO,CAMT;AAED,6EAA6E;AAC7E,MAAM,WAAW,oBAAoB;IACnC,qGAAqG;IACrG,OAAO,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,gBAAgB,EAAE,MAAM,CAAC;IACzB,gGAAgG;IAChG,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EAChD,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,oBAAoB,GACxB,oBAAoB,GAAG,IAAI,CAK7B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAChC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,EACvG,GAAG,EAAE;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,GAC1B,OAAO,CAKT;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,CAmcvF"}
|
|
@@ -4,6 +4,8 @@ exports.sourceSetSignature = sourceSetSignature;
|
|
|
4
4
|
exports.sourceScopeDirs = sourceScopeDirs;
|
|
5
5
|
exports.podRecycleReason = podRecycleReason;
|
|
6
6
|
exports.lensFrontendStale = lensFrontendStale;
|
|
7
|
+
exports.lensChildWedged = lensChildWedged;
|
|
8
|
+
exports.isPortConflictExit = isPortConflictExit;
|
|
7
9
|
exports.createServiceLayer = createServiceLayer;
|
|
8
10
|
const tslib_1 = require("tslib");
|
|
9
11
|
/**
|
|
@@ -17,6 +19,7 @@ const http = tslib_1.__importStar(require("node:http"));
|
|
|
17
19
|
const path = tslib_1.__importStar(require("node:path"));
|
|
18
20
|
const node_util_1 = require("node:util");
|
|
19
21
|
const discovery_1 = require("./discovery");
|
|
22
|
+
const scope_1 = require("./scope");
|
|
20
23
|
const pod_env_1 = require("./pod-env");
|
|
21
24
|
const types_1 = require("./types");
|
|
22
25
|
/**
|
|
@@ -97,7 +100,7 @@ async function gitIgnoredPaths(root) {
|
|
|
97
100
|
* actually resolve, instead of the whole monorepo — see that function for WHY. Omitted ⇒ the whole root
|
|
98
101
|
* (the original workspace-wide behavior, still used when a child's project root is unknown). Root-level
|
|
99
102
|
* loose files are always counted, so a `package.json`/`tsconfig.base.json` add reaches every scope. */
|
|
100
|
-
async function sourceSetSignature(root, scopeDirs) {
|
|
103
|
+
async function sourceSetSignature(root, scopeDirs, extraExclude) {
|
|
101
104
|
let count = 0;
|
|
102
105
|
let xor = 0;
|
|
103
106
|
const ignored = await gitIgnoredPaths(root);
|
|
@@ -121,7 +124,7 @@ async function sourceSetSignature(root, scopeDirs) {
|
|
|
121
124
|
if (e.isDirectory()) {
|
|
122
125
|
if (!recurse)
|
|
123
126
|
continue; // scoped mode: root's own dirs are entered only via scopeDirs
|
|
124
|
-
if (name.startsWith('.') || SRC_EXCLUDE.has(name) || name.endsWith('-devserver'))
|
|
127
|
+
if (name.startsWith('.') || SRC_EXCLUDE.has(name) || extraExclude?.has(name) || name.endsWith('-devserver'))
|
|
125
128
|
continue;
|
|
126
129
|
if (ignored.has(`${rel(abs)}/`))
|
|
127
130
|
continue; // a gitignored output dir (foodguard's `artifacts/`)
|
|
@@ -170,7 +173,7 @@ async function sourceSetSignature(root, scopeDirs) {
|
|
|
170
173
|
* Both args are absolute; the result is absolute and deduped. An empty result (the project root IS the
|
|
171
174
|
* workspace root, or it sits outside it) means "no narrowing" → the caller falls back to the whole root.
|
|
172
175
|
*/
|
|
173
|
-
function sourceScopeDirs(workspaceRoot, projectRoot, managedBuckets, topLevelDirs) {
|
|
176
|
+
function sourceScopeDirs(workspaceRoot, projectRoot, managedBuckets, topLevelDirs, extraExclude) {
|
|
174
177
|
const bucketOf = (abs) => {
|
|
175
178
|
const r = path.relative(workspaceRoot, abs).split(path.sep).filter(Boolean);
|
|
176
179
|
return r.length === 0 || r[0].startsWith('..') ? '' : r[0];
|
|
@@ -181,7 +184,7 @@ function sourceScopeDirs(workspaceRoot, projectRoot, managedBuckets, topLevelDir
|
|
|
181
184
|
const managed = new Set(managedBuckets.filter(Boolean));
|
|
182
185
|
const keep = new Set([own]);
|
|
183
186
|
for (const d of topLevelDirs) {
|
|
184
|
-
if (d.startsWith('.') || SRC_EXCLUDE.has(d) || d.endsWith('-devserver'))
|
|
187
|
+
if (d.startsWith('.') || SRC_EXCLUDE.has(d) || extraExclude?.has(d) || d.endsWith('-devserver'))
|
|
185
188
|
continue;
|
|
186
189
|
if (!managed.has(d))
|
|
187
190
|
keep.add(d); // a bucket owned by no managed project = shared/leaf → always watch
|
|
@@ -229,6 +232,56 @@ function lensFrontendStale(spawnedAt, lastRecycleAt, sourceSetChangedAt, now, cf
|
|
|
229
232
|
return false; // let an add/remove burst settle → one recycle
|
|
230
233
|
return true;
|
|
231
234
|
}
|
|
235
|
+
/**
|
|
236
|
+
* PURE decision: is this ALIVE lens-frontend child WEDGED — process running, port NOT listening?
|
|
237
|
+
*
|
|
238
|
+
* The third self-heal blind spot. `lensFrontendStale` catches a vite serving a stale FS view; the exit
|
|
239
|
+
* handler catches a vite that DIED. Neither sees a vite that is alive, has printed `VITE ready`, and has
|
|
240
|
+
* NO LISTEN socket — its event loop held open by the lens WS server / bridge sockets while its HTTP
|
|
241
|
+
* listener is gone. Nothing exits, so nothing auto-heals, and the app 502s until a human kills the pid
|
|
242
|
+
* (foodguard 2026-07-29 — see {@link LENS_PROBE_ENABLED}).
|
|
243
|
+
*
|
|
244
|
+
* The caller owns the socket I/O (a cheap TCP connect per tick) and accumulates `probeFails`; this owns
|
|
245
|
+
* WHEN that evidence justifies a kill. Three guards, each earning its place:
|
|
246
|
+
* - **grace** — a still-booting vite has no listener YET; killing it would be the outage, not the fix.
|
|
247
|
+
* - **N consecutive failures** (≥2, enforced here so even a misconfigured `failureThreshold: 1` can't
|
|
248
|
+
* single-blip recycle) — a momentary refusal is not a wedge.
|
|
249
|
+
* - **cooldown** — the same per-project window the two recycle paths share, so probe + staleness can
|
|
250
|
+
* never thrash one project between them.
|
|
251
|
+
* Returns the recycle reason or `null`. Kept pure (no process, no socket) so all four are unit-tested.
|
|
252
|
+
*/
|
|
253
|
+
function lensChildWedged(child, lastRecycleAt, now, cfg) {
|
|
254
|
+
if (now - child.spawnedAt < cfg.graceMs)
|
|
255
|
+
return null; // still booting — it has no listener YET
|
|
256
|
+
if (now - lastRecycleAt < cfg.cooldownMs)
|
|
257
|
+
return null; // anti-thrash (shared with the staleness recycle)
|
|
258
|
+
if (child.probeFails < Math.max(2, cfg.failureThreshold))
|
|
259
|
+
return null; // one blip is never a wedge
|
|
260
|
+
return 'wedged-no-listener';
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* PURE decision: was this child's exit a lost BIND RACE (EADDRINUSE) rather than a crash?
|
|
264
|
+
*
|
|
265
|
+
* The auto-heal respawned on a fixed backoff while the child it replaced still held the pinned port, so
|
|
266
|
+
* vite's `--strictPort` exited code=1 on bind — and that self-inflicted exit consumed a heal attempt,
|
|
267
|
+
* pushing the backoff up and lengthening the very 502 it was healing (observed: `listen EADDRINUSE
|
|
268
|
+
* 127.0.0.1:50059` → `exited (code=1 sig=null); auto-healing in 3s (attempt 2)`). A bind race is an
|
|
269
|
+
* ENVIRONMENT condition, not a code fault, so it must not spend the crash-loop budget.
|
|
270
|
+
*
|
|
271
|
+
* The signature, from evidence we already have (stdio is `inherit`, so the child's stderr is unreadable):
|
|
272
|
+
* a non-zero exit, NOT from a signal (we killed it), FAST enough that it cannot have served traffic, while
|
|
273
|
+
* some OTHER process still holds the port. If the port is free by now the next spawn will simply succeed,
|
|
274
|
+
* so it is treated as a normal heal — the conservative direction.
|
|
275
|
+
*/
|
|
276
|
+
function isPortConflictExit(exit, cfg) {
|
|
277
|
+
if (exit.signal)
|
|
278
|
+
return false; // we (or the OS) killed it — a recycle/reap, never a bind failure
|
|
279
|
+
if (exit.code === null || exit.code === 0)
|
|
280
|
+
return false; // a clean exit is not a start failure
|
|
281
|
+
if (exit.ranMs > cfg.fastFailMs)
|
|
282
|
+
return false; // it RAN ⇒ it held the port ⇒ cannot be a bind race
|
|
283
|
+
return exit.portHeld; // someone else still holds it ⇒ this exit lost the bind race
|
|
284
|
+
}
|
|
232
285
|
function createServiceLayer(rt, obs) {
|
|
233
286
|
const { emit } = obs;
|
|
234
287
|
// Route a service's lifecycle events (starting/up/down/cold-start/…) to ITS OWNING workspace's bus —
|
|
@@ -615,11 +668,11 @@ function createServiceLayer(rt, obs) {
|
|
|
615
668
|
};
|
|
616
669
|
/** project → the absolute dirs its child must watch, or `undefined` when it cannot be narrowed (unknown
|
|
617
670
|
* project root / no `projectsConfigurations`) → that child keeps the workspace-wide clock. */
|
|
618
|
-
const scopeFor = (project, topLevel, managed) => {
|
|
671
|
+
const scopeFor = (project, topLevel, managed, extraExclude) => {
|
|
619
672
|
const pr = rt.projectRoots?.[project];
|
|
620
673
|
if (!pr)
|
|
621
674
|
return undefined;
|
|
622
|
-
const dirs = sourceScopeDirs(rt.root, pr, managed, topLevel);
|
|
675
|
+
const dirs = sourceScopeDirs(rt.root, pr, managed, topLevel, extraExclude);
|
|
623
676
|
return dirs.length > 0 ? dirs : undefined;
|
|
624
677
|
};
|
|
625
678
|
// Kick a bounded, async source-set scan when due; on a SET change (file added/removed) stamp the clocks so
|
|
@@ -634,7 +687,10 @@ function createServiceLayer(rt, obs) {
|
|
|
634
687
|
scanInFlight = true;
|
|
635
688
|
lastScanAt = Date.now();
|
|
636
689
|
void (async () => {
|
|
637
|
-
|
|
690
|
+
// Per-workspace `.lensmcp/config.json` sourceSet.exclude — re-read per scan so an
|
|
691
|
+
// edit takes effect without a gateway restart (one tiny fs read per scan tick).
|
|
692
|
+
const extraExclude = (0, scope_1.readSourceSetExclude)(rt.root);
|
|
693
|
+
const wide = await sourceSetSignature(rt.root, undefined, extraExclude);
|
|
638
694
|
if (prevSig !== undefined && wide !== prevSig) {
|
|
639
695
|
rt.sourceSetChangedAt = Date.now();
|
|
640
696
|
emit('info', 'source file set changed — stale pods recycle when idle', 'cluster-gateway', { kind: 'source-set-changed' });
|
|
@@ -660,7 +716,7 @@ function createServiceLayer(rt, obs) {
|
|
|
660
716
|
// bucket → the scope dirs + the projects sharing it (one walk per bucket).
|
|
661
717
|
const byBucket = new Map();
|
|
662
718
|
for (const project of projects) {
|
|
663
|
-
const dirs = scopeFor(project, topLevel, managed);
|
|
719
|
+
const dirs = scopeFor(project, topLevel, managed, extraExclude);
|
|
664
720
|
if (!dirs)
|
|
665
721
|
continue;
|
|
666
722
|
const key = dirs.join('|');
|
|
@@ -670,7 +726,7 @@ function createServiceLayer(rt, obs) {
|
|
|
670
726
|
}
|
|
671
727
|
const now = Date.now();
|
|
672
728
|
for (const [key, { dirs, projects: sharing }] of byBucket) {
|
|
673
|
-
const sig = await sourceSetSignature(rt.root, dirs);
|
|
729
|
+
const sig = await sourceSetSignature(rt.root, dirs, extraExclude);
|
|
674
730
|
const prev = prevScopedSig.get(key);
|
|
675
731
|
prevScopedSig.set(key, sig);
|
|
676
732
|
if (prev === undefined || sig === prev)
|
|
@@ -6,7 +6,10 @@ import type { ServiceLayer } from './lifecycle';
|
|
|
6
6
|
export interface ProxyServer {
|
|
7
7
|
web(req: http.IncomingMessage, res: http.ServerResponse, opts: Record<string, unknown>, cb: (err: Error) => void): void;
|
|
8
8
|
ws(req: http.IncomingMessage, socket: stream.Duplex, head: Buffer, opts: Record<string, unknown>, cb: (err?: Error) => void): void;
|
|
9
|
-
on(event: 'proxyRes', cb: (proxyRes: http.IncomingMessage, req: http.IncomingMessage) => void): void;
|
|
9
|
+
on(event: 'proxyRes', cb: (proxyRes: http.IncomingMessage, req: http.IncomingMessage, res: http.ServerResponse) => void): void;
|
|
10
|
+
/** Emitted once the upstream ClientRequest has its socket — the only handle http-proxy gives us on the
|
|
11
|
+
* OUTBOUND request, and so the only place the edge-cancel propagation below can be wired. */
|
|
12
|
+
on(event: 'proxyReq', cb: (proxyReq: http.ClientRequest, req: http.IncomingMessage, res: http.ServerResponse) => void): void;
|
|
10
13
|
close?(): void;
|
|
11
14
|
}
|
|
12
15
|
export interface ProxyLayer {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/proxy.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,cAAc,EAAW,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA8BhD,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACxH,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACnI,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/proxy.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,cAAc,EAAW,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA8BhD,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACxH,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACnI,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,KAAK,IAAI,GAAG,IAAI,CAAC;IAC/H;kGAC8F;IAC9F,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,KAAK,IAAI,GAAG,IAAI,CAAC;IAC7H,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,OAAO,YAAY,EAAE,KAAK,CAAC;IACzE,aAAa,IAAI,IAAI,CAAC;IACtB,UAAU,IAAI,IAAI,CAAC;CACpB;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,GAAG,UAAU,CAuYtG"}
|
|
@@ -56,8 +56,46 @@ function createProxy(rt, obs, svcLayer) {
|
|
|
56
56
|
// and leave the gateway-request trace open; the timeout fires the proxy
|
|
57
57
|
// error path → 502 → res 'close' → inflight decrements + trace finishes.
|
|
58
58
|
const proxy = httpProxy.createProxyServer({ xfwd: true, secure: false, proxyTimeout: 120_000, agent: httpAgent });
|
|
59
|
+
/**
|
|
60
|
+
* EDGE-CANCEL PROPAGATION for the H1 (node-http-proxy) path — the H1 twin of `pipeUpstream`'s
|
|
61
|
+
* `onEdgeClose`, and a real leak fix.
|
|
62
|
+
*
|
|
63
|
+
* node-http-proxy 1.18.1 propagates a client cancel through exactly ONE hook:
|
|
64
|
+
* `req.on('aborted', () => proxyReq.abort())` (web-incoming.js). `'aborted'` is the event Node
|
|
65
|
+
* DEPRECATED in v16 in favour of `'close'`, and it does not fire here — MEASURED on Node 24 with the
|
|
66
|
+
* real pooled-agent setup below: on an edge cancel mid-response the upstream request was never aborted
|
|
67
|
+
* (`aborted` 0 times), the upstream response never closed, the upstream kept STREAMING forever into a
|
|
68
|
+
* socket nobody reads, and the pooled socket stayed permanently CHECKED OUT of the agent
|
|
69
|
+
* (`in-use=1, free=0`) — leaked, never reused, never released. Every cancelled request (an unmounted
|
|
70
|
+
* SSE stream, an HMR reload, a navigated-away fetch) leaked one upstream socket AND pinned the pod
|
|
71
|
+
* generating events for a client that is gone; enough of them walk the agent to `maxSockets`, after
|
|
72
|
+
* which new requests queue behind sockets that can never come free.
|
|
73
|
+
*
|
|
74
|
+
* So the cancel is wired here instead, on both halves of the upstream exchange:
|
|
75
|
+
* - `proxyReq` — cancel BEFORE the response arrives ⇒ destroy the outbound request.
|
|
76
|
+
* - `proxyRes` — cancel MID-response ⇒ destroy the upstream response.
|
|
77
|
+
* DESTROY, never release: a destroyed socket is dropped from the agent instead of returned to
|
|
78
|
+
* `freeSockets`, so a half-drained socket can never be picked up by a later request. Measured after the
|
|
79
|
+
* fix: upstream closed, `in-use=0, free=0`, and http-proxy's error callback NOT invoked (so this raises
|
|
80
|
+
* no spurious 502 and triggers no heal retry). Each listener is removed when its upstream half closes,
|
|
81
|
+
* so nothing accumulates on a retried request (see the leak note in `pipeUpstream`).
|
|
82
|
+
*/
|
|
83
|
+
const propagateEdgeCancel = (upstream, res, completed) => {
|
|
84
|
+
const onEdgeClose = () => {
|
|
85
|
+
if (completed())
|
|
86
|
+
return; // the response finished normally — nothing to cancel
|
|
87
|
+
if (!upstream.destroyed)
|
|
88
|
+
upstream.destroy();
|
|
89
|
+
};
|
|
90
|
+
res.once('close', onEdgeClose);
|
|
91
|
+
upstream.once('close', () => res.removeListener('close', onEdgeClose));
|
|
92
|
+
};
|
|
93
|
+
proxy.on('proxyReq', (proxyReq, _req, res) => {
|
|
94
|
+
// `writableEnded` ⇒ we already flushed the whole edge response, so this 'close' is normal completion.
|
|
95
|
+
propagateEdgeCancel(proxyReq, res, () => res.writableEnded);
|
|
96
|
+
});
|
|
59
97
|
// Surface WHICH pod served the request (round-robin made visible).
|
|
60
|
-
proxy.on('proxyRes', (proxyRes, req) => {
|
|
98
|
+
proxy.on('proxyRes', (proxyRes, req, res) => {
|
|
61
99
|
const pod = req.__lensmcpPod;
|
|
62
100
|
if (pod)
|
|
63
101
|
proxyRes.headers['x-lensmcp-pod'] = pod;
|
|
@@ -65,6 +103,10 @@ function createProxy(rt, obs, svcLayer) {
|
|
|
65
103
|
// (e.g. NestJS/Express `x-powered-by`) and brand the hop as the LensMCP gateway.
|
|
66
104
|
delete proxyRes.headers['x-powered-by'];
|
|
67
105
|
proxyRes.headers['server'] = 'LensMCP';
|
|
106
|
+
// A mid-response cancel must stop the upstream too: `proxyRes.pipe(res)` only UNPIPES when the edge
|
|
107
|
+
// dies (Node's `pipe` never destroys the source), which is what stranded the socket.
|
|
108
|
+
if (res)
|
|
109
|
+
propagateEdgeCancel(proxyRes, res, () => res.writableEnded);
|
|
68
110
|
});
|
|
69
111
|
// --- native HTTP/2 forwarder ------------------------------------------------
|
|
70
112
|
// node-http-proxy is HTTP/1.1-only: it copies connection-specific headers onto the
|
|
@@ -114,8 +156,34 @@ function createProxy(rt, obs, svcLayer) {
|
|
|
114
156
|
// `res.destroy(err)` below) emits 'error'; UNHANDLED, that throws and can
|
|
115
157
|
// escalate to tear the whole h2 session down — the very "burst" symptom.
|
|
116
158
|
// A no-op listener keeps the fault local to THIS stream. (Harmless on h1.)
|
|
117
|
-
|
|
159
|
+
//
|
|
160
|
+
// ONCE PER REQUEST, not per attempt. `pipeUpstream` is re-entered for every TCP heal RETRY (see
|
|
161
|
+
// `tryTcp`), and containment is a property of the STREAM, not of one upstream attempt — so a plain
|
|
162
|
+
// `res.on('error')` here accumulated one listener per retry. With an 8s heal window at 300ms steps
|
|
163
|
+
// that is ~26 listeners on a single response, which Node reported as
|
|
164
|
+
// `MaxListenersExceededWarning: 11 error listeners added to [Http2ServerResponse]` (and the twin for
|
|
165
|
+
// [Http2ServerRequest] via the `req.on('error')` below). Each leaked closure also RETAINS the
|
|
166
|
+
// request/response objects — a memory bug, not just log noise. The marker makes re-entry idempotent.
|
|
167
|
+
const guard = res;
|
|
168
|
+
if (!guard.__lensmcpErrContained) {
|
|
169
|
+
guard.__lensmcpErrContained = true;
|
|
170
|
+
res.on('error', () => { });
|
|
171
|
+
}
|
|
172
|
+
// Per-ATTEMPT listeners live on the SHARED per-request `req`/`res`, so each attempt must REMOVE its own
|
|
173
|
+
// on the way out — otherwise a heal retry (`tryTcp`) stacks one more of each per pass (the leak above).
|
|
174
|
+
// `upstream` is captured so the edge-close handler can be built (and detached) outside the response cb.
|
|
175
|
+
let upstream;
|
|
176
|
+
const onReqError = () => { upstreamReq.destroy(); };
|
|
177
|
+
// The client canceled: stop reading the upstream so its pooled keep-alive socket isn't left flowing
|
|
178
|
+
// (a half-drained socket is corrupt on the next reuse → would break a later request).
|
|
179
|
+
const onEdgeClose = () => { if (upstream && !upstream.destroyed)
|
|
180
|
+
upstream.destroy(); };
|
|
181
|
+
const detach = () => {
|
|
182
|
+
req.removeListener('error', onReqError);
|
|
183
|
+
res.removeListener('close', onEdgeClose);
|
|
184
|
+
};
|
|
118
185
|
const upstreamReq = mod.request(opts, (upstreamRes) => {
|
|
186
|
+
upstream = upstreamRes;
|
|
119
187
|
cbs.onResponse();
|
|
120
188
|
if (res.headersSent || res.writableEnded || res.destroyed) {
|
|
121
189
|
upstreamRes.destroy();
|
|
@@ -154,15 +222,17 @@ function createProxy(rt, obs, svcLayer) {
|
|
|
154
222
|
// normal-completion 'close' a no-op. (An unhandled 'error' on upstreamRes would also throw.)
|
|
155
223
|
const abortEdge = () => { if (!res.writableEnded && !res.destroyed)
|
|
156
224
|
res.destroy(); };
|
|
157
|
-
upstreamRes.
|
|
158
|
-
upstreamRes.
|
|
159
|
-
|
|
160
|
-
// (a half-drained socket is corrupt on the next reuse → would break a later request).
|
|
161
|
-
res.on('close', () => { if (!upstreamRes.destroyed)
|
|
162
|
-
upstreamRes.destroy(); });
|
|
225
|
+
upstreamRes.once('error', abortEdge);
|
|
226
|
+
upstreamRes.once('close', abortEdge);
|
|
227
|
+
res.on('close', onEdgeClose);
|
|
163
228
|
});
|
|
164
|
-
upstreamReq.
|
|
165
|
-
|
|
229
|
+
upstreamReq.once('error', (err) => cbs.onError(err));
|
|
230
|
+
// ClientRequest 'close' fires exactly once however the attempt ended (completed, errored, destroyed), so
|
|
231
|
+
// it is the one place that can un-register this attempt's listeners — and drop the closures retaining
|
|
232
|
+
// `req`/`res`/`upstream`. For a long-lived stream it fires only when the stream ends, so `onEdgeClose`
|
|
233
|
+
// stays armed for the whole stream (a mid-stream client cancel MUST still destroy the upstream).
|
|
234
|
+
upstreamReq.once('close', detach);
|
|
235
|
+
req.on('error', onReqError);
|
|
166
236
|
// GET/HEAD/OPTIONS carry no body — end immediately; else stream the request body upstream.
|
|
167
237
|
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS')
|
|
168
238
|
upstreamReq.end();
|
|
@@ -234,8 +304,11 @@ function createProxy(rt, obs, svcLayer) {
|
|
|
234
304
|
pipeUpstream(tcpTarget, req, res, {
|
|
235
305
|
onResponse: () => { },
|
|
236
306
|
onError: (err) => {
|
|
237
|
-
|
|
238
|
-
|
|
307
|
+
// `!res.destroyed`: once the CLIENT is gone there is nobody to heal for — retrying would hammer a
|
|
308
|
+
// recovering upstream for the rest of the window and then write an error into a dead stream.
|
|
309
|
+
if (isConnError(err) && Date.now() < tcpDeadline && !res.headersSent && !res.writableEnded && !res.destroyed) {
|
|
310
|
+
const t = setTimeout(tryTcp, types_1.UPSTREAM_HEAL_STEP_MS);
|
|
311
|
+
t.unref?.(); // a pending heal retry must never keep the gateway process alive
|
|
239
312
|
return;
|
|
240
313
|
}
|
|
241
314
|
// Past the response headers there's nothing to send an error body into — reset the stream.
|
|
@@ -352,10 +425,14 @@ function createProxy(rt, obs, svcLayer) {
|
|
|
352
425
|
const tcpDeadline = Date.now() + types_1.UPSTREAM_HEAL_WINDOW_MS;
|
|
353
426
|
const tryTcp = () => {
|
|
354
427
|
proxy.web(req, res, tcpOpts, (err) => {
|
|
355
|
-
|
|
356
|
-
|
|
428
|
+
// `!res.destroyed` — see the h2 twin above: with the client gone there is nothing to heal for.
|
|
429
|
+
if (isConnError(err) && Date.now() < tcpDeadline && !res.headersSent && !res.writableEnded && !res.destroyed) {
|
|
430
|
+
const t = setTimeout(tryTcp, types_1.UPSTREAM_HEAL_STEP_MS);
|
|
431
|
+
t.unref?.();
|
|
357
432
|
return;
|
|
358
433
|
}
|
|
434
|
+
if (res.destroyed)
|
|
435
|
+
return; // the client left — no stream to write an error into
|
|
359
436
|
console.error(`[gateway] upstream ${route.project} (${String(tcpTarget)}):`, err.message);
|
|
360
437
|
(0, edge_1.sendError)(res, req, 502, 'unavailable', `Upstream ${route.project} unavailable.`);
|
|
361
438
|
});
|
|
@@ -11,6 +11,18 @@ export declare function readLensScope(root: string): {
|
|
|
11
11
|
dashboardPort: number;
|
|
12
12
|
basePath: string;
|
|
13
13
|
};
|
|
14
|
+
/**
|
|
15
|
+
* Per-workspace extra SOURCE-SET excludes (`.lensmcp/config.json` →
|
|
16
|
+
* `sourceSet.exclude: string[]`) — top-level dir names the staleness scanner
|
|
17
|
+
* must NOT treat as source. The scanner deliberately over-includes every
|
|
18
|
+
* unmanaged bucket (`shared/`, `docs/`, …) in every pod's scope, but a bucket
|
|
19
|
+
* like `docs/` can host source-extension files (`.json`, `.mdx`) that no pod
|
|
20
|
+
* can ever import — each add there flips the signature and recycles pods for
|
|
21
|
+
* nothing (foodguard 2026-07-29: a `docs/**\/*-intake.json` add recycled the
|
|
22
|
+
* dashboard; 475 recycles in one day were scope-C adds). Names only (no
|
|
23
|
+
* globs), matched exactly like the built-in excludes. Empty when unset.
|
|
24
|
+
*/
|
|
25
|
+
export declare function readSourceSetExclude(root: string): Set<string>;
|
|
14
26
|
export declare function lensKeyFrom(root: string): string;
|
|
15
27
|
export declare function lensSlug(input: string): string;
|
|
16
28
|
//# sourceMappingURL=scope.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scope.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/scope.ts"],"names":[],"mappings":"AAOA;;;8CAG8C;AAC9C,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAUpD;AAED;;yEAEyE;AACzE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAmBpG;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQhD;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQ9C"}
|
|
1
|
+
{"version":3,"file":"scope.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/scope.ts"],"names":[],"mappings":"AAOA;;;8CAG8C;AAC9C,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAUpD;AAED;;yEAEyE;AACzE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAmBpG;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAa9D;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQhD;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQ9C"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.baseDomainOf = baseDomainOf;
|
|
4
4
|
exports.readLensScope = readLensScope;
|
|
5
|
+
exports.readSourceSetExclude = readSourceSetExclude;
|
|
5
6
|
exports.lensKeyFrom = lensKeyFrom;
|
|
6
7
|
exports.lensSlug = lensSlug;
|
|
7
8
|
const tslib_1 = require("tslib");
|
|
@@ -48,6 +49,30 @@ function readLensScope(root) {
|
|
|
48
49
|
const key = lensKeyFrom(root);
|
|
49
50
|
return { key, dashboardPort: 4321, basePath: '/' + key };
|
|
50
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Per-workspace extra SOURCE-SET excludes (`.lensmcp/config.json` →
|
|
54
|
+
* `sourceSet.exclude: string[]`) — top-level dir names the staleness scanner
|
|
55
|
+
* must NOT treat as source. The scanner deliberately over-includes every
|
|
56
|
+
* unmanaged bucket (`shared/`, `docs/`, …) in every pod's scope, but a bucket
|
|
57
|
+
* like `docs/` can host source-extension files (`.json`, `.mdx`) that no pod
|
|
58
|
+
* can ever import — each add there flips the signature and recycles pods for
|
|
59
|
+
* nothing (foodguard 2026-07-29: a `docs/**\/*-intake.json` add recycled the
|
|
60
|
+
* dashboard; 475 recycles in one day were scope-C adds). Names only (no
|
|
61
|
+
* globs), matched exactly like the built-in excludes. Empty when unset.
|
|
62
|
+
*/
|
|
63
|
+
function readSourceSetExclude(root) {
|
|
64
|
+
try {
|
|
65
|
+
const cfg = JSON.parse(fs.readFileSync(path.join(root, '.lensmcp', 'config.json'), 'utf8'));
|
|
66
|
+
const raw = cfg?.sourceSet?.exclude;
|
|
67
|
+
if (Array.isArray(raw)) {
|
|
68
|
+
return new Set(raw.filter((d) => typeof d === 'string' && d.length > 0 && !d.includes('/')));
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
/* no config — nothing extra to exclude */
|
|
73
|
+
}
|
|
74
|
+
return new Set();
|
|
75
|
+
}
|
|
51
76
|
function lensKeyFrom(root) {
|
|
52
77
|
try {
|
|
53
78
|
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
|
|
@@ -15,5 +15,38 @@ export declare function buildEdgeVerifierPool(getRoutes: () => Route[]): EdgeVer
|
|
|
15
15
|
/** Client-abort errnos that are NORMAL front-door traffic (a closed tab, a sleeping laptop, a canceled
|
|
16
16
|
* HMR socket) — dropped quietly. Anything else on an inbound socket is logged before the drop. */
|
|
17
17
|
export declare const BENIGN_SOCKET_ERRNOS: ReadonlySet<string>;
|
|
18
|
+
/**
|
|
19
|
+
* PURE classification of Node's H1 `clientError` — what to answer, and whether it is worth reporting.
|
|
20
|
+
*
|
|
21
|
+
* The old handler answered a BARE, UNLOGGED `400 Bad Request` to every cause and threw `err` away. That is
|
|
22
|
+
* how a gateway-generated 400 became indistinguishable from an application 400: a dev sees `GET
|
|
23
|
+
* /api/…/artifacts → 400` in the browser while the service logged `200`, with nothing anywhere to say the
|
|
24
|
+
* front door invented it. (Worked failure, foodguard: spurious empty-bodied 400s on real API calls, which
|
|
25
|
+
* a dev-proxy hop forwards verbatim to the browser, so they look exactly like app bugs.) MEASURED causes on
|
|
26
|
+
* Node 24: `HPE_INVALID_METHOD`, `HPE_INVALID_HEADER_TOKEN`, `HPE_HEADER_OVERFLOW`, `HPE_INVALID_EOF_STATE`.
|
|
27
|
+
*
|
|
28
|
+
* Three corrections fall out of that list:
|
|
29
|
+
* - `HPE_HEADER_OVERFLOW` is **431**, not 400 — the request was well-formed, just too big.
|
|
30
|
+
* - `ERR_HTTP_REQUEST_TIMEOUT` is **408** (what Node's own default handler answers) — a slow client is not
|
|
31
|
+
* a malformed one, and calling it 400 sends a dev hunting a nonexistent parse bug.
|
|
32
|
+
* - `HPE_INVALID_EOF_STATE` means the peer half-closed MID-MESSAGE — the signature of a client that went
|
|
33
|
+
* away, and of the classic keep-alive REUSE RACE (a pooled client socket written to just as the server
|
|
34
|
+
* closes it, see `GATEWAY_KEEPALIVE_TIMEOUT_MS`). Writing a 400 into that socket is precisely what turns
|
|
35
|
+
* a harmless race into a client-visible 400, so it is treated like `ECONNRESET`: destroy, say nothing.
|
|
36
|
+
* Everything unrecognized keeps the previous behavior (400 + destroy), now logged.
|
|
37
|
+
*/
|
|
38
|
+
export interface ClientErrorVerdict {
|
|
39
|
+
/** Status line to best-effort write, or `null` for "write nothing, just destroy". */
|
|
40
|
+
status: 400 | 408 | 431 | null;
|
|
41
|
+
/** Log + emit? A routine client abort is not worth a line (and a scanner must not be able to flood). */
|
|
42
|
+
report: boolean;
|
|
43
|
+
/** Short cause label for the log/event. */
|
|
44
|
+
cause: string;
|
|
45
|
+
}
|
|
46
|
+
export declare function classifyClientError(code: string | undefined, message: string): ClientErrorVerdict;
|
|
47
|
+
/** Per-cause log throttle for `clientError`: a desynced client or a port scanner can produce these in a
|
|
48
|
+
* tight loop, and the point is a dev NOTICING one — not drowning in thousands. One line per cause per
|
|
49
|
+
* window; the suppressed count rides the next admitted line. Exported for the unit test. */
|
|
50
|
+
export declare function createClientErrorLogGate(windowMs?: number): (cause: string) => boolean;
|
|
18
51
|
export declare function startGateway(options: GatewayRuntimeOptions, context: GatewayContext): Promise<GatewayHandle>;
|
|
19
52
|
//# sourceMappingURL=server.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/server.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAkB,KAAK,gBAAgB,EAAE,MAAM,QAAQ,CAAC;AAK/D,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAkB,qBAAqB,EAAe,KAAK,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/server.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAkB,KAAK,gBAAgB,EAAE,MAAM,QAAQ,CAAC;AAK/D,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAkB,qBAAqB,EAAe,KAAK,EAAE,MAAM,SAAS,CAAC;AA2BxH;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,KAAK,EAAE,GAAG,gBAAgB,CA4ChF;AAED;mGACmG;AACnG,eAAO,MAAM,oBAAoB,EAAE,WAAW,CAAC,MAAM,CAA8E,CAAC;AAEpI;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,kBAAkB;IACjC,qFAAqF;IACrF,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC;IAC/B,wGAAwG;IACxG,MAAM,EAAE,OAAO,CAAC;IAChB,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,kBAAkB,CAQjG;AAQD;;6FAE6F;AAC7F,wBAAgB,wBAAwB,CAAC,QAAQ,SAAS,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAStF;AAkBD,wBAAsB,YAAY,CAChC,OAAO,EAAE,qBAAqB,EAC9B,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,aAAa,CAAC,CAsUxB"}
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BENIGN_SOCKET_ERRNOS = void 0;
|
|
4
4
|
exports.buildEdgeVerifierPool = buildEdgeVerifierPool;
|
|
5
|
+
exports.classifyClientError = classifyClientError;
|
|
6
|
+
exports.createClientErrorLogGate = createClientErrorLogGate;
|
|
5
7
|
exports.startGateway = startGateway;
|
|
6
8
|
const tslib_1 = require("tslib");
|
|
7
9
|
/**
|
|
@@ -31,6 +33,7 @@ const hooks_1 = require("./hooks");
|
|
|
31
33
|
const handler_1 = require("./handler");
|
|
32
34
|
const upgrade_1 = require("./upgrade");
|
|
33
35
|
const jwks_verify_1 = require("../jwks-verify");
|
|
36
|
+
const types_1 = require("./types");
|
|
34
37
|
/**
|
|
35
38
|
* A `fetch`-shaped shim for the JWKS pull that accepts the dev gateway's OWN self-signed cert. The dev
|
|
36
39
|
* IdP's JWKS is served behind the same local HTTPS front door (e.g. `https://auth.tetros.ai.local`), whose
|
|
@@ -116,6 +119,39 @@ function buildEdgeVerifierPool(getRoutes) {
|
|
|
116
119
|
/** Client-abort errnos that are NORMAL front-door traffic (a closed tab, a sleeping laptop, a canceled
|
|
117
120
|
* HMR socket) — dropped quietly. Anything else on an inbound socket is logged before the drop. */
|
|
118
121
|
exports.BENIGN_SOCKET_ERRNOS = new Set(['ECONNRESET', 'EPIPE', 'ETIMEDOUT', 'ECONNABORTED', 'ECANCELED']);
|
|
122
|
+
function classifyClientError(code, message) {
|
|
123
|
+
if (code && exports.BENIGN_SOCKET_ERRNOS.has(code))
|
|
124
|
+
return { status: null, report: false, cause: 'client-abort' };
|
|
125
|
+
// Peer half-closed mid-message: nobody is waiting for a 400, and writing one is the spurious-400 vector.
|
|
126
|
+
if (code === 'HPE_INVALID_EOF_STATE')
|
|
127
|
+
return { status: null, report: false, cause: 'truncated-at-eof' };
|
|
128
|
+
if (code === 'ERR_HTTP_REQUEST_TIMEOUT')
|
|
129
|
+
return { status: 408, report: true, cause: 'request-timeout' };
|
|
130
|
+
if (code === 'HPE_HEADER_OVERFLOW')
|
|
131
|
+
return { status: 431, report: true, cause: 'header-overflow' };
|
|
132
|
+
if (code?.startsWith('HPE_'))
|
|
133
|
+
return { status: 400, report: true, cause: 'parse-error' };
|
|
134
|
+
return { status: 400, report: true, cause: code ?? message ?? 'unknown' };
|
|
135
|
+
}
|
|
136
|
+
const CLIENT_ERROR_REASON = {
|
|
137
|
+
400: 'Bad Request',
|
|
138
|
+
408: 'Request Timeout',
|
|
139
|
+
431: 'Request Header Fields Too Large',
|
|
140
|
+
};
|
|
141
|
+
/** Per-cause log throttle for `clientError`: a desynced client or a port scanner can produce these in a
|
|
142
|
+
* tight loop, and the point is a dev NOTICING one — not drowning in thousands. One line per cause per
|
|
143
|
+
* window; the suppressed count rides the next admitted line. Exported for the unit test. */
|
|
144
|
+
function createClientErrorLogGate(windowMs = 10_000) {
|
|
145
|
+
const lastAt = new Map();
|
|
146
|
+
return (cause) => {
|
|
147
|
+
const now = Date.now();
|
|
148
|
+
const prev = lastAt.get(cause) ?? 0;
|
|
149
|
+
if (now - prev < windowMs)
|
|
150
|
+
return false;
|
|
151
|
+
lastAt.set(cause, now);
|
|
152
|
+
return true;
|
|
153
|
+
};
|
|
154
|
+
}
|
|
119
155
|
/** Attach the inbound-socket error guard. EVERY socket the front door accepts gets an 'error' listener
|
|
120
156
|
* the moment it exists — both the raw TCP socket ('connection') and the post-handshake TLSSocket
|
|
121
157
|
* ('secureConnection', a DIFFERENT emitter). A client abort is routine, but with NO listener Node turns
|
|
@@ -202,6 +238,8 @@ async function startGateway(options, context) {
|
|
|
202
238
|
};
|
|
203
239
|
// --- layers -----------------------------------------------------------------
|
|
204
240
|
const obs = (0, observability_1.createObservability)(eventFile, { trafficFlushMs });
|
|
241
|
+
// One gate for every front-door port — a desynced client must not get N log lines because we bound N ports.
|
|
242
|
+
const admitClientErrorLog = createClientErrorLogGate();
|
|
205
243
|
const svcLayer = (0, lifecycle_1.createServiceLayer)(rt, obs);
|
|
206
244
|
const lens = (0, lens_children_1.createLensChildren)(rt, obs, options);
|
|
207
245
|
// The daemon CONTROL PLANE (loopback ~/.lensmcp/control.sock): a second workspace's `gateway start`
|
|
@@ -334,11 +372,42 @@ async function startGateway(options, context) {
|
|
|
334
372
|
server.on('connection', guardInboundSocket);
|
|
335
373
|
server.on('secureConnection', guardInboundSocket); // never fires on a plain-http server — harmless
|
|
336
374
|
server.on('tlsClientError', (_err, tlsSocket) => tlsSocket.destroy());
|
|
337
|
-
server.on('clientError', (
|
|
338
|
-
|
|
339
|
-
|
|
375
|
+
server.on('clientError', (rawErr, socket) => {
|
|
376
|
+
// Node adds `bytesParsed`/`rawPacket` on an HTTP parse error (not on the base ErrnoException type):
|
|
377
|
+
// `bytesParsed` is the single most useful field for telling a truncated reuse race (0 / very low) from
|
|
378
|
+
// a genuinely malformed request, which is exactly what the old handler discarded.
|
|
379
|
+
const err = rawErr;
|
|
380
|
+
const v = classifyClientError(err.code, err.message);
|
|
381
|
+
// OBSERVABILITY (this used to be silent — the whole reason a gateway-invented 400 looked like an app
|
|
382
|
+
// bug). Rate-limited per cause so a port scanner or a desynced client cannot flood the log; the bus
|
|
383
|
+
// `emit` is separately flood-guarded by its fingerprint.
|
|
384
|
+
if (v.report && admitClientErrorLog(v.cause)) {
|
|
385
|
+
const peer = socket.remoteAddress ?? 'unknown';
|
|
386
|
+
const answer = v.status === null ? 'dropped' : String(v.status);
|
|
387
|
+
console.warn(`[gateway] inbound H1 clientError from ${peer}: ${v.cause} (${err.code ?? err.message}${err.bytesParsed !== undefined ? `, bytesParsed=${err.bytesParsed}` : ''}) → ${answer}. This status is the GATEWAY's, not the upstream service's.`);
|
|
388
|
+
obs.emit('warning', `inbound clientError: ${v.cause} → ${answer}`, `gateway-client-error:${v.cause}`, {
|
|
389
|
+
kind: 'gateway-client-error', cause: v.cause, code: err.code ?? null, status: v.status, peer,
|
|
390
|
+
bytesParsed: err.bytesParsed ?? null,
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
// Preserved intent: a best-effort status on a still-writable socket, then destroy.
|
|
394
|
+
if (v.status !== null && socket.writable)
|
|
395
|
+
socket.end(`HTTP/1.1 ${v.status} ${CLIENT_ERROR_REASON[v.status]}\r\n\r\n`);
|
|
340
396
|
socket.destroy();
|
|
341
397
|
});
|
|
398
|
+
// KEEP-ALIVE REUSE RACE. Node's server default `keepAliveTimeout` is 5s — and a Node HTTP client's
|
|
399
|
+
// pooled agent (Node 19+ `globalAgent` is keepAlive:true; vite's dev proxy uses it) idles on ~the same
|
|
400
|
+
// 5s. Both sides then race the SAME deadline: the client writes a request onto a socket the server is
|
|
401
|
+
// simultaneously closing, the server reads a truncated message, and the front door answers a bare
|
|
402
|
+
// `400` that the dev proxy forwards to the browser verbatim — a 400 on a request the service would
|
|
403
|
+
// have served 200. The documented fix is for the server's idle window to EXCEED every client's, so the
|
|
404
|
+
// server never closes first and the client always controls socket reuse. `headersTimeout` must stay
|
|
405
|
+
// above `keepAliveTimeout` (it bounds a request that has started arriving), and `requestTimeout` keeps
|
|
406
|
+
// its Node default. Dev-only front door, a handful of clients → holding idle sockets longer is free.
|
|
407
|
+
server.keepAliveTimeout = types_1.GATEWAY_KEEPALIVE_TIMEOUT_MS;
|
|
408
|
+
if (server.headersTimeout < types_1.GATEWAY_KEEPALIVE_TIMEOUT_MS + 5_000) {
|
|
409
|
+
server.headersTimeout = types_1.GATEWAY_KEEPALIVE_TIMEOUT_MS + 5_000;
|
|
410
|
+
}
|
|
342
411
|
// h2 SESSION resilience: without a `sessionError` handler ANY session-level
|
|
343
412
|
// fault (a client GOAWAY mid-burst, a flood-protection trip, a decode error)
|
|
344
413
|
// is unhandled → Node tears the session down, failing EVERY sibling stream at
|
|
@@ -213,6 +213,61 @@ export declare const POD_STALE_SCAN_MS: number;
|
|
|
213
213
|
/** Per-service cooldown between staleness recycles — stops a source-churn burst (codegen) thrash-recycling
|
|
214
214
|
* the same pod. Override `LENSMCP_POD_STALE_RECYCLE_COOLDOWN_MS`. Default 60s. */
|
|
215
215
|
export declare const POD_STALE_RECYCLE_COOLDOWN_MS: number;
|
|
216
|
+
/**
|
|
217
|
+
* LENS-CHILD LIVENESS PROBE (dev only). The FOURTH failure mode, and the one that wedges an app in a
|
|
218
|
+
* PERMANENT 502 with nothing in the log: a gateway-spawned vite that is ALIVE but NOT LISTENING. Child
|
|
219
|
+
* health was modelled ONLY as "has the process exited", so a vite whose HTTP listener closes/faults while
|
|
220
|
+
* its event loop stays held open (by the lensmcp plugin's WS server + the bridge connections) is INVISIBLE:
|
|
221
|
+
* `viteChildren` still holds it as live, no `lens child vite exited` line is ever logged, and the
|
|
222
|
+
* exit-driven auto-heal never fires. Worked failure (foodguard, 2026-07-29): two 43-minute-old vite children
|
|
223
|
+
* (pids 89518/89519) that had both printed `VITE ready in 1005 ms` + `Local: http://localhost:4200/`, yet
|
|
224
|
+
* `lsof -nP -p 89519` showed ZERO LISTEN sockets and `lsof -iTCP:4200 -sTCP:LISTEN` was empty —
|
|
225
|
+
* `https://app.foodguard.local` 502'd indefinitely until the pids were hand-killed (the exit handler then
|
|
226
|
+
* respawned them and the app returned 200 within 8s). So the sweeper now TCP-probes each live child's pinned
|
|
227
|
+
* port and SIGKILLs a confirmed wedge, letting the proven exit→auto-heal path do the rest.
|
|
228
|
+
*/
|
|
229
|
+
/** Master switch for the liveness probe. Deliberately INDEPENDENT of {@link POD_RECYCLE_ENABLED}: a user who
|
|
230
|
+
* sets `LENSMCP_POD_RECYCLE=0` to stop staleness churn still wants a wedged 502 healed. Set
|
|
231
|
+
* `LENSMCP_LENS_PROBE=0` to disable. Default ON (dev). */
|
|
232
|
+
export declare const LENS_PROBE_ENABLED: boolean;
|
|
233
|
+
/** Spawn grace: a child is never probed (and so never recycled) until it has had this long to BIND its port
|
|
234
|
+
* — a cold vite/nx boot in a big monorepo takes many seconds, and killing a still-booting dev server would
|
|
235
|
+
* be a self-inflicted outage. Override `LENSMCP_LENS_PROBE_GRACE_MS`. Default 45s. */
|
|
236
|
+
export declare const LENS_PROBE_GRACE_MS: number;
|
|
237
|
+
/** CONSECUTIVE probe failures required before a recycle — one blip (a momentary accept backlog, a laptop
|
|
238
|
+
* sleep-wake, a GC pause) must never recycle a healthy dev server. Floored at 2 by
|
|
239
|
+
* {@link lensChildWedged}. Override `LENSMCP_LENS_PROBE_FAILS`. Default 2. */
|
|
240
|
+
export declare const LENS_PROBE_FAILS: number;
|
|
241
|
+
/** Per-probe TCP connect timeout. Short: a listening socket accepts a loopback connect in microseconds, so
|
|
242
|
+
* anything slower is indistinguishable from absent. Override `LENSMCP_LENS_PROBE_TIMEOUT_MS`. Default 1s. */
|
|
243
|
+
export declare const LENS_PROBE_TIMEOUT_MS: number;
|
|
244
|
+
/**
|
|
245
|
+
* RESPAWN-INTO-A-HELD-PORT (the EADDRINUSE self-inflicted crash). The auto-heal respawned after a FIXED
|
|
246
|
+
* backoff while the previous child still held the pinned port — vite's `--strictPort` then exits code=1 on
|
|
247
|
+
* bind, burning a heal attempt and EXTENDING the 502 window. Observed in the gateway log:
|
|
248
|
+
* `Error: listen EADDRINUSE: address already in use 127.0.0.1:50059` immediately followed by
|
|
249
|
+
* `lens child vite exited (code=1 sig=null); auto-healing in 3s (attempt 2).` (3 such events in one log).
|
|
250
|
+
* Two fixes: WAIT for the port to become bindable before spawning, and classify a bind-race exit as
|
|
251
|
+
* RETRYABLE so it does not consume the crash-loop budget a real code fault does.
|
|
252
|
+
*/
|
|
253
|
+
/** How long a respawn polls for its pinned port to become bindable before spawning anyway (past the budget
|
|
254
|
+
* the normal heal backoff takes over). Override `LENSMCP_PORT_FREE_WAIT_MS`. Default 10s. */
|
|
255
|
+
export declare const PORT_FREE_WAIT_MS: number;
|
|
256
|
+
/** Poll interval while waiting for a port to free. Override `LENSMCP_PORT_FREE_POLL_MS`. Default 250ms. */
|
|
257
|
+
export declare const PORT_FREE_POLL_MS: number;
|
|
258
|
+
/** A non-zero exit sooner than this is a START failure (it never bound); later means the child RAN and so
|
|
259
|
+
* had the port, which cannot be a bind race. Override `LENSMCP_CHILD_START_FAIL_MS`. Default 10s. */
|
|
260
|
+
export declare const CHILD_START_FAIL_MS: number;
|
|
261
|
+
/** Ultimate backstop: after this many CONSECUTIVE bind-race exits for one label, stop forgiving them and
|
|
262
|
+
* fall back to the normal exponential backoff (so a permanently-squatted port can't 10s-loop forever).
|
|
263
|
+
* Override `LENSMCP_MAX_PORT_CONFLICT_RETRIES`. Default 5. */
|
|
264
|
+
export declare const MAX_PORT_CONFLICT_RETRIES: number;
|
|
265
|
+
/** Inbound keep-alive idle window for the front door. MUST exceed every client's pooled-socket idle timeout
|
|
266
|
+
* so the SERVER never closes a keep-alive socket a client is about to reuse — otherwise both sides race the
|
|
267
|
+
* same deadline and the truncated request surfaces as a spurious front-door `400` (see
|
|
268
|
+
* `classifyClientError`). Node's server default is 5s and a Node client agent's is ~5s, i.e. exactly tied.
|
|
269
|
+
* Override `LENSMCP_KEEPALIVE_TIMEOUT_MS`. Default 76s. */
|
|
270
|
+
export declare const GATEWAY_KEEPALIVE_TIMEOUT_MS: number;
|
|
216
271
|
/** Test-tunable knobs on top of the user-facing schema. */
|
|
217
272
|
export interface GatewayRuntimeOptions extends GatewayExecutorSchema {
|
|
218
273
|
/** Traffic-edge flush period (ms). Default 5000. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,YAAY,EAAE,KAAK,EAAE,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sGAAsG;IACtG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B;;iGAE6F;IAC7F,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAE1F,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;oGACgG;IAChG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;gEAC4D;IAC5D,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;yBAGyB;AACzB,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB;;iHAE6G;IAC7G,IAAI,EAAE,MAAM,CAAC;IACb;yGACqG;IACrG,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;iGAE6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;6GAGyG;IACzG,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;mCAE+B;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;2EAGuE;IACvE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;yBAGqB;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAWD,qGAAqG;AACrG,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C;;wDAEwD;AACxD,eAAO,MAAM,oBAAoB,IAAI,CAAC;AACtC,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAEhD;;sGAEsG;AACtG,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;;;sDAOsD;AACtD,eAAO,MAAM,uBAAuB,QAGhC,CAAC;AAEL;;;;;;;iHAOiH;AACjH,eAAO,MAAM,sBAAsB,QAG/B,CAAC;AAEL;;;;;;;;;GASG;AACH,8GAA8G;AAC9G,eAAO,MAAM,mBAAmB,SAA6C,CAAC;AAC9E;;4CAE4C;AAC5C,eAAO,MAAM,cAAc,QAA0D,CAAC;AACtF;;0FAE0F;AAC1F,eAAO,MAAM,kBAAkB,QAA+C,CAAC;AAC/E;;mDAEmD;AACnD,eAAO,MAAM,qBAAqB,QAAiD,CAAC;AACpF;gEACgE;AAChE,eAAO,MAAM,iBAAiB,QAA8C,CAAC;AAC7E;mFACmF;AACnF,eAAO,MAAM,6BAA6B,QAA0D,CAAC;AAErG,2DAA2D;AAC3D,MAAM,WAAW,qBAAsB,SAAQ,qBAAqB;IAClE,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAGD,MAAM,WAAW,OAAO;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAGD;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;sDAEkD;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;+GAC2G;IAC3G,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;IACzB;8GAC0G;IAC1G,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,sGAAsG;IACtG,QAAQ,CAAC,aAAa,EAAE,MAAM,IAAI,CAAC;IACnC,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB;;;mHAG+G;IAC/G,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;0EAIsE;IACtE,mBAAmB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C;;qFAEiF;IACjF,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD;wFACoF;IACpF,OAAO,IAAI,OAAO,CAAC;CACpB;AAGD,2EAA2E;AAC3E,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,YAAY,GACZ,WAAW,GACX,eAAe,GACf,YAAY,GACZ,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAErE;6EAC6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;CAC3B;AAED;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,CAAC;IACnC,kDAAkD;IAClD,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,sFAAsF;IACtF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7E,gGAAgG;IAChG,SAAS,IAAI,OAAO,CAAC;CACtB;AAED;;;0DAG0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,OAAO,CAAC;IACZ,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,YAAY,EAAE,KAAK,EAAE,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sGAAsG;IACtG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B;;iGAE6F;IAC7F,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAE1F,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;oGACgG;IAChG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;gEAC4D;IAC5D,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;yBAGyB;AACzB,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB;;iHAE6G;IAC7G,IAAI,EAAE,MAAM,CAAC;IACb;yGACqG;IACrG,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;iGAE6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;6GAGyG;IACzG,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;mCAE+B;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;2EAGuE;IACvE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;yBAGqB;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAWD,qGAAqG;AACrG,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C;;wDAEwD;AACxD,eAAO,MAAM,oBAAoB,IAAI,CAAC;AACtC,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAEhD;;sGAEsG;AACtG,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;;;sDAOsD;AACtD,eAAO,MAAM,uBAAuB,QAGhC,CAAC;AAEL;;;;;;;iHAOiH;AACjH,eAAO,MAAM,sBAAsB,QAG/B,CAAC;AAEL;;;;;;;;;GASG;AACH,8GAA8G;AAC9G,eAAO,MAAM,mBAAmB,SAA6C,CAAC;AAC9E;;4CAE4C;AAC5C,eAAO,MAAM,cAAc,QAA0D,CAAC;AACtF;;0FAE0F;AAC1F,eAAO,MAAM,kBAAkB,QAA+C,CAAC;AAC/E;;mDAEmD;AACnD,eAAO,MAAM,qBAAqB,QAAiD,CAAC;AACpF;gEACgE;AAChE,eAAO,MAAM,iBAAiB,QAA8C,CAAC;AAC7E;mFACmF;AACnF,eAAO,MAAM,6BAA6B,QAA0D,CAAC;AAErG;;;;;;;;;;;;GAYG;AACH;;2DAE2D;AAC3D,eAAO,MAAM,kBAAkB,SAA4C,CAAC;AAC5E;;uFAEuF;AACvF,eAAO,MAAM,mBAAmB,QAAgD,CAAC;AACjF;;+EAE+E;AAC/E,eAAO,MAAM,gBAAgB,QAAwC,CAAC;AACtE;8GAC8G;AAC9G,eAAO,MAAM,qBAAqB,QAAiD,CAAC;AAEpF;;;;;;;;GAQG;AACH;8FAC8F;AAC9F,eAAO,MAAM,iBAAiB,QAA8C,CAAC;AAC7E,2GAA2G;AAC3G,eAAO,MAAM,iBAAiB,QAA2C,CAAC;AAC1E;sGACsG;AACtG,eAAO,MAAM,mBAAmB,QAAgD,CAAC;AACjF;;+DAE+D;AAC/D,eAAO,MAAM,yBAAyB,QAAiD,CAAC;AAExF;;;;4DAI4D;AAC5D,eAAO,MAAM,4BAA4B,QAAiD,CAAC;AAE3F,2DAA2D;AAC3D,MAAM,WAAW,qBAAsB,SAAQ,qBAAqB;IAClE,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAGD,MAAM,WAAW,OAAO;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAGD;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;sDAEkD;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;+GAC2G;IAC3G,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;IACzB;8GAC0G;IAC1G,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,sGAAsG;IACtG,QAAQ,CAAC,aAAa,EAAE,MAAM,IAAI,CAAC;IACnC,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB;;;mHAG+G;IAC/G,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;0EAIsE;IACtE,mBAAmB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C;;qFAEiF;IACjF,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD;wFACoF;IACpF,OAAO,IAAI,OAAO,CAAC;CACpB;AAGD,2EAA2E;AAC3E,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,YAAY,GACZ,WAAW,GACX,eAAe,GACf,YAAY,GACZ,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAErE;6EAC6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;CAC3B;AAED;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,CAAC;IACnC,kDAAkD;IAClD,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,sFAAsF;IACtF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7E,gGAAgG;IAChG,SAAS,IAAI,OAAO,CAAC;CACtB;AAED;;;0DAG0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,OAAO,CAAC;IACZ,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.POD_STALE_RECYCLE_COOLDOWN_MS = exports.POD_STALE_SCAN_MS = exports.POD_RECYCLE_SETTLE_MS = exports.POD_STALE_GRACE_MS = exports.POD_MAX_AGE_MS = exports.POD_RECYCLE_ENABLED = exports.WEDGE_RECYCLE_GRACE_MS = exports.POD_RESPONSE_TIMEOUT_MS = exports.UPSTREAM_HEAL_STEP_MS = exports.UPSTREAM_HEAL_WINDOW_MS = exports.SERVICE_RESTART_WINDOW_MS = exports.SERVICE_MAX_RESTARTS = exports.SERVICE_ERROR_THROTTLE_MS = void 0;
|
|
3
|
+
exports.GATEWAY_KEEPALIVE_TIMEOUT_MS = exports.MAX_PORT_CONFLICT_RETRIES = exports.CHILD_START_FAIL_MS = exports.PORT_FREE_POLL_MS = exports.PORT_FREE_WAIT_MS = exports.LENS_PROBE_TIMEOUT_MS = exports.LENS_PROBE_FAILS = exports.LENS_PROBE_GRACE_MS = exports.LENS_PROBE_ENABLED = exports.POD_STALE_RECYCLE_COOLDOWN_MS = exports.POD_STALE_SCAN_MS = exports.POD_RECYCLE_SETTLE_MS = exports.POD_STALE_GRACE_MS = exports.POD_MAX_AGE_MS = exports.POD_RECYCLE_ENABLED = exports.WEDGE_RECYCLE_GRACE_MS = exports.POD_RESPONSE_TIMEOUT_MS = exports.UPSTREAM_HEAL_STEP_MS = exports.UPSTREAM_HEAL_WINDOW_MS = exports.SERVICE_RESTART_WINDOW_MS = exports.SERVICE_MAX_RESTARTS = exports.SERVICE_ERROR_THROTTLE_MS = void 0;
|
|
4
4
|
/** Parse a positive-integer env override; fall back to `def` on missing/NaN/negative. `allowZero` lets a
|
|
5
5
|
* `0` through (used as a "disable" sentinel, e.g. {@link POD_MAX_AGE_MS}); otherwise `0` also falls back. */
|
|
6
6
|
function envInt(name, def, allowZero = false) {
|
|
@@ -77,3 +77,58 @@ exports.POD_STALE_SCAN_MS = envInt('LENSMCP_POD_STALE_SCAN_MS', 30_000);
|
|
|
77
77
|
/** Per-service cooldown between staleness recycles — stops a source-churn burst (codegen) thrash-recycling
|
|
78
78
|
* the same pod. Override `LENSMCP_POD_STALE_RECYCLE_COOLDOWN_MS`. Default 60s. */
|
|
79
79
|
exports.POD_STALE_RECYCLE_COOLDOWN_MS = envInt('LENSMCP_POD_STALE_RECYCLE_COOLDOWN_MS', 60_000);
|
|
80
|
+
/**
|
|
81
|
+
* LENS-CHILD LIVENESS PROBE (dev only). The FOURTH failure mode, and the one that wedges an app in a
|
|
82
|
+
* PERMANENT 502 with nothing in the log: a gateway-spawned vite that is ALIVE but NOT LISTENING. Child
|
|
83
|
+
* health was modelled ONLY as "has the process exited", so a vite whose HTTP listener closes/faults while
|
|
84
|
+
* its event loop stays held open (by the lensmcp plugin's WS server + the bridge connections) is INVISIBLE:
|
|
85
|
+
* `viteChildren` still holds it as live, no `lens child vite exited` line is ever logged, and the
|
|
86
|
+
* exit-driven auto-heal never fires. Worked failure (foodguard, 2026-07-29): two 43-minute-old vite children
|
|
87
|
+
* (pids 89518/89519) that had both printed `VITE ready in 1005 ms` + `Local: http://localhost:4200/`, yet
|
|
88
|
+
* `lsof -nP -p 89519` showed ZERO LISTEN sockets and `lsof -iTCP:4200 -sTCP:LISTEN` was empty —
|
|
89
|
+
* `https://app.foodguard.local` 502'd indefinitely until the pids were hand-killed (the exit handler then
|
|
90
|
+
* respawned them and the app returned 200 within 8s). So the sweeper now TCP-probes each live child's pinned
|
|
91
|
+
* port and SIGKILLs a confirmed wedge, letting the proven exit→auto-heal path do the rest.
|
|
92
|
+
*/
|
|
93
|
+
/** Master switch for the liveness probe. Deliberately INDEPENDENT of {@link POD_RECYCLE_ENABLED}: a user who
|
|
94
|
+
* sets `LENSMCP_POD_RECYCLE=0` to stop staleness churn still wants a wedged 502 healed. Set
|
|
95
|
+
* `LENSMCP_LENS_PROBE=0` to disable. Default ON (dev). */
|
|
96
|
+
exports.LENS_PROBE_ENABLED = process.env['LENSMCP_LENS_PROBE'] !== '0';
|
|
97
|
+
/** Spawn grace: a child is never probed (and so never recycled) until it has had this long to BIND its port
|
|
98
|
+
* — a cold vite/nx boot in a big monorepo takes many seconds, and killing a still-booting dev server would
|
|
99
|
+
* be a self-inflicted outage. Override `LENSMCP_LENS_PROBE_GRACE_MS`. Default 45s. */
|
|
100
|
+
exports.LENS_PROBE_GRACE_MS = envInt('LENSMCP_LENS_PROBE_GRACE_MS', 45_000);
|
|
101
|
+
/** CONSECUTIVE probe failures required before a recycle — one blip (a momentary accept backlog, a laptop
|
|
102
|
+
* sleep-wake, a GC pause) must never recycle a healthy dev server. Floored at 2 by
|
|
103
|
+
* {@link lensChildWedged}. Override `LENSMCP_LENS_PROBE_FAILS`. Default 2. */
|
|
104
|
+
exports.LENS_PROBE_FAILS = envInt('LENSMCP_LENS_PROBE_FAILS', 2);
|
|
105
|
+
/** Per-probe TCP connect timeout. Short: a listening socket accepts a loopback connect in microseconds, so
|
|
106
|
+
* anything slower is indistinguishable from absent. Override `LENSMCP_LENS_PROBE_TIMEOUT_MS`. Default 1s. */
|
|
107
|
+
exports.LENS_PROBE_TIMEOUT_MS = envInt('LENSMCP_LENS_PROBE_TIMEOUT_MS', 1_000);
|
|
108
|
+
/**
|
|
109
|
+
* RESPAWN-INTO-A-HELD-PORT (the EADDRINUSE self-inflicted crash). The auto-heal respawned after a FIXED
|
|
110
|
+
* backoff while the previous child still held the pinned port — vite's `--strictPort` then exits code=1 on
|
|
111
|
+
* bind, burning a heal attempt and EXTENDING the 502 window. Observed in the gateway log:
|
|
112
|
+
* `Error: listen EADDRINUSE: address already in use 127.0.0.1:50059` immediately followed by
|
|
113
|
+
* `lens child vite exited (code=1 sig=null); auto-healing in 3s (attempt 2).` (3 such events in one log).
|
|
114
|
+
* Two fixes: WAIT for the port to become bindable before spawning, and classify a bind-race exit as
|
|
115
|
+
* RETRYABLE so it does not consume the crash-loop budget a real code fault does.
|
|
116
|
+
*/
|
|
117
|
+
/** How long a respawn polls for its pinned port to become bindable before spawning anyway (past the budget
|
|
118
|
+
* the normal heal backoff takes over). Override `LENSMCP_PORT_FREE_WAIT_MS`. Default 10s. */
|
|
119
|
+
exports.PORT_FREE_WAIT_MS = envInt('LENSMCP_PORT_FREE_WAIT_MS', 10_000);
|
|
120
|
+
/** Poll interval while waiting for a port to free. Override `LENSMCP_PORT_FREE_POLL_MS`. Default 250ms. */
|
|
121
|
+
exports.PORT_FREE_POLL_MS = envInt('LENSMCP_PORT_FREE_POLL_MS', 250);
|
|
122
|
+
/** A non-zero exit sooner than this is a START failure (it never bound); later means the child RAN and so
|
|
123
|
+
* had the port, which cannot be a bind race. Override `LENSMCP_CHILD_START_FAIL_MS`. Default 10s. */
|
|
124
|
+
exports.CHILD_START_FAIL_MS = envInt('LENSMCP_CHILD_START_FAIL_MS', 10_000);
|
|
125
|
+
/** Ultimate backstop: after this many CONSECUTIVE bind-race exits for one label, stop forgiving them and
|
|
126
|
+
* fall back to the normal exponential backoff (so a permanently-squatted port can't 10s-loop forever).
|
|
127
|
+
* Override `LENSMCP_MAX_PORT_CONFLICT_RETRIES`. Default 5. */
|
|
128
|
+
exports.MAX_PORT_CONFLICT_RETRIES = envInt('LENSMCP_MAX_PORT_CONFLICT_RETRIES', 5);
|
|
129
|
+
/** Inbound keep-alive idle window for the front door. MUST exceed every client's pooled-socket idle timeout
|
|
130
|
+
* so the SERVER never closes a keep-alive socket a client is about to reuse — otherwise both sides race the
|
|
131
|
+
* same deadline and the truncated request surfaces as a spurious front-door `400` (see
|
|
132
|
+
* `classifyClientError`). Node's server default is 5s and a Node client agent's is ~5s, i.e. exactly tied.
|
|
133
|
+
* Override `LENSMCP_KEEPALIVE_TIMEOUT_MS`. Default 76s. */
|
|
134
|
+
exports.GATEWAY_KEEPALIVE_TIMEOUT_MS = envInt('LENSMCP_KEEPALIVE_TIMEOUT_MS', 76_000);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lensmcp/cluster",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.17.0",
|
|
4
4
|
"description": "Run your Nx workspace as a local production cluster: pods on unix sockets with true HMR, one https gateway with per-project domains, scale-from-zero, autoscale, idle-kill, local-CA TLS — observed by the LensMCP lens.",
|
|
5
5
|
"main": "./index.js",
|
|
6
6
|
"types": "./index.d.ts",
|
|
@@ -65,8 +65,8 @@
|
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
67
|
"dependencies": {
|
|
68
|
-
"@lensmcp/node-instrumentation": "1.
|
|
69
|
-
"@lensmcp/nx-plugin": "1.
|
|
68
|
+
"@lensmcp/node-instrumentation": "1.17.0",
|
|
69
|
+
"@lensmcp/nx-plugin": "1.17.0",
|
|
70
70
|
"fork-ts-checker-webpack-plugin": "^9.0.0",
|
|
71
71
|
"glob": "^11.0.0",
|
|
72
72
|
"http-proxy": "^1.18.0",
|