@lensmcp/cluster 1.16.4 → 1.16.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/executors/gateway/runtime/lens-children.d.ts.map +1 -1
- package/executors/gateway/runtime/lens-children.js +51 -0
- package/executors/gateway/runtime/lifecycle.d.ts +31 -0
- package/executors/gateway/runtime/lifecycle.d.ts.map +1 -1
- package/executors/gateway/runtime/lifecycle.js +153 -1
- package/executors/gateway/runtime/proxy.d.ts.map +1 -1
- package/executors/gateway/runtime/proxy.js +7 -3
- package/executors/gateway/runtime/types.d.ts +40 -0
- package/executors/gateway/runtime/types.d.ts.map +1 -1
- package/executors/gateway/runtime/types.js +41 -1
- package/package.json +3 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lens-children.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lens-children.ts"],"names":[],"mappings":"AAQA,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,kCAAkC,CAAC;
|
|
1
|
+
{"version":3,"file":"lens-children.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lens-children.ts"],"names":[],"mappings":"AAQA,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,kCAAkC,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,qBAAqB,EAAgB,MAAM,SAAS,CAAC;AAQnF,OAAO,EAA0B,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAM7E,MAAM,WAAW,YAAY;IAC3B,iBAAiB,EAAE,UAAU,CAAC;IAC9B;iFAC6E;IAC7E,qBAAqB,IAAI,IAAI,CAAC;IAC9B,wDAAwD;IACxD,OAAO,IAAI,IAAI,CAAC;CACjB;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,qBAAqB,GAAG,YAAY,CAiKvH"}
|
|
@@ -11,6 +11,8 @@ const node_child_process_1 = require("node:child_process");
|
|
|
11
11
|
const manifest_1 = require("../manifest");
|
|
12
12
|
const lens_frontend_1 = require("@lensmcp/nx-plugin/lens-frontend");
|
|
13
13
|
const scope_1 = require("./scope");
|
|
14
|
+
const lifecycle_1 = require("./lifecycle");
|
|
15
|
+
const types_1 = require("./types");
|
|
14
16
|
const observability_1 = require("./observability");
|
|
15
17
|
const HEAL_BASE_MS = 1_500; // first restart delay
|
|
16
18
|
const HEAL_MAX_MS = 30_000; // backoff cap — a true boot-loop settles here, never hot-loops
|
|
@@ -21,6 +23,13 @@ function createLensChildren(rt, obs, options) {
|
|
|
21
23
|
// Auto-heal bookkeeping per child label: consecutive restart attempts (drives the exponential
|
|
22
24
|
// backoff) so a boot-loop settles instead of hammering, reset once a child has run healthily.
|
|
23
25
|
const lensHeal = new Map();
|
|
26
|
+
// Zombie recycle: the live `lens:true` app VITE children (label 'vite') + when each spawned, so the lens
|
|
27
|
+
// sweeper can restart one whose fs-watcher missed a post-spawn file add (the reported dashboard zombie).
|
|
28
|
+
// The capture sidecar + the dashboard/MCP singletons are NOT tracked (they don't serve app source).
|
|
29
|
+
const viteChildren = new Map();
|
|
30
|
+
// Per-project last-recycle time — the cooldown survives the kill→auto-heal→respawn (a new child handle).
|
|
31
|
+
const viteRecycleAt = new Map();
|
|
32
|
+
let lensSweeper;
|
|
24
33
|
// A `SpawnChild` over the gateway's managed-children array — the SAME shape `agent-dev` passes to
|
|
25
34
|
// `spawnLensFrontend`. EVERY gateway-managed child AUTO-HEALS: the dashboard + MCP singletons AND each
|
|
26
35
|
// `lens:true` vite app are restarted when they exit while the gateway is up, with exponential backoff.
|
|
@@ -34,10 +43,15 @@ function createLensChildren(rt, obs, options) {
|
|
|
34
43
|
const spawnedAt = Date.now();
|
|
35
44
|
const child = (0, node_child_process_1.spawn)(bin, args, { cwd: rt.root, env: { ...process.env, ...(env ?? {}) }, stdio: 'inherit' });
|
|
36
45
|
lensChildren.push(child);
|
|
46
|
+
// Track a `lens:true` app's vite dev server (label 'vite') for the zombie recycle. Its `LENSMCP_PROJECT`
|
|
47
|
+
// (set by lens-children when it spawns the frontend) names the app; a fresh respawn re-adds via this path.
|
|
48
|
+
if (label === 'vite')
|
|
49
|
+
viteChildren.set(child, { project: env?.['LENSMCP_PROJECT'] ?? 'app', spawnedAt });
|
|
37
50
|
child.on('exit', (code, sig) => {
|
|
38
51
|
const idx = lensChildren.indexOf(child);
|
|
39
52
|
if (idx >= 0)
|
|
40
53
|
lensChildren.splice(idx, 1); // drop the dead handle so stop() won't signal it
|
|
54
|
+
viteChildren.delete(child); // the auto-heal respawn re-registers a fresh handle with a new spawnedAt
|
|
41
55
|
if (rt.stopped())
|
|
42
56
|
return; // the gateway is shutting down — do NOT resurrect
|
|
43
57
|
const heal = lensHeal.get(label) ?? { attempts: 0 };
|
|
@@ -53,6 +67,38 @@ function createLensChildren(rt, obs, options) {
|
|
|
53
67
|
});
|
|
54
68
|
return child;
|
|
55
69
|
};
|
|
70
|
+
// ZOMBIE recycle for lens frontends: restart a `lens:true` app's vite whose fs-watcher went stale — it is
|
|
71
|
+
// ALIVE + reachable yet MISSED files created after it spawned (500 `Failed to resolve import` for a file
|
|
72
|
+
// that exists; the reported 17.7h dashboard pod). The shared `rt.sourceSetChangedAt` (stamped by the
|
|
73
|
+
// lifecycle sweeper's source-set scan) drives it: a vite that spawned before a settled set change is
|
|
74
|
+
// killed → this manager's own `on('exit')` auto-heal respawns a fresh vite that re-scans the FS. Only on a
|
|
75
|
+
// set CHANGE (add/remove), never a content save, and never a blind max-age (no idle signal for a frontend).
|
|
76
|
+
const startLensSweeper = () => {
|
|
77
|
+
if (!types_1.POD_RECYCLE_ENABLED || lensSweeper)
|
|
78
|
+
return;
|
|
79
|
+
const cfg = { graceMs: types_1.POD_STALE_GRACE_MS, settleMs: types_1.POD_RECYCLE_SETTLE_MS, cooldownMs: types_1.POD_STALE_RECYCLE_COOLDOWN_MS };
|
|
80
|
+
lensSweeper = setInterval(() => {
|
|
81
|
+
if (rt.stopped())
|
|
82
|
+
return;
|
|
83
|
+
const changedAt = rt.sourceSetChangedAt ?? 0;
|
|
84
|
+
if (changedAt <= 0 || viteChildren.size === 0)
|
|
85
|
+
return;
|
|
86
|
+
const now = Date.now();
|
|
87
|
+
for (const [child, meta] of viteChildren) {
|
|
88
|
+
const last = viteRecycleAt.get(meta.project) ?? 0;
|
|
89
|
+
if (!(0, lifecycle_1.lensFrontendStale)(meta.spawnedAt, last, changedAt, now, cfg))
|
|
90
|
+
continue;
|
|
91
|
+
viteRecycleAt.set(meta.project, now);
|
|
92
|
+
console.log(`[gateway] zombie recycle: lens app ${meta.project} (stale-source-set) — vite missed a file add; restarting it`);
|
|
93
|
+
emit('warning', `zombie recycle: ${meta.project} (stale-source-set)`, `cluster-lens-app:${meta.project}`, { kind: 'pod-recycle', project: meta.project, reason: 'stale-source-set' });
|
|
94
|
+
try {
|
|
95
|
+
child.kill('SIGTERM');
|
|
96
|
+
}
|
|
97
|
+
catch { /* already gone — the exit auto-heal will respawn */ }
|
|
98
|
+
}
|
|
99
|
+
}, Math.min(types_1.POD_STALE_SCAN_MS, 15_000));
|
|
100
|
+
lensSweeper.unref?.();
|
|
101
|
+
};
|
|
56
102
|
const bootSingletonsAndApps = () => {
|
|
57
103
|
const lensApps = [...new Map(rt.routes.filter((r) => r.lens).map((r) => [r.project, r.lens])).values()];
|
|
58
104
|
const wantDashboard = options.dashboard !== false;
|
|
@@ -124,8 +170,13 @@ function createLensChildren(rt, obs, options) {
|
|
|
124
170
|
console.error(`[gateway] lens app ${app.project}: ${e.message}`);
|
|
125
171
|
}
|
|
126
172
|
}
|
|
173
|
+
startLensSweeper(); // watch the just-spawned lens vites for a stale-source-set zombie
|
|
127
174
|
};
|
|
128
175
|
const reapAll = () => {
|
|
176
|
+
if (lensSweeper) {
|
|
177
|
+
clearInterval(lensSweeper);
|
|
178
|
+
lensSweeper = undefined;
|
|
179
|
+
}
|
|
129
180
|
for (const c of lensChildren) {
|
|
130
181
|
try {
|
|
131
182
|
c.kill('SIGTERM');
|
|
@@ -1,5 +1,36 @@
|
|
|
1
1
|
import type { GatewayRuntime, ServiceCtl, SockPool } from './types';
|
|
2
2
|
import type { Observability } from './observability';
|
|
3
|
+
/** Bounded, async, non-blocking walk of the workspace source tree → a signature of the file SET
|
|
4
|
+
* (`<count>:<xor-of-path-hashes>`). Skips node_modules/dist/build/dotdirs + `*-devserver` sock dirs, and
|
|
5
|
+
* only counts source-extension files. Order-independent (readdir order irrelevant) and null-safe. */
|
|
6
|
+
export declare function sourceSetSignature(root: string): Promise<string>;
|
|
7
|
+
export interface PodRecycleConfig {
|
|
8
|
+
maxAgeMs: number;
|
|
9
|
+
graceMs: number;
|
|
10
|
+
settleMs: number;
|
|
11
|
+
cooldownMs: number;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* PURE decision: should this pod be recycled as a stale-view zombie, and why? A pod qualifies only when it
|
|
15
|
+
* is a live, UP, gateway-spawned pod that is currently QUIET (0 in-flight AND idle for `settleMs`, so a busy
|
|
16
|
+
* pod is never interrupted) and past its per-service cooldown. Then it recycles if the source file SET
|
|
17
|
+
* changed after it spawned (`stale-source-set`) or as a plain age backstop (`max-age`). Returns `null` when
|
|
18
|
+
* it must be left alone. Kept pure (no `ServiceCtl` mutation, no fs) so the sweeper's logic is unit-tested
|
|
19
|
+
* without spawning a process. `sourceSetChangedAt` is 0 until the gateway first observes a set change.
|
|
20
|
+
*/
|
|
21
|
+
export declare function podRecycleReason(svc: Pick<ServiceCtl, 'proc' | 'state' | 'spawnAt' | 'inflight' | 'lastUsed' | 'lastStaleRecycleAt'>, now: number, sourceSetChangedAt: number, cfg: PodRecycleConfig): 'stale-source-set' | 'max-age' | null;
|
|
22
|
+
/**
|
|
23
|
+
* PURE decision for a LENS-FRONTEND vite child (a `lens:true` app's dev server — the reported zombie). It
|
|
24
|
+
* has no in-flight/idle signal like a pod, so it recycles ONLY on a settled source-file-SET change that
|
|
25
|
+
* POST-dates its spawn (the precise "watcher missed a new file" trigger) — never a blind max-age (which
|
|
26
|
+
* could interrupt active work). `killing` it lets the manager's `on('exit')` auto-heal respawn a fresh vite
|
|
27
|
+
* that re-scans the FS. Kept pure + exported so the recycle logic is unit-tested without a process.
|
|
28
|
+
*/
|
|
29
|
+
export declare function lensFrontendStale(spawnedAt: number, lastRecycleAt: number, sourceSetChangedAt: number, now: number, cfg: {
|
|
30
|
+
graceMs: number;
|
|
31
|
+
settleMs: number;
|
|
32
|
+
cooldownMs: number;
|
|
33
|
+
}): boolean;
|
|
3
34
|
export interface ServiceLayer {
|
|
4
35
|
scanNow(pool: SockPool): number;
|
|
5
36
|
spawnService(svc: ServiceCtl): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAapE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA4CrD;;sGAEsG;AACtG,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuBtE;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,oBAAoB,CAAC,EACpG,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,MAAM,EAC1B,GAAG,EAAE,gBAAgB,GACpB,kBAAkB,GAAG,SAAS,GAAG,IAAI,CAQvC;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,kBAAkB,EAAE,MAAM,EAC1B,GAAG,EAAE,MAAM,EACX,GAAG,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC7D,OAAO,CAMT;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,CA2PvF"}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.sourceSetSignature = sourceSetSignature;
|
|
4
|
+
exports.podRecycleReason = podRecycleReason;
|
|
5
|
+
exports.lensFrontendStale = lensFrontendStale;
|
|
3
6
|
exports.createServiceLayer = createServiceLayer;
|
|
4
7
|
const tslib_1 = require("tslib");
|
|
5
8
|
/**
|
|
@@ -38,6 +41,103 @@ function eastWestEnv(rt, project) {
|
|
|
38
41
|
return env;
|
|
39
42
|
}
|
|
40
43
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
44
|
+
// --- zombie-pod recycle: source-file-SET signature -------------------------------------------------------
|
|
45
|
+
// A pod goes "stale" when the set of source files CHANGES (a file added/removed) after it spawned and its
|
|
46
|
+
// degraded watcher missed it. We detect a set change — NOT a content change — by a cheap order-independent
|
|
47
|
+
// signature over source file PATHS (count + xor of per-path hashes): adding/removing a path flips the
|
|
48
|
+
// signature; editing a file's CONTENTS does not, so ordinary saves never trigger a recycle.
|
|
49
|
+
const SRC_EXCLUDE = new Set(['node_modules', 'dist', 'tmp', 'coverage', 'build', 'out', 'target']);
|
|
50
|
+
const SRC_EXT = new Set([
|
|
51
|
+
'.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.css', '.scss', '.sass', '.less',
|
|
52
|
+
'.json', '.html', '.htm', '.vue', '.svelte', '.mdx',
|
|
53
|
+
]);
|
|
54
|
+
const SCAN_FILE_CAP = 100_000;
|
|
55
|
+
function fnv1a(s) {
|
|
56
|
+
let h = 2166136261;
|
|
57
|
+
for (let i = 0; i < s.length; i += 1) {
|
|
58
|
+
h ^= s.charCodeAt(i);
|
|
59
|
+
h = Math.imul(h, 16777619);
|
|
60
|
+
}
|
|
61
|
+
return h >>> 0;
|
|
62
|
+
}
|
|
63
|
+
/** Bounded, async, non-blocking walk of the workspace source tree → a signature of the file SET
|
|
64
|
+
* (`<count>:<xor-of-path-hashes>`). Skips node_modules/dist/build/dotdirs + `*-devserver` sock dirs, and
|
|
65
|
+
* only counts source-extension files. Order-independent (readdir order irrelevant) and null-safe. */
|
|
66
|
+
async function sourceSetSignature(root) {
|
|
67
|
+
let count = 0;
|
|
68
|
+
let xor = 0;
|
|
69
|
+
const walk = async (dir) => {
|
|
70
|
+
if (count > SCAN_FILE_CAP)
|
|
71
|
+
return;
|
|
72
|
+
let entries;
|
|
73
|
+
try {
|
|
74
|
+
entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
for (const e of entries) {
|
|
80
|
+
if (count > SCAN_FILE_CAP)
|
|
81
|
+
return;
|
|
82
|
+
const name = e.name;
|
|
83
|
+
if (e.isDirectory()) {
|
|
84
|
+
if (name.startsWith('.') || SRC_EXCLUDE.has(name) || name.endsWith('-devserver'))
|
|
85
|
+
continue;
|
|
86
|
+
await walk(path.join(dir, name));
|
|
87
|
+
}
|
|
88
|
+
else if (e.isFile()) {
|
|
89
|
+
const dot = name.lastIndexOf('.');
|
|
90
|
+
if (dot <= 0 || !SRC_EXT.has(name.slice(dot)))
|
|
91
|
+
continue;
|
|
92
|
+
count += 1;
|
|
93
|
+
xor ^= fnv1a(path.join(dir, name));
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
await walk(root);
|
|
98
|
+
return `${count}:${(xor >>> 0).toString(36)}`;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* PURE decision: should this pod be recycled as a stale-view zombie, and why? A pod qualifies only when it
|
|
102
|
+
* is a live, UP, gateway-spawned pod that is currently QUIET (0 in-flight AND idle for `settleMs`, so a busy
|
|
103
|
+
* pod is never interrupted) and past its per-service cooldown. Then it recycles if the source file SET
|
|
104
|
+
* changed after it spawned (`stale-source-set`) or as a plain age backstop (`max-age`). Returns `null` when
|
|
105
|
+
* it must be left alone. Kept pure (no `ServiceCtl` mutation, no fs) so the sweeper's logic is unit-tested
|
|
106
|
+
* without spawning a process. `sourceSetChangedAt` is 0 until the gateway first observes a set change.
|
|
107
|
+
*/
|
|
108
|
+
function podRecycleReason(svc, now, sourceSetChangedAt, cfg) {
|
|
109
|
+
if (!svc.proc || svc.state !== 'up' || svc.spawnAt == null)
|
|
110
|
+
return null; // only a live, up, spawned pod
|
|
111
|
+
if (svc.inflight > 0)
|
|
112
|
+
return null; // never mid-request
|
|
113
|
+
if (now - svc.lastUsed < cfg.settleMs)
|
|
114
|
+
return null; // wait for an inter-request lull
|
|
115
|
+
if (svc.lastStaleRecycleAt != null && now - svc.lastStaleRecycleAt < cfg.cooldownMs)
|
|
116
|
+
return null; // anti-thrash
|
|
117
|
+
if (sourceSetChangedAt > 0 && sourceSetChangedAt - svc.spawnAt > cfg.graceMs)
|
|
118
|
+
return 'stale-source-set';
|
|
119
|
+
if (cfg.maxAgeMs > 0 && now - svc.spawnAt > cfg.maxAgeMs)
|
|
120
|
+
return 'max-age';
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* PURE decision for a LENS-FRONTEND vite child (a `lens:true` app's dev server — the reported zombie). It
|
|
125
|
+
* has no in-flight/idle signal like a pod, so it recycles ONLY on a settled source-file-SET change that
|
|
126
|
+
* POST-dates its spawn (the precise "watcher missed a new file" trigger) — never a blind max-age (which
|
|
127
|
+
* could interrupt active work). `killing` it lets the manager's `on('exit')` auto-heal respawn a fresh vite
|
|
128
|
+
* that re-scans the FS. Kept pure + exported so the recycle logic is unit-tested without a process.
|
|
129
|
+
*/
|
|
130
|
+
function lensFrontendStale(spawnedAt, lastRecycleAt, sourceSetChangedAt, now, cfg) {
|
|
131
|
+
if (now - lastRecycleAt < cfg.cooldownMs)
|
|
132
|
+
return false; // anti-thrash on a source-churn burst
|
|
133
|
+
if (sourceSetChangedAt <= 0)
|
|
134
|
+
return false; // no set change observed yet
|
|
135
|
+
if (sourceSetChangedAt - spawnedAt <= cfg.graceMs)
|
|
136
|
+
return false; // this vite already postdates the change
|
|
137
|
+
if (now - sourceSetChangedAt < cfg.settleMs)
|
|
138
|
+
return false; // let an add/remove burst settle → one recycle
|
|
139
|
+
return true;
|
|
140
|
+
}
|
|
41
141
|
function createServiceLayer(rt, obs) {
|
|
42
142
|
const { emit } = obs;
|
|
43
143
|
const scanNow = (pool) => { pool.scannedAt = 0; return (0, discovery_1.pickSock)(pool, rt.scanTtlMs) ? pool.socks.length : 0; };
|
|
@@ -243,8 +343,51 @@ function createServiceLayer(rt, obs) {
|
|
|
243
343
|
spawnService(svc);
|
|
244
344
|
}, 1500).unref?.();
|
|
245
345
|
};
|
|
346
|
+
// --- zombie-pod recycle: source-set-change tracking + a fresh-pod recycle for a live-but-stale pod ------
|
|
347
|
+
const recycleCfg = {
|
|
348
|
+
maxAgeMs: types_1.POD_MAX_AGE_MS, graceMs: types_1.POD_STALE_GRACE_MS,
|
|
349
|
+
settleMs: types_1.POD_RECYCLE_SETTLE_MS, cooldownMs: types_1.POD_STALE_RECYCLE_COOLDOWN_MS,
|
|
350
|
+
};
|
|
351
|
+
let prevSig;
|
|
352
|
+
let scanInFlight = false;
|
|
353
|
+
let lastScanAt = 0;
|
|
354
|
+
// Kick a bounded, async source-set scan when due; on a SET change (file added/removed) stamp the shared
|
|
355
|
+
// clock so pods that predate it recycle. Non-blocking — the sweeper never awaits it.
|
|
356
|
+
const maybeScanSourceSet = () => {
|
|
357
|
+
if (scanInFlight || Date.now() - lastScanAt < types_1.POD_STALE_SCAN_MS)
|
|
358
|
+
return;
|
|
359
|
+
scanInFlight = true;
|
|
360
|
+
lastScanAt = Date.now();
|
|
361
|
+
void sourceSetSignature(rt.root)
|
|
362
|
+
.then((sig) => {
|
|
363
|
+
if (prevSig !== undefined && sig !== prevSig) {
|
|
364
|
+
rt.sourceSetChangedAt = Date.now();
|
|
365
|
+
emit('info', 'source file set changed — stale pods recycle when idle', 'cluster-gateway', { kind: 'source-set-changed' });
|
|
366
|
+
}
|
|
367
|
+
prevSig = sig;
|
|
368
|
+
})
|
|
369
|
+
.catch(() => undefined)
|
|
370
|
+
.finally(() => { scanInFlight = false; });
|
|
371
|
+
};
|
|
372
|
+
// Recycle a live-but-STALE-VIEW pod (alive + reachable, yet its watcher missed post-spawn file adds).
|
|
373
|
+
// Kill it → scale-from-zero (or eager respawn) → a fresh pod re-scans the FS. NOT counted against the
|
|
374
|
+
// crash-loop cap (staleness ≠ crash); the per-service cooldown ({@link podRecycleReason}) stops thrash.
|
|
375
|
+
const recyclePod = (svc, reason) => {
|
|
376
|
+
svc.lastStaleRecycleAt = Date.now();
|
|
377
|
+
console.log(`[gateway] zombie recycle: ${svc.project} (${reason}) — stale fs view; fresh pod incoming`);
|
|
378
|
+
emit('warning', `zombie recycle: ${svc.project} (${reason})`, `cluster-service:${svc.project}`, { kind: 'pod-recycle', project: svc.project, reason });
|
|
379
|
+
const eager = !!svc.decl.eager;
|
|
380
|
+
killSpawned(svc, `${reason} — recycling stale pod`); // clears proc, state='down', unlinks socks (+1200ms)
|
|
381
|
+
if (eager)
|
|
382
|
+
setTimeout(() => { if (!rt.stopped() && !svc.proc && svc.state === 'down')
|
|
383
|
+
spawnService(svc); }, 1500).unref?.();
|
|
384
|
+
};
|
|
246
385
|
const startSweeper = () => {
|
|
247
386
|
const sweeper = setInterval(() => {
|
|
387
|
+
if (rt.stopped())
|
|
388
|
+
return;
|
|
389
|
+
if (types_1.POD_RECYCLE_ENABLED)
|
|
390
|
+
maybeScanSourceSet();
|
|
248
391
|
for (const svc of rt.services) {
|
|
249
392
|
if (rt.stopped() || !svc.proc) {
|
|
250
393
|
svc.wedgedSince = undefined;
|
|
@@ -259,8 +402,17 @@ function createServiceLayer(rt, obs) {
|
|
|
259
402
|
}
|
|
260
403
|
// WEDGE detection: a live parent with 0 live pods that is NOT legitimately cold-starting is wedged.
|
|
261
404
|
const coldStarting = svc.state === 'starting' && svc.spawnAt !== undefined && Date.now() - svc.spawnAt < rt.startupTimeoutMs;
|
|
262
|
-
|
|
405
|
+
const live = scanNow(svc.pool) > 0;
|
|
406
|
+
if (live || coldStarting) {
|
|
263
407
|
svc.wedgedSince = undefined;
|
|
408
|
+
// ZOMBIE recycle: a live pod (live parent + live socket, answers requests) whose fs view went
|
|
409
|
+
// stale — neither exit-respawn nor wedge-recycle sees it. Idle-gated in podRecycleReason so a busy
|
|
410
|
+
// pod is never interrupted.
|
|
411
|
+
if (live && !coldStarting && types_1.POD_RECYCLE_ENABLED) {
|
|
412
|
+
const reason = podRecycleReason(svc, Date.now(), rt.sourceSetChangedAt ?? 0, recycleCfg);
|
|
413
|
+
if (reason)
|
|
414
|
+
recyclePod(svc, reason);
|
|
415
|
+
}
|
|
264
416
|
continue;
|
|
265
417
|
}
|
|
266
418
|
svc.wedgedSince ??= Date.now();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/proxy.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,cAAc,EAAW,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA8BhD,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACxH,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACnI,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,GAAG,IAAI,CAAC;IACrG,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,OAAO,YAAY,EAAE,KAAK,CAAC;IACzE,aAAa,IAAI,IAAI,CAAC;IACtB,UAAU,IAAI,IAAI,CAAC;CACpB;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,GAAG,UAAU,
|
|
1
|
+
{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/proxy.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,cAAc,EAAW,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA8BhD,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACxH,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACnI,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,GAAG,IAAI,CAAC;IACrG,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,OAAO,YAAY,EAAE,KAAK,CAAC;IACzE,aAAa,IAAI,IAAI,CAAC;IACtB,UAAU,IAAI,IAAI,CAAC;CACpB;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,GAAG,UAAU,CAgTtG"}
|
|
@@ -144,9 +144,13 @@ function createProxy(rt, obs, svcLayer) {
|
|
|
144
144
|
upstreamRes.pipe(res);
|
|
145
145
|
// The upstream dies MID-BODY (a pod rolling-swap or a Vite restart resets the pooled socket): RST
|
|
146
146
|
// this h2 stream cleanly instead of leaving it half-written — a dangling h2 stream is what Chrome
|
|
147
|
-
// reports as ERR_HTTP2_PROTOCOL_ERROR.
|
|
148
|
-
|
|
149
|
-
|
|
147
|
+
// reports as ERR_HTTP2_PROTOCOL_ERROR. A premature close surfaces as 'error' OR just 'close'
|
|
148
|
+
// ('aborted') depending on the Node version, so handle BOTH; the `writableEnded` guard makes the
|
|
149
|
+
// normal-completion 'close' a no-op. (An unhandled 'error' on upstreamRes would also throw.)
|
|
150
|
+
const abortEdge = () => { if (!res.writableEnded && !res.destroyed)
|
|
151
|
+
res.destroy(); };
|
|
152
|
+
upstreamRes.on('error', abortEdge);
|
|
153
|
+
upstreamRes.on('close', abortEdge);
|
|
150
154
|
// The client canceled: stop reading the upstream so its pooled keep-alive socket isn't left flowing
|
|
151
155
|
// (a half-drained socket is corrupt on the next reuse → would break a later request).
|
|
152
156
|
res.on('close', () => { if (!upstreamRes.destroyed)
|
|
@@ -131,6 +131,11 @@ export interface ServiceCtl {
|
|
|
131
131
|
* cold-starting). Cleared once a pod appears; once it exceeds {@link WEDGE_RECYCLE_GRACE_MS} the sweeper
|
|
132
132
|
* recycles the wedged tree. */
|
|
133
133
|
wedgedSince?: number;
|
|
134
|
+
/** Zombie recycle: when this pod was last recycled for STALENESS/age (a live-but-stale-view pod, distinct
|
|
135
|
+
* from a crash or a wedge). A per-service cooldown ({@link POD_STALE_RECYCLE_COOLDOWN_MS}) keyed off this
|
|
136
|
+
* stops a rapid source-churn (e.g. a codegen run) from thrash-recycling the same pod. NOT counted against
|
|
137
|
+
* the crash-loop cap ({@link restarts}) — staleness is not a crash. */
|
|
138
|
+
lastStaleRecycleAt?: number;
|
|
134
139
|
}
|
|
135
140
|
/** Min interval (ms) between repeat 'service error' emits for the SAME service + same error line. */
|
|
136
141
|
export declare const SERVICE_ERROR_THROTTLE_MS = 5000;
|
|
@@ -162,6 +167,36 @@ export declare const POD_RESPONSE_TIMEOUT_MS: number;
|
|
|
162
167
|
* legit in-process hot-swap (the socket is briefly gone, then re-bound). A code (build) error is left to
|
|
163
168
|
* the user's fix but still recycled under the crash-loop cap. Override with `LENSMCP_WEDGE_RECYCLE_GRACE_MS`. */
|
|
164
169
|
export declare const WEDGE_RECYCLE_GRACE_MS: number;
|
|
170
|
+
/**
|
|
171
|
+
* ZOMBIE-POD RECYCLE (dev only). The exit-respawn (crashed pod) and wedge-recycle (live parent, 0 live
|
|
172
|
+
* pods) self-heals both assume a DEAD or SOCKET-LESS pod. This one catches the third kind: a pod that is
|
|
173
|
+
* ALIVE and reachable (a live socket, answers requests) yet serves a STALE FILESYSTEM VIEW — its dev-server
|
|
174
|
+
* file-watcher degraded (macOS fsevents after long uptime / a sleep-wake) and MISSED files created AFTER it
|
|
175
|
+
* spawned, so it 500s `Failed to resolve import "./x"` for a file that EXISTS on disk (the reported
|
|
176
|
+
* incident: a 17.7h-old vite pod that predated new files by ~6h). The sweeper detects it two ways and
|
|
177
|
+
* recycles the pod (kill → scale-from-zero → a fresh pod re-scans the FS) — but ONLY when the pod is idle
|
|
178
|
+
* (no in-flight request + a brief lull), so an active pod is never interrupted.
|
|
179
|
+
*/
|
|
180
|
+
/** Master switch — set `LENSMCP_POD_RECYCLE=0` to disable zombie-pod recycling entirely. Default ON (dev). */
|
|
181
|
+
export declare const POD_RECYCLE_ENABLED: boolean;
|
|
182
|
+
/** Backstop: recycle a gateway-spawned pod once it is older than this, even with no detected source change
|
|
183
|
+
* (a long-lived pod is the one whose watcher degrades). `0` disables the age backstop. Override
|
|
184
|
+
* `LENSMCP_POD_MAX_AGE_MS`. Default 4h. */
|
|
185
|
+
export declare const POD_MAX_AGE_MS: number;
|
|
186
|
+
/** A pod is "stale" when the source file SET changed (a file added/removed — NOT a content change, so
|
|
187
|
+
* ordinary saves never recycle) more than this AFTER the pod spawned. The grace absorbs clock skew + the
|
|
188
|
+
* pod's own first-compile writes. Override `LENSMCP_POD_STALE_GRACE_MS`. Default 10s. */
|
|
189
|
+
export declare const POD_STALE_GRACE_MS: number;
|
|
190
|
+
/** Recycle only after the pod has been quiet this long (0 in-flight AND lastUsed older than this) — a busy
|
|
191
|
+
* pod is never interrupted; the recycle lands in an inter-request lull. Override
|
|
192
|
+
* `LENSMCP_POD_RECYCLE_SETTLE_MS`. Default 8s. */
|
|
193
|
+
export declare const POD_RECYCLE_SETTLE_MS: number;
|
|
194
|
+
/** How often the sweeper rescans the workspace source file SET signature (a bounded, async, non-blocking
|
|
195
|
+
* walk). Override `LENSMCP_POD_STALE_SCAN_MS`. Default 30s. */
|
|
196
|
+
export declare const POD_STALE_SCAN_MS: number;
|
|
197
|
+
/** Per-service cooldown between staleness recycles — stops a source-churn burst (codegen) thrash-recycling
|
|
198
|
+
* the same pod. Override `LENSMCP_POD_STALE_RECYCLE_COOLDOWN_MS`. Default 60s. */
|
|
199
|
+
export declare const POD_STALE_RECYCLE_COOLDOWN_MS: number;
|
|
165
200
|
/** Test-tunable knobs on top of the user-facing schema. */
|
|
166
201
|
export interface GatewayRuntimeOptions extends GatewayExecutorSchema {
|
|
167
202
|
/** Traffic-edge flush period (ms). Default 5000. */
|
|
@@ -212,6 +247,11 @@ export interface GatewayRuntime {
|
|
|
212
247
|
readonly sweepMs: number;
|
|
213
248
|
readonly trafficFlushMs: number;
|
|
214
249
|
readonly https: boolean;
|
|
250
|
+
/** Zombie-pod recycle: the last time the workspace source FILE SET changed (a file added/removed), stamped
|
|
251
|
+
* by the lifecycle sweeper's periodic scan. SHARED so both managers — the pod sweeper (lifecycle) and the
|
|
252
|
+
* lens-frontend sweeper (lens-children) — recycle any child that spawned before it. `0`/undefined until
|
|
253
|
+
* the first change is observed. Mutable (the one non-readonly runtime field, written only by the scanner). */
|
|
254
|
+
sourceSetChangedAt?: number;
|
|
215
255
|
/** Teardown flag — a GETTER, never a captured boolean, so every layer reads
|
|
216
256
|
* the LIVE value (an in-flight cold start / auto-heal must not act after stop). */
|
|
217
257
|
stopped(): boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,YAAY,EAAE,KAAK,EAAE,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sGAAsG;IACtG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B;;iGAE6F;IAC7F,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAE1F,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;gEAC4D;IAC5D,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;yBAGyB;AACzB,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;iGAE6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;6GAGyG;IACzG,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;mCAE+B;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,YAAY,EAAE,KAAK,EAAE,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sGAAsG;IACtG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B;;iGAE6F;IAC7F,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAE1F,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;gEAC4D;IAC5D,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;yBAGyB;AACzB,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;iGAE6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;6GAGyG;IACzG,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;mCAE+B;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;2EAGuE;IACvE,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAWD,qGAAqG;AACrG,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C;;wDAEwD;AACxD,eAAO,MAAM,oBAAoB,IAAI,CAAC;AACtC,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAEhD;;sGAEsG;AACtG,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;;;sDAOsD;AACtD,eAAO,MAAM,uBAAuB,QAGhC,CAAC;AAEL;;;;;;;iHAOiH;AACjH,eAAO,MAAM,sBAAsB,QAG/B,CAAC;AAEL;;;;;;;;;GASG;AACH,8GAA8G;AAC9G,eAAO,MAAM,mBAAmB,SAA6C,CAAC;AAC9E;;4CAE4C;AAC5C,eAAO,MAAM,cAAc,QAA0D,CAAC;AACtF;;0FAE0F;AAC1F,eAAO,MAAM,kBAAkB,QAA+C,CAAC;AAC/E;;mDAEmD;AACnD,eAAO,MAAM,qBAAqB,QAAiD,CAAC;AACpF;gEACgE;AAChE,eAAO,MAAM,iBAAiB,QAA8C,CAAC;AAC7E;mFACmF;AACnF,eAAO,MAAM,6BAA6B,QAA0D,CAAC;AAErG,2DAA2D;AAC3D,MAAM,WAAW,qBAAsB,SAAQ,qBAAqB;IAClE,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAGD,MAAM,WAAW,OAAO;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAGD;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB;;;mHAG+G;IAC/G,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;wFACoF;IACpF,OAAO,IAAI,OAAO,CAAC;CACpB;AAGD,2EAA2E;AAC3E,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,YAAY,GACZ,WAAW,GACX,eAAe,GACf,YAAY,GACZ,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAErE;6EAC6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;CAC3B;AAED;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,CAAC;IACnC,kDAAkD;IAClD,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,sFAAsF;IACtF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7E,gGAAgG;IAChG,SAAS,IAAI,OAAO,CAAC;CACtB;AAED;;;0DAG0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,OAAO,CAAC;IACZ,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB"}
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.WEDGE_RECYCLE_GRACE_MS = exports.POD_RESPONSE_TIMEOUT_MS = exports.UPSTREAM_HEAL_STEP_MS = exports.UPSTREAM_HEAL_WINDOW_MS = exports.SERVICE_RESTART_WINDOW_MS = exports.SERVICE_MAX_RESTARTS = exports.SERVICE_ERROR_THROTTLE_MS = void 0;
|
|
3
|
+
exports.POD_STALE_RECYCLE_COOLDOWN_MS = exports.POD_STALE_SCAN_MS = exports.POD_RECYCLE_SETTLE_MS = exports.POD_STALE_GRACE_MS = exports.POD_MAX_AGE_MS = exports.POD_RECYCLE_ENABLED = exports.WEDGE_RECYCLE_GRACE_MS = exports.POD_RESPONSE_TIMEOUT_MS = exports.UPSTREAM_HEAL_STEP_MS = exports.UPSTREAM_HEAL_WINDOW_MS = exports.SERVICE_RESTART_WINDOW_MS = exports.SERVICE_MAX_RESTARTS = exports.SERVICE_ERROR_THROTTLE_MS = void 0;
|
|
4
|
+
/** Parse a positive-integer env override; fall back to `def` on missing/NaN/negative. `allowZero` lets a
|
|
5
|
+
* `0` through (used as a "disable" sentinel, e.g. {@link POD_MAX_AGE_MS}); otherwise `0` also falls back. */
|
|
6
|
+
function envInt(name, def, allowZero = false) {
|
|
7
|
+
const n = Number(process.env[name]);
|
|
8
|
+
if (!Number.isFinite(n) || n < 0)
|
|
9
|
+
return def;
|
|
10
|
+
if (n === 0)
|
|
11
|
+
return allowZero ? 0 : def;
|
|
12
|
+
return n;
|
|
13
|
+
}
|
|
4
14
|
/** Min interval (ms) between repeat 'service error' emits for the SAME service + same error line. */
|
|
5
15
|
exports.SERVICE_ERROR_THROTTLE_MS = 5000;
|
|
6
16
|
/** Self-heal: an unexpectedly-exited pod (crash, not a clean exit or a build error) is auto-respawned
|
|
@@ -37,3 +47,33 @@ exports.WEDGE_RECYCLE_GRACE_MS = (() => {
|
|
|
37
47
|
const n = Number(process.env['LENSMCP_WEDGE_RECYCLE_GRACE_MS']);
|
|
38
48
|
return Number.isFinite(n) && n > 0 ? n : 15_000;
|
|
39
49
|
})();
|
|
50
|
+
/**
|
|
51
|
+
* ZOMBIE-POD RECYCLE (dev only). The exit-respawn (crashed pod) and wedge-recycle (live parent, 0 live
|
|
52
|
+
* pods) self-heals both assume a DEAD or SOCKET-LESS pod. This one catches the third kind: a pod that is
|
|
53
|
+
* ALIVE and reachable (a live socket, answers requests) yet serves a STALE FILESYSTEM VIEW — its dev-server
|
|
54
|
+
* file-watcher degraded (macOS fsevents after long uptime / a sleep-wake) and MISSED files created AFTER it
|
|
55
|
+
* spawned, so it 500s `Failed to resolve import "./x"` for a file that EXISTS on disk (the reported
|
|
56
|
+
* incident: a 17.7h-old vite pod that predated new files by ~6h). The sweeper detects it two ways and
|
|
57
|
+
* recycles the pod (kill → scale-from-zero → a fresh pod re-scans the FS) — but ONLY when the pod is idle
|
|
58
|
+
* (no in-flight request + a brief lull), so an active pod is never interrupted.
|
|
59
|
+
*/
|
|
60
|
+
/** Master switch — set `LENSMCP_POD_RECYCLE=0` to disable zombie-pod recycling entirely. Default ON (dev). */
|
|
61
|
+
exports.POD_RECYCLE_ENABLED = process.env['LENSMCP_POD_RECYCLE'] !== '0';
|
|
62
|
+
/** Backstop: recycle a gateway-spawned pod once it is older than this, even with no detected source change
|
|
63
|
+
* (a long-lived pod is the one whose watcher degrades). `0` disables the age backstop. Override
|
|
64
|
+
* `LENSMCP_POD_MAX_AGE_MS`. Default 4h. */
|
|
65
|
+
exports.POD_MAX_AGE_MS = envInt('LENSMCP_POD_MAX_AGE_MS', 4 * 60 * 60_000, true);
|
|
66
|
+
/** A pod is "stale" when the source file SET changed (a file added/removed — NOT a content change, so
|
|
67
|
+
* ordinary saves never recycle) more than this AFTER the pod spawned. The grace absorbs clock skew + the
|
|
68
|
+
* pod's own first-compile writes. Override `LENSMCP_POD_STALE_GRACE_MS`. Default 10s. */
|
|
69
|
+
exports.POD_STALE_GRACE_MS = envInt('LENSMCP_POD_STALE_GRACE_MS', 10_000);
|
|
70
|
+
/** Recycle only after the pod has been quiet this long (0 in-flight AND lastUsed older than this) — a busy
|
|
71
|
+
* pod is never interrupted; the recycle lands in an inter-request lull. Override
|
|
72
|
+
* `LENSMCP_POD_RECYCLE_SETTLE_MS`. Default 8s. */
|
|
73
|
+
exports.POD_RECYCLE_SETTLE_MS = envInt('LENSMCP_POD_RECYCLE_SETTLE_MS', 8_000);
|
|
74
|
+
/** How often the sweeper rescans the workspace source file SET signature (a bounded, async, non-blocking
|
|
75
|
+
* walk). Override `LENSMCP_POD_STALE_SCAN_MS`. Default 30s. */
|
|
76
|
+
exports.POD_STALE_SCAN_MS = envInt('LENSMCP_POD_STALE_SCAN_MS', 30_000);
|
|
77
|
+
/** Per-service cooldown between staleness recycles — stops a source-churn burst (codegen) thrash-recycling
|
|
78
|
+
* the same pod. Override `LENSMCP_POD_STALE_RECYCLE_COOLDOWN_MS`. Default 60s. */
|
|
79
|
+
exports.POD_STALE_RECYCLE_COOLDOWN_MS = envInt('LENSMCP_POD_STALE_RECYCLE_COOLDOWN_MS', 60_000);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lensmcp/cluster",
|
|
3
|
-
"version": "1.16.
|
|
3
|
+
"version": "1.16.6",
|
|
4
4
|
"description": "Run your Nx workspace as a local production cluster: pods on unix sockets with true HMR, one https gateway with per-project domains, scale-from-zero, autoscale, idle-kill, local-CA TLS — observed by the LensMCP lens.",
|
|
5
5
|
"main": "./index.js",
|
|
6
6
|
"types": "./index.d.ts",
|
|
@@ -65,8 +65,8 @@
|
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
67
|
"dependencies": {
|
|
68
|
-
"@lensmcp/node-instrumentation": "1.16.
|
|
69
|
-
"@lensmcp/nx-plugin": "1.16.
|
|
68
|
+
"@lensmcp/node-instrumentation": "1.16.6",
|
|
69
|
+
"@lensmcp/nx-plugin": "1.16.6",
|
|
70
70
|
"fork-ts-checker-webpack-plugin": "^9.0.0",
|
|
71
71
|
"glob": "^11.0.0",
|
|
72
72
|
"http-proxy": "^1.18.0",
|