@lensmcp/cluster 1.16.2 → 1.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA0BrD,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,
|
|
1
|
+
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA0BrD,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,CA4MvF"}
|
|
@@ -219,14 +219,55 @@ function createServiceLayer(rt, obs) {
|
|
|
219
219
|
svc.pool.scannedAt = 0;
|
|
220
220
|
}, 1200).unref?.();
|
|
221
221
|
};
|
|
222
|
+
// SELF-HEAL a WEDGED service: its gateway-spawned parent is ALIVE but 0 pods are live — an HMR hot-swap
|
|
223
|
+
// (or the proxy wedge/connection evict) unlinked the child socket, and `ensureUp` won't respawn because
|
|
224
|
+
// its `!svc.proc` guard sees the alive parent (the "ghost-socket zombie" — the evict→respawn gap). Kill
|
|
225
|
+
// the wedged tree (killSpawned clears svc.proc + unlinks stale socks) then respawn a fresh pod. A
|
|
226
|
+
// crash-loop cap (shared with the exit-self-heal) stops a genuinely-broken service hot-looping.
|
|
227
|
+
const recycleWedged = (svc) => {
|
|
228
|
+
const now = Date.now();
|
|
229
|
+
svc.restarts = (svc.restarts ?? []).filter((t) => now - t < types_1.SERVICE_RESTART_WINDOW_MS);
|
|
230
|
+
if (svc.restarts.length >= types_1.SERVICE_MAX_RESTARTS) {
|
|
231
|
+
emit('error', `service crash-looping: ${svc.project} — wedged ${types_1.SERVICE_MAX_RESTARTS}+×/min, not auto-recycling (the next request retries)`, `cluster-service:${svc.project}`, { kind: 'service-crash-loop', project: svc.project });
|
|
232
|
+
killSpawned(svc, 'wedged (crash-loop cap) — leaving down for the next request'); // svc.proc cleared → scale-from-zero can retry
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
svc.restarts.push(now);
|
|
236
|
+
console.log(`[gateway] self-heal: recycling WEDGED ${svc.project} (live parent, 0 live pods; ${svc.restarts.length}/${types_1.SERVICE_MAX_RESTARTS})`);
|
|
237
|
+
emit('warning', `self-heal: recycling wedged ${svc.project} (0 live pods)`, `cluster-service:${svc.project}`, { kind: 'service-recycle', project: svc.project, attempt: svc.restarts.length });
|
|
238
|
+
killSpawned(svc, 'wedged: 0 live pods — recycling'); // kills the tree + unlinks stale socks (+1200ms) + svc.proc=undefined + state='down'
|
|
239
|
+
// A wedge is unexpected — bring it straight back (don't wait for a request). Respawn AFTER killSpawned's
|
|
240
|
+
// deferred sock cleanup (1200ms) so the fresh child binds a clean socket path.
|
|
241
|
+
setTimeout(() => {
|
|
242
|
+
if (!rt.stopped() && !svc.proc && svc.state === 'down')
|
|
243
|
+
spawnService(svc);
|
|
244
|
+
}, 1500).unref?.();
|
|
245
|
+
};
|
|
222
246
|
const startSweeper = () => {
|
|
223
247
|
const sweeper = setInterval(() => {
|
|
224
248
|
for (const svc of rt.services) {
|
|
225
|
-
|
|
226
|
-
|
|
249
|
+
if (rt.stopped() || !svc.proc) {
|
|
250
|
+
svc.wedgedSince = undefined;
|
|
227
251
|
continue;
|
|
228
|
-
|
|
252
|
+
}
|
|
253
|
+
const idle = svc.decl.idleKillSec;
|
|
254
|
+
// Idle kill (unchanged): scale a lazy service to zero after its idle window.
|
|
255
|
+
if (idle && svc.inflight === 0 && Date.now() - svc.lastUsed > idle * 1000) {
|
|
229
256
|
killSpawned(svc, `idle ${idle}s`);
|
|
257
|
+
svc.wedgedSince = undefined;
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
// WEDGE detection: a live parent with 0 live pods that is NOT legitimately cold-starting is wedged.
|
|
261
|
+
const coldStarting = svc.state === 'starting' && svc.spawnAt !== undefined && Date.now() - svc.spawnAt < rt.startupTimeoutMs;
|
|
262
|
+
if (scanNow(svc.pool) > 0 || coldStarting) {
|
|
263
|
+
svc.wedgedSince = undefined;
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
svc.wedgedSince ??= Date.now();
|
|
267
|
+
if (Date.now() - svc.wedgedSince > types_1.WEDGE_RECYCLE_GRACE_MS) {
|
|
268
|
+
svc.wedgedSince = undefined;
|
|
269
|
+
recycleWedged(svc);
|
|
270
|
+
}
|
|
230
271
|
}
|
|
231
272
|
}, rt.sweepMs);
|
|
232
273
|
sweeper.unref();
|
|
@@ -122,10 +122,15 @@ export interface ServiceCtl {
|
|
|
122
122
|
* we re-emit only when the error line CHANGES or after {@link SERVICE_ERROR_THROTTLE_MS}. */
|
|
123
123
|
lastErrorEmitSig?: string;
|
|
124
124
|
lastErrorEmitAt?: number;
|
|
125
|
-
/** Self-heal: timestamps (ms) of recent auto-respawns, kept for the crash-loop window. An
|
|
126
|
-
* exited pod is respawned
|
|
127
|
-
* window ⇒ stop hot-looping (the next request
|
|
125
|
+
/** Self-heal: timestamps (ms) of recent auto-respawns/recycles, kept for the crash-loop window. An
|
|
126
|
+
* unexpectedly exited pod is respawned + a WEDGED pod (live parent, 0 pods) is recycled so a transient
|
|
127
|
+
* failure recovers without a manual restart; too many in the window ⇒ stop hot-looping (the next request
|
|
128
|
+
* still spawns it via scale-from-zero). Shared by the exit-self-heal AND the sweeper's wedge-recycle. */
|
|
128
129
|
restarts?: number[];
|
|
130
|
+
/** Self-heal: when the sweeper FIRST observed this service wedged (live `proc`, 0 live pods, not
|
|
131
|
+
* cold-starting). Cleared once a pod appears; once it exceeds {@link WEDGE_RECYCLE_GRACE_MS} the sweeper
|
|
132
|
+
* recycles the wedged tree. */
|
|
133
|
+
wedgedSince?: number;
|
|
129
134
|
}
|
|
130
135
|
/** Min interval (ms) between repeat 'service error' emits for the SAME service + same error line. */
|
|
131
136
|
export declare const SERVICE_ERROR_THROTTLE_MS = 5000;
|
|
@@ -148,6 +153,15 @@ export declare const UPSTREAM_HEAL_STEP_MS = 300;
|
|
|
148
153
|
* Only fires BEFORE any response byte (a legit long request streams AFTER headers, unaffected).
|
|
149
154
|
* Override with `LENSMCP_POD_RESPONSE_TIMEOUT_MS`. */
|
|
150
155
|
export declare const POD_RESPONSE_TIMEOUT_MS: number;
|
|
156
|
+
/**
|
|
157
|
+
* Self-heal: how long a service may sit WEDGED — its gateway-spawned parent process ALIVE but with ZERO
|
|
158
|
+
* live pods (an HMR hot-swap evicted the child's socket, and `ensureUp` won't respawn because its
|
|
159
|
+
* `!svc.proc` guard sees the alive parent) — before the sweeper force-RECYCLES it (kill the wedged tree →
|
|
160
|
+
* respawn a fresh pod). This closes the evict→respawn gap: `evictPod` (proxy) empties the pool but never
|
|
161
|
+
* touches `svc.proc`, so a "next request scale-from-zero" is structurally blocked. The grace absorbs a
|
|
162
|
+
* legit in-process hot-swap (the socket is briefly gone, then re-bound). A code (build) error is left to
|
|
163
|
+
* the user's fix but still recycled under the crash-loop cap. Override with `LENSMCP_WEDGE_RECYCLE_GRACE_MS`. */
|
|
164
|
+
export declare const WEDGE_RECYCLE_GRACE_MS: number;
|
|
151
165
|
/** Test-tunable knobs on top of the user-facing schema. */
|
|
152
166
|
export interface GatewayRuntimeOptions extends GatewayExecutorSchema {
|
|
153
167
|
/** Traffic-edge flush period (ms). Default 5000. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,YAAY,EAAE,KAAK,EAAE,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sGAAsG;IACtG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B;;iGAE6F;IAC7F,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAE1F,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;gEAC4D;IAC5D,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;yBAGyB;AACzB,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;iGAE6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,YAAY,EAAE,KAAK,EAAE,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sGAAsG;IACtG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B;;iGAE6F;IAC7F,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAE1F,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;gEAC4D;IAC5D,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;yBAGyB;AACzB,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;iGAE6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;6GAGyG;IACzG,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;mCAE+B;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,qGAAqG;AACrG,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C;;wDAEwD;AACxD,eAAO,MAAM,oBAAoB,IAAI,CAAC;AACtC,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAEhD;;sGAEsG;AACtG,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;;;sDAOsD;AACtD,eAAO,MAAM,uBAAuB,QAGhC,CAAC;AAEL;;;;;;;iHAOiH;AACjH,eAAO,MAAM,sBAAsB,QAG/B,CAAC;AAEL,2DAA2D;AAC3D,MAAM,WAAW,qBAAsB,SAAQ,qBAAqB;IAClE,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAGD,MAAM,WAAW,OAAO;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAGD;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB;wFACoF;IACpF,OAAO,IAAI,OAAO,CAAC;CACpB;AAGD,2EAA2E;AAC3E,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,YAAY,GACZ,WAAW,GACX,eAAe,GACf,YAAY,GACZ,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAErE;6EAC6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;CAC3B;AAED;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,CAAC;IACnC,kDAAkD;IAClD,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,sFAAsF;IACtF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7E,gGAAgG;IAChG,SAAS,IAAI,OAAO,CAAC;CACtB;AAED;;;0DAG0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,OAAO,CAAC;IACZ,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.POD_RESPONSE_TIMEOUT_MS = exports.UPSTREAM_HEAL_STEP_MS = exports.UPSTREAM_HEAL_WINDOW_MS = exports.SERVICE_RESTART_WINDOW_MS = exports.SERVICE_MAX_RESTARTS = exports.SERVICE_ERROR_THROTTLE_MS = void 0;
|
|
3
|
+
exports.WEDGE_RECYCLE_GRACE_MS = exports.POD_RESPONSE_TIMEOUT_MS = exports.UPSTREAM_HEAL_STEP_MS = exports.UPSTREAM_HEAL_WINDOW_MS = exports.SERVICE_RESTART_WINDOW_MS = exports.SERVICE_MAX_RESTARTS = exports.SERVICE_ERROR_THROTTLE_MS = void 0;
|
|
4
4
|
/** Min interval (ms) between repeat 'service error' emits for the SAME service + same error line. */
|
|
5
5
|
exports.SERVICE_ERROR_THROTTLE_MS = 5000;
|
|
6
6
|
/** Self-heal: an unexpectedly-exited pod (crash, not a clean exit or a build error) is auto-respawned
|
|
@@ -25,3 +25,15 @@ exports.POD_RESPONSE_TIMEOUT_MS = (() => {
|
|
|
25
25
|
const n = Number(process.env['LENSMCP_POD_RESPONSE_TIMEOUT_MS']);
|
|
26
26
|
return Number.isFinite(n) && n > 0 ? n : 30_000;
|
|
27
27
|
})();
|
|
28
|
+
/**
|
|
29
|
+
* Self-heal: how long a service may sit WEDGED — its gateway-spawned parent process ALIVE but with ZERO
|
|
30
|
+
* live pods (an HMR hot-swap evicted the child's socket, and `ensureUp` won't respawn because its
|
|
31
|
+
* `!svc.proc` guard sees the alive parent) — before the sweeper force-RECYCLES it (kill the wedged tree →
|
|
32
|
+
* respawn a fresh pod). This closes the evict→respawn gap: `evictPod` (proxy) empties the pool but never
|
|
33
|
+
* touches `svc.proc`, so a "next request scale-from-zero" is structurally blocked. The grace absorbs a
|
|
34
|
+
* legit in-process hot-swap (the socket is briefly gone, then re-bound). A code (build) error is left to
|
|
35
|
+
* the user's fix but still recycled under the crash-loop cap. Override with `LENSMCP_WEDGE_RECYCLE_GRACE_MS`. */
|
|
36
|
+
exports.WEDGE_RECYCLE_GRACE_MS = (() => {
|
|
37
|
+
const n = Number(process.env['LENSMCP_WEDGE_RECYCLE_GRACE_MS']);
|
|
38
|
+
return Number.isFinite(n) && n > 0 ? n : 15_000;
|
|
39
|
+
})();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lensmcp/cluster",
|
|
3
|
-
"version": "1.16.
|
|
3
|
+
"version": "1.16.3",
|
|
4
4
|
"description": "Run your Nx workspace as a local production cluster: pods on unix sockets with true HMR, one https gateway with per-project domains, scale-from-zero, autoscale, idle-kill, local-CA TLS — observed by the LensMCP lens.",
|
|
5
5
|
"main": "./index.js",
|
|
6
6
|
"types": "./index.d.ts",
|
|
@@ -65,8 +65,8 @@
|
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
67
|
"dependencies": {
|
|
68
|
-
"@lensmcp/node-instrumentation": "1.16.
|
|
69
|
-
"@lensmcp/nx-plugin": "1.16.
|
|
68
|
+
"@lensmcp/node-instrumentation": "1.16.3",
|
|
69
|
+
"@lensmcp/nx-plugin": "1.16.3",
|
|
70
70
|
"fork-ts-checker-webpack-plugin": "^9.0.0",
|
|
71
71
|
"glob": "^11.0.0",
|
|
72
72
|
"http-proxy": "^1.18.0",
|