@lensmcp/cluster 1.16.1 → 1.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/executors/gateway/runtime/lifecycle.d.ts.map +1 -1
- package/executors/gateway/runtime/lifecycle.js +44 -3
- package/executors/gateway/runtime/proxy.d.ts.map +1 -1
- package/executors/gateway/runtime/proxy.js +38 -10
- package/executors/gateway/runtime/types.d.ts +26 -3
- package/executors/gateway/runtime/types.d.ts.map +1 -1
- package/executors/gateway/runtime/types.js +25 -1
- package/package.json +3 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA0BrD,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,
|
|
1
|
+
{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA0BrD,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,CA4MvF"}
|
|
@@ -219,14 +219,55 @@ function createServiceLayer(rt, obs) {
|
|
|
219
219
|
svc.pool.scannedAt = 0;
|
|
220
220
|
}, 1200).unref?.();
|
|
221
221
|
};
|
|
222
|
+
// SELF-HEAL a WEDGED service: its gateway-spawned parent is ALIVE but 0 pods are live — an HMR hot-swap
|
|
223
|
+
// (or the proxy wedge/connection evict) unlinked the child socket, and `ensureUp` won't respawn because
|
|
224
|
+
// its `!svc.proc` guard sees the alive parent (the "ghost-socket zombie" — the evict→respawn gap). Kill
|
|
225
|
+
// the wedged tree (killSpawned clears svc.proc + unlinks stale socks) then respawn a fresh pod. A
|
|
226
|
+
// crash-loop cap (shared with the exit-self-heal) stops a genuinely-broken service hot-looping.
|
|
227
|
+
const recycleWedged = (svc) => {
|
|
228
|
+
const now = Date.now();
|
|
229
|
+
svc.restarts = (svc.restarts ?? []).filter((t) => now - t < types_1.SERVICE_RESTART_WINDOW_MS);
|
|
230
|
+
if (svc.restarts.length >= types_1.SERVICE_MAX_RESTARTS) {
|
|
231
|
+
emit('error', `service crash-looping: ${svc.project} — wedged ${types_1.SERVICE_MAX_RESTARTS}+×/min, not auto-recycling (the next request retries)`, `cluster-service:${svc.project}`, { kind: 'service-crash-loop', project: svc.project });
|
|
232
|
+
killSpawned(svc, 'wedged (crash-loop cap) — leaving down for the next request'); // svc.proc cleared → scale-from-zero can retry
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
svc.restarts.push(now);
|
|
236
|
+
console.log(`[gateway] self-heal: recycling WEDGED ${svc.project} (live parent, 0 live pods; ${svc.restarts.length}/${types_1.SERVICE_MAX_RESTARTS})`);
|
|
237
|
+
emit('warning', `self-heal: recycling wedged ${svc.project} (0 live pods)`, `cluster-service:${svc.project}`, { kind: 'service-recycle', project: svc.project, attempt: svc.restarts.length });
|
|
238
|
+
killSpawned(svc, 'wedged: 0 live pods — recycling'); // kills the tree + unlinks stale socks (+1200ms) + svc.proc=undefined + state='down'
|
|
239
|
+
// A wedge is unexpected — bring it straight back (don't wait for a request). Respawn AFTER killSpawned's
|
|
240
|
+
// deferred sock cleanup (1200ms) so the fresh child binds a clean socket path.
|
|
241
|
+
setTimeout(() => {
|
|
242
|
+
if (!rt.stopped() && !svc.proc && svc.state === 'down')
|
|
243
|
+
spawnService(svc);
|
|
244
|
+
}, 1500).unref?.();
|
|
245
|
+
};
|
|
222
246
|
const startSweeper = () => {
|
|
223
247
|
const sweeper = setInterval(() => {
|
|
224
248
|
for (const svc of rt.services) {
|
|
225
|
-
|
|
226
|
-
|
|
249
|
+
if (rt.stopped() || !svc.proc) {
|
|
250
|
+
svc.wedgedSince = undefined;
|
|
227
251
|
continue;
|
|
228
|
-
|
|
252
|
+
}
|
|
253
|
+
const idle = svc.decl.idleKillSec;
|
|
254
|
+
// Idle kill (unchanged): scale a lazy service to zero after its idle window.
|
|
255
|
+
if (idle && svc.inflight === 0 && Date.now() - svc.lastUsed > idle * 1000) {
|
|
229
256
|
killSpawned(svc, `idle ${idle}s`);
|
|
257
|
+
svc.wedgedSince = undefined;
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
// WEDGE detection: a live parent with 0 live pods that is NOT legitimately cold-starting is wedged.
|
|
261
|
+
const coldStarting = svc.state === 'starting' && svc.spawnAt !== undefined && Date.now() - svc.spawnAt < rt.startupTimeoutMs;
|
|
262
|
+
if (scanNow(svc.pool) > 0 || coldStarting) {
|
|
263
|
+
svc.wedgedSince = undefined;
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
svc.wedgedSince ??= Date.now();
|
|
267
|
+
if (Date.now() - svc.wedgedSince > types_1.WEDGE_RECYCLE_GRACE_MS) {
|
|
268
|
+
svc.wedgedSince = undefined;
|
|
269
|
+
recycleWedged(svc);
|
|
270
|
+
}
|
|
230
271
|
}
|
|
231
272
|
}, rt.sweepMs);
|
|
232
273
|
sweeper.unref();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/proxy.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,cAAc,EAAW,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAqBhD,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACxH,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACnI,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,GAAG,IAAI,CAAC;IACrG,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,OAAO,YAAY,EAAE,KAAK,CAAC;IACzE,aAAa,IAAI,IAAI,CAAC;IACtB,UAAU,IAAI,IAAI,CAAC;CACpB;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,GAAG,UAAU,
|
|
1
|
+
{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/proxy.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,cAAc,EAAW,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAqBhD,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACxH,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACnI,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,GAAG,IAAI,CAAC;IACrG,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,OAAO,YAAY,EAAE,KAAK,CAAC;IACzE,aAAa,IAAI,IAAI,CAAC;IACtB,UAAU,IAAI,IAAI,CAAC;CACpB;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,GAAG,UAAU,CAwItG"}
|
|
@@ -92,22 +92,50 @@ function createProxy(rt, obs, svcLayer) {
|
|
|
92
92
|
(0, edge_1.sendError)(res, req, 503, 'unavailable', `Service ${route.project} is not reachable (no pods in ${route.pool.dir}).${errs} The watcher recovers and the next request retries automatically.`);
|
|
93
93
|
return;
|
|
94
94
|
}
|
|
95
|
-
|
|
96
|
-
|
|
95
|
+
const podName = path.basename(sock, '.sock');
|
|
96
|
+
req.__lensmcpPod = podName;
|
|
97
|
+
traceStep(trace, 'pod-select', { pod: podName, pods: route.pool.socks.length, strategy: 'round-robin' });
|
|
98
|
+
// EVICT a dead/wedged pod: drop its socket from the pool + reset the round-robin index + unlink the
|
|
99
|
+
// ghost socket file, so rescans stay truthful and the next request re-scans / scale-from-zero
|
|
100
|
+
// respawns. Shared by the connection-error cb AND the wedge timer below (same proven recovery).
|
|
101
|
+
let settled = false;
|
|
102
|
+
const evictPod = (reason) => {
|
|
103
|
+
route.pool.socks = route.pool.socks.filter((s) => s !== sock); // drop dead pod until rescan
|
|
104
|
+
route.pool.idx = -1; // removing an element shifts indices — reset so the next pick walks cleanly from 0
|
|
105
|
+
try {
|
|
106
|
+
fs.unlinkSync(sock);
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
/* already gone */
|
|
110
|
+
}
|
|
111
|
+
console.error(`[gateway] pod ${podName} of ${route.project}: ${reason}`);
|
|
112
|
+
};
|
|
113
|
+
// WEDGE GUARD (self-heal the "request pending 2+ minutes" hang): a pod that ACCEPTS the socket but
|
|
114
|
+
// never sends response headers — a stale socket left by an HMR hot-swap — is NOT a connection error,
|
|
115
|
+
// so http-proxy's error cb never fires and the request would pin until `proxyTimeout` (120s). Fire on
|
|
116
|
+
// POD_RESPONSE_TIMEOUT_MS instead: evict the pod (→ respawn on the next request) and 502 fast. Only
|
|
117
|
+
// acts BEFORE the first response byte, so a legit long request (streaming after headers) is untouched.
|
|
118
|
+
const wedgeTimer = setTimeout(() => {
|
|
119
|
+
if (settled || res.headersSent || res.writableEnded)
|
|
120
|
+
return;
|
|
121
|
+
settled = true;
|
|
122
|
+
evictPod(`no response within ${types_1.POD_RESPONSE_TIMEOUT_MS}ms (wedged socket) — evicted + respawning`);
|
|
123
|
+
(0, edge_1.sendError)(res, req, 502, 'unavailable', `Pod ${podName} of ${route.project} was unresponsive — evicted, retry.`);
|
|
124
|
+
}, types_1.POD_RESPONSE_TIMEOUT_MS);
|
|
125
|
+
wedgeTimer.unref?.(); // never keep the gateway process alive for this timer
|
|
126
|
+
res.on('close', () => clearTimeout(wedgeTimer));
|
|
97
127
|
// NO autoRewrite for a socketPath target: it's an OBJECT, and http-proxy's setRedirectHostRewrite
|
|
98
128
|
// does url.parse(options.target) on ANY 3xx response — which throws ("url must be a string, received
|
|
99
129
|
// Object") and CRASHES the gateway the instant a pod returns a redirect (e.g. the auth IdP's
|
|
100
130
|
// /authorize → login). A socket target has no host to rewrite anyway, and pods emit ABSOLUTE
|
|
101
131
|
// external redirect URLs, so no rewrite is needed.
|
|
102
132
|
proxy.web(req, res, { target: { socketPath: sock }, agent: httpAgent }, (err) => {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
console.error(`[gateway] pod ${path.basename(sock)} of ${route.project}:`, err.message);
|
|
110
|
-
(0, edge_1.sendError)(res, req, 502, 'unavailable', `Pod ${path.basename(sock)} of ${route.project} unavailable — retry.`);
|
|
133
|
+
clearTimeout(wedgeTimer);
|
|
134
|
+
if (settled)
|
|
135
|
+
return; // the wedge timer already evicted + responded
|
|
136
|
+
settled = true;
|
|
137
|
+
evictPod(err.message); // a refused sock is a GHOST file — the evict removes it so rescans stay truthful
|
|
138
|
+
(0, edge_1.sendError)(res, req, 502, 'unavailable', `Pod ${podName} of ${route.project} unavailable — retry.`);
|
|
111
139
|
});
|
|
112
140
|
return;
|
|
113
141
|
}
|
|
@@ -122,10 +122,15 @@ export interface ServiceCtl {
|
|
|
122
122
|
* we re-emit only when the error line CHANGES or after {@link SERVICE_ERROR_THROTTLE_MS}. */
|
|
123
123
|
lastErrorEmitSig?: string;
|
|
124
124
|
lastErrorEmitAt?: number;
|
|
125
|
-
/** Self-heal: timestamps (ms) of recent auto-respawns, kept for the crash-loop window. An
|
|
126
|
-
* exited pod is respawned
|
|
127
|
-
* window ⇒ stop hot-looping (the next request
|
|
125
|
+
/** Self-heal: timestamps (ms) of recent auto-respawns/recycles, kept for the crash-loop window. An
|
|
126
|
+
* unexpectedly exited pod is respawned + a WEDGED pod (live parent, 0 pods) is recycled so a transient
|
|
127
|
+
* failure recovers without a manual restart; too many in the window ⇒ stop hot-looping (the next request
|
|
128
|
+
* still spawns it via scale-from-zero). Shared by the exit-self-heal AND the sweeper's wedge-recycle. */
|
|
128
129
|
restarts?: number[];
|
|
130
|
+
/** Self-heal: when the sweeper FIRST observed this service wedged (live `proc`, 0 live pods, not
|
|
131
|
+
* cold-starting). Cleared once a pod appears; once it exceeds {@link WEDGE_RECYCLE_GRACE_MS} the sweeper
|
|
132
|
+
* recycles the wedged tree. */
|
|
133
|
+
wedgedSince?: number;
|
|
129
134
|
}
|
|
130
135
|
/** Min interval (ms) between repeat 'service error' emits for the SAME service + same error line. */
|
|
131
136
|
export declare const SERVICE_ERROR_THROTTLE_MS = 5000;
|
|
@@ -139,6 +144,24 @@ export declare const SERVICE_RESTART_WINDOW_MS = 60000;
|
|
|
139
144
|
* finally 502s. Retried only BEFORE any response byte is written, and only for a connection error. */
|
|
140
145
|
export declare const UPSTREAM_HEAL_WINDOW_MS = 8000;
|
|
141
146
|
export declare const UPSTREAM_HEAL_STEP_MS = 300;
|
|
147
|
+
/** Self-heal: how long the proxy waits for a POD to start responding (first byte / headers) before it
|
|
148
|
+
* treats the pod as WEDGED — a child that ACCEPTS its unix socket but never replies (e.g. a stale socket
|
|
149
|
+
* left by an HMR hot-swap). A connection error is caught by http-proxy's error cb immediately, but an
|
|
150
|
+
* accepted-then-silent socket would otherwise pin the request until {@link http-proxy}'s `proxyTimeout`
|
|
151
|
+
* (120s) — the "request pending 2+ minutes" symptom. On this deadline the proxy EVICTS the pod (drops its
|
|
152
|
+
* socket + unlinks the ghost file) so the lifecycle sweeper / scale-from-zero respawns it, and 502s fast.
|
|
153
|
+
* Only fires BEFORE any response byte (a legit long request streams AFTER headers, unaffected).
|
|
154
|
+
* Override with `LENSMCP_POD_RESPONSE_TIMEOUT_MS`. */
|
|
155
|
+
export declare const POD_RESPONSE_TIMEOUT_MS: number;
|
|
156
|
+
/**
|
|
157
|
+
* Self-heal: how long a service may sit WEDGED — its gateway-spawned parent process ALIVE but with ZERO
|
|
158
|
+
* live pods (an HMR hot-swap evicted the child's socket, and `ensureUp` won't respawn because its
|
|
159
|
+
* `!svc.proc` guard sees the alive parent) — before the sweeper force-RECYCLES it (kill the wedged tree →
|
|
160
|
+
* respawn a fresh pod). This closes the evict→respawn gap: `evictPod` (proxy) empties the pool but never
|
|
161
|
+
* touches `svc.proc`, so a "next request scale-from-zero" is structurally blocked. The grace absorbs a
|
|
162
|
+
* legit in-process hot-swap (the socket is briefly gone, then re-bound). A code (build) error is left to
|
|
163
|
+
* the user's fix but still recycled under the crash-loop cap. Override with `LENSMCP_WEDGE_RECYCLE_GRACE_MS`. */
|
|
164
|
+
export declare const WEDGE_RECYCLE_GRACE_MS: number;
|
|
142
165
|
/** Test-tunable knobs on top of the user-facing schema. */
|
|
143
166
|
export interface GatewayRuntimeOptions extends GatewayExecutorSchema {
|
|
144
167
|
/** Traffic-edge flush period (ms). Default 5000. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,YAAY,EAAE,KAAK,EAAE,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sGAAsG;IACtG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B;;iGAE6F;IAC7F,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAE1F,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;gEAC4D;IAC5D,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;yBAGyB;AACzB,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;iGAE6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,YAAY,EAAE,KAAK,EAAE,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACzE;AAED,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sGAAsG;IACtG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B;;iGAE6F;IAC7F,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,2FAA2F;IAC3F,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAE1F,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB;gEAC4D;IAC5D,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AAED;;;yBAGyB;AACzB,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,kBAAkB,CAAC;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;iGAE6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;6GAGyG;IACzG,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB;;mCAE+B;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,qGAAqG;AACrG,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C;;wDAEwD;AACxD,eAAO,MAAM,oBAAoB,IAAI,CAAC;AACtC,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAEhD;;sGAEsG;AACtG,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;;;sDAOsD;AACtD,eAAO,MAAM,uBAAuB,QAGhC,CAAC;AAEL;;;;;;;iHAOiH;AACjH,eAAO,MAAM,sBAAsB,QAG/B,CAAC;AAEL,2DAA2D;AAC3D,MAAM,WAAW,qBAAsB,SAAQ,qBAAqB;IAClE,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAGD,MAAM,WAAW,OAAO;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAGD;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB;wFACoF;IACpF,OAAO,IAAI,OAAO,CAAC;CACpB;AAGD,2EAA2E;AAC3E,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,YAAY,GACZ,WAAW,GACX,eAAe,GACf,YAAY,GACZ,SAAS,GACT,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAErE;6EAC6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;CAC3B;AAED;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,CAAC;IACnC,kDAAkD;IAClD,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC;IAChC,gEAAgE;IAChE,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,sFAAsF;IACtF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7E,gGAAgG;IAChG,SAAS,IAAI,OAAO,CAAC;CACtB;AAED;;;0DAG0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,OAAO,CAAC;IACZ,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.UPSTREAM_HEAL_STEP_MS = exports.UPSTREAM_HEAL_WINDOW_MS = exports.SERVICE_RESTART_WINDOW_MS = exports.SERVICE_MAX_RESTARTS = exports.SERVICE_ERROR_THROTTLE_MS = void 0;
|
|
3
|
+
exports.WEDGE_RECYCLE_GRACE_MS = exports.POD_RESPONSE_TIMEOUT_MS = exports.UPSTREAM_HEAL_STEP_MS = exports.UPSTREAM_HEAL_WINDOW_MS = exports.SERVICE_RESTART_WINDOW_MS = exports.SERVICE_MAX_RESTARTS = exports.SERVICE_ERROR_THROTTLE_MS = void 0;
|
|
4
4
|
/** Min interval (ms) between repeat 'service error' emits for the SAME service + same error line. */
|
|
5
5
|
exports.SERVICE_ERROR_THROTTLE_MS = 5000;
|
|
6
6
|
/** Self-heal: an unexpectedly-exited pod (crash, not a clean exit or a build error) is auto-respawned
|
|
@@ -13,3 +13,27 @@ exports.SERVICE_RESTART_WINDOW_MS = 60_000;
|
|
|
13
13
|
* finally 502s. Retried only BEFORE any response byte is written, and only for a connection error. */
|
|
14
14
|
exports.UPSTREAM_HEAL_WINDOW_MS = 8000;
|
|
15
15
|
exports.UPSTREAM_HEAL_STEP_MS = 300;
|
|
16
|
+
/** Self-heal: how long the proxy waits for a POD to start responding (first byte / headers) before it
|
|
17
|
+
* treats the pod as WEDGED — a child that ACCEPTS its unix socket but never replies (e.g. a stale socket
|
|
18
|
+
* left by an HMR hot-swap). A connection error is caught by http-proxy's error cb immediately, but an
|
|
19
|
+
* accepted-then-silent socket would otherwise pin the request until {@link http-proxy}'s `proxyTimeout`
|
|
20
|
+
* (120s) — the "request pending 2+ minutes" symptom. On this deadline the proxy EVICTS the pod (drops its
|
|
21
|
+
* socket + unlinks the ghost file) so the lifecycle sweeper / scale-from-zero respawns it, and 502s fast.
|
|
22
|
+
* Only fires BEFORE any response byte (a legit long request streams AFTER headers, unaffected).
|
|
23
|
+
* Override with `LENSMCP_POD_RESPONSE_TIMEOUT_MS`. */
|
|
24
|
+
exports.POD_RESPONSE_TIMEOUT_MS = (() => {
|
|
25
|
+
const n = Number(process.env['LENSMCP_POD_RESPONSE_TIMEOUT_MS']);
|
|
26
|
+
return Number.isFinite(n) && n > 0 ? n : 30_000;
|
|
27
|
+
})();
|
|
28
|
+
/**
|
|
29
|
+
* Self-heal: how long a service may sit WEDGED — its gateway-spawned parent process ALIVE but with ZERO
|
|
30
|
+
* live pods (an HMR hot-swap evicted the child's socket, and `ensureUp` won't respawn because its
|
|
31
|
+
* `!svc.proc` guard sees the alive parent) — before the sweeper force-RECYCLES it (kill the wedged tree →
|
|
32
|
+
* respawn a fresh pod). This closes the evict→respawn gap: `evictPod` (proxy) empties the pool but never
|
|
33
|
+
* touches `svc.proc`, so a "next request scale-from-zero" is structurally blocked. The grace absorbs a
|
|
34
|
+
* legit in-process hot-swap (the socket is briefly gone, then re-bound). A code (build) error is left to
|
|
35
|
+
* the user's fix but still recycled under the crash-loop cap. Override with `LENSMCP_WEDGE_RECYCLE_GRACE_MS`. */
|
|
36
|
+
exports.WEDGE_RECYCLE_GRACE_MS = (() => {
|
|
37
|
+
const n = Number(process.env['LENSMCP_WEDGE_RECYCLE_GRACE_MS']);
|
|
38
|
+
return Number.isFinite(n) && n > 0 ? n : 15_000;
|
|
39
|
+
})();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lensmcp/cluster",
|
|
3
|
-
"version": "1.16.
|
|
3
|
+
"version": "1.16.3",
|
|
4
4
|
"description": "Run your Nx workspace as a local production cluster: pods on unix sockets with true HMR, one https gateway with per-project domains, scale-from-zero, autoscale, idle-kill, local-CA TLS — observed by the LensMCP lens.",
|
|
5
5
|
"main": "./index.js",
|
|
6
6
|
"types": "./index.d.ts",
|
|
@@ -65,8 +65,8 @@
|
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
67
|
"dependencies": {
|
|
68
|
-
"@lensmcp/node-instrumentation": "1.16.
|
|
69
|
-
"@lensmcp/nx-plugin": "1.16.
|
|
68
|
+
"@lensmcp/node-instrumentation": "1.16.3",
|
|
69
|
+
"@lensmcp/nx-plugin": "1.16.3",
|
|
70
70
|
"fork-ts-checker-webpack-plugin": "^9.0.0",
|
|
71
71
|
"glob": "^11.0.0",
|
|
72
72
|
"http-proxy": "^1.18.0",
|