@aexol/spectral 0.9.182 → 0.9.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/index.d.ts.map +1 -1
- package/dist/agent/index.js +1 -6
- package/dist/memory/config.d.ts +2 -12
- package/dist/memory/config.d.ts.map +1 -1
- package/dist/memory/config.js +2 -20
- package/dist/memory/hooks/observer-trigger.d.ts.map +1 -1
- package/dist/memory/hooks/observer-trigger.js +11 -6
- package/dist/memory/tools/compact-context.d.ts +0 -10
- package/dist/memory/tools/compact-context.d.ts.map +1 -1
- package/dist/memory/tools/compact-context.js +6 -230
- package/dist/relay/client.d.ts +40 -4
- package/dist/relay/client.d.ts.map +1 -1
- package/dist/relay/client.js +351 -180
- package/dist/sdk/ai/cache-benchmark/benchmark.d.ts.map +1 -1
- package/dist/sdk/ai/cache-benchmark/benchmark.js +31 -7
- package/dist/sdk/ai/cache-benchmark/strategies.d.ts.map +1 -1
- package/dist/sdk/ai/cache-benchmark/strategies.js +29 -4
- package/dist/sdk/ai/cache-benchmark/types.d.ts +31 -0
- package/dist/sdk/ai/cache-benchmark/types.d.ts.map +1 -1
- package/dist/server/handlers/settings.d.ts +0 -6
- package/dist/server/handlers/settings.d.ts.map +1 -1
- package/dist/server/handlers/settings.js +0 -21
- package/dist/server/session-stream.d.ts +1 -3
- package/dist/server/session-stream.d.ts.map +1 -1
- package/dist/server/session-stream.js +0 -5
- package/dist/server/wire.d.ts +0 -4
- package/dist/server/wire.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/relay/client.js
CHANGED
|
@@ -7,12 +7,16 @@
|
|
|
7
7
|
* machine JWT via `Authorization: Bearer <jwt>`.
|
|
8
8
|
* - Reply `{kind:"pong"}` to backend `{kind:"ping"}`. Backend closes
|
|
9
9
|
* `4408 heartbeat-timeout` if it doesn't hear from us within 90s; we
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* keep this app-level reply in addition to the protocol-level heartbeat
|
|
11
|
+
* below.
|
|
12
|
+
* - **Client heartbeat:** additionally emits a protocol-level `ws` ping
|
|
13
|
+
* every `HEARTBEAT_INTERVAL_MS` and requires a `pong` within
|
|
14
|
+
* `HEARTBEAT_TIMEOUT_MS`. This detects a half-open send path (e.g. after
|
|
15
|
+
* laptop sleep/wake) where `readyState` still reports `OPEN`.
|
|
12
16
|
* - **Watchdog timer:** tracks `lastActivityMs` on every received frame /
|
|
13
17
|
* ping. Every 15 s, if no activity within `WATCHDOG_MS` (120 s = 2×
|
|
14
|
-
* backend timeout),
|
|
15
|
-
*
|
|
18
|
+
* backend timeout), terminates the socket and triggers a reconnect.
|
|
19
|
+
* Detects silent backend death (Docker OOM
|
|
16
20
|
* kill, network partition without TCP RST) where the socket stays open
|
|
17
21
|
* but no data flows.
|
|
18
22
|
* - **Pre-reconnect health check:** before opening a new WS, does a quick
|
|
@@ -66,6 +70,14 @@ const WATCHDOG_INTERVAL_MS = 15_000;
|
|
|
66
70
|
const WATCHDOG_SILENCE_MS = 120_000;
|
|
67
71
|
/** HTTP timeout for the pre-reconnect health check. */
|
|
68
72
|
const HEALTH_CHECK_TIMEOUT_MS = 5_000;
|
|
73
|
+
/** TCP keepalive initial delay (ms). Lets the OS detect a dead path after sleep. */
|
|
74
|
+
const TCP_KEEPALIVE_MS = 30_000;
|
|
75
|
+
/** Client-originated protocol-level ping interval (ms). */
|
|
76
|
+
const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
77
|
+
/** Client pong timeout before forcing a reconnect (ms). */
|
|
78
|
+
const HEARTBEAT_TIMEOUT_MS = 10_000;
|
|
79
|
+
/** Outbound write acknowledgement timeout (ms). */
|
|
80
|
+
const SEND_TIMEOUT_MS = 10_000;
|
|
69
81
|
export class RelayClient extends EventEmitter {
|
|
70
82
|
relayUrl;
|
|
71
83
|
machineJwt;
|
|
@@ -86,11 +98,19 @@ export class RelayClient extends EventEmitter {
|
|
|
86
98
|
tightLoopWindowStart = 0;
|
|
87
99
|
reconnectTimer = null;
|
|
88
100
|
sendQueue = [];
|
|
101
|
+
/** Wire currently waiting on a `ws.send` callback. Only one in flight to preserve FIFO. */
|
|
102
|
+
sendInFlight = null;
|
|
103
|
+
sendTimer = null;
|
|
89
104
|
/** Set true when the WS error handler sees an HTTP 401/403 upgrade rejection. */
|
|
90
105
|
authFailed = false;
|
|
91
106
|
/** Timestamp of last received frame / ping — drives the watchdog. */
|
|
92
107
|
lastActivityMs = 0;
|
|
93
108
|
watchdogTimer = null;
|
|
109
|
+
heartbeatTimer = null;
|
|
110
|
+
heartbeatPongTimer = null;
|
|
111
|
+
heartbeatPending = false;
|
|
112
|
+
/** Guards the async window of `scheduleReconnect` (health check). */
|
|
113
|
+
reconnectInFlight = false;
|
|
94
114
|
constructor(opts) {
|
|
95
115
|
super();
|
|
96
116
|
this.relayUrl = opts.relayUrl;
|
|
@@ -106,7 +126,7 @@ export class RelayClient extends EventEmitter {
|
|
|
106
126
|
connect() {
|
|
107
127
|
if (this.disposed)
|
|
108
128
|
return;
|
|
109
|
-
if (this.ws)
|
|
129
|
+
if (this.ws || this.reconnectTimer || this.reconnectInFlight)
|
|
110
130
|
return;
|
|
111
131
|
this.openSocket();
|
|
112
132
|
}
|
|
@@ -122,22 +142,8 @@ export class RelayClient extends EventEmitter {
|
|
|
122
142
|
if (this.disposed)
|
|
123
143
|
return false;
|
|
124
144
|
const wire = typeof frame === "string" ? frame : JSON.stringify(frame);
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
try {
|
|
128
|
-
ws.send(wire);
|
|
129
|
-
return true;
|
|
130
|
-
}
|
|
131
|
-
catch (err) {
|
|
132
|
-
this.emit("error", err);
|
|
133
|
-
return false;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
// Queue it.
|
|
137
|
-
if (this.sendQueue.length >= SEND_QUEUE_CAP) {
|
|
138
|
-
this.sendQueue.shift();
|
|
139
|
-
}
|
|
140
|
-
this.sendQueue.push(wire);
|
|
145
|
+
this.enqueueWire(wire);
|
|
146
|
+
this.flushSendQueue();
|
|
141
147
|
return true;
|
|
142
148
|
}
|
|
143
149
|
/**
|
|
@@ -156,6 +162,8 @@ export class RelayClient extends EventEmitter {
|
|
|
156
162
|
clearInterval(this.watchdogTimer);
|
|
157
163
|
this.watchdogTimer = null;
|
|
158
164
|
}
|
|
165
|
+
this.stopHeartbeat();
|
|
166
|
+
this.clearSendInFlight();
|
|
159
167
|
const ws = this.ws;
|
|
160
168
|
this.ws = null;
|
|
161
169
|
if (ws) {
|
|
@@ -183,25 +191,20 @@ export class RelayClient extends EventEmitter {
|
|
|
183
191
|
this.lastActivityMs = Date.now();
|
|
184
192
|
this.startWatchdog();
|
|
185
193
|
ws.on("open", () => {
|
|
194
|
+
if (this.ws !== ws)
|
|
195
|
+
return;
|
|
186
196
|
this.lastActivityMs = Date.now();
|
|
187
197
|
this.openedAtMs = Date.now();
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
ws.send(wire);
|
|
194
|
-
}
|
|
195
|
-
catch (err) {
|
|
196
|
-
// Re-queue on failure and stop draining; the next open will retry.
|
|
197
|
-
this.sendQueue = queued.slice(queued.indexOf(wire));
|
|
198
|
-
this.emit("error", err);
|
|
199
|
-
return;
|
|
200
|
-
}
|
|
201
|
-
}
|
|
198
|
+
this.setTcpKeepAlive(ws);
|
|
199
|
+
this.startHeartbeat();
|
|
200
|
+
// Flush any queued frames with write acknowledgements and timeouts.
|
|
201
|
+
this.clearSendInFlight();
|
|
202
|
+
this.flushSendQueue();
|
|
202
203
|
this.emit("open");
|
|
203
204
|
});
|
|
204
205
|
ws.on("message", (data) => {
|
|
206
|
+
if (this.ws !== ws)
|
|
207
|
+
return;
|
|
205
208
|
this.lastActivityMs = Date.now();
|
|
206
209
|
let parsed;
|
|
207
210
|
try {
|
|
@@ -213,12 +216,7 @@ export class RelayClient extends EventEmitter {
|
|
|
213
216
|
}
|
|
214
217
|
// Heartbeat: reply pong inline, do NOT propagate.
|
|
215
218
|
if (parsed && parsed.kind === "ping") {
|
|
216
|
-
|
|
217
|
-
ws.send(JSON.stringify({ kind: "pong" }));
|
|
218
|
-
}
|
|
219
|
-
catch (err) {
|
|
220
|
-
this.emit("error", err);
|
|
221
|
-
}
|
|
219
|
+
this.send({ kind: "pong" });
|
|
222
220
|
return;
|
|
223
221
|
}
|
|
224
222
|
if (parsed && parsed.kind === "welcome") {
|
|
@@ -230,7 +228,21 @@ export class RelayClient extends EventEmitter {
|
|
|
230
228
|
}
|
|
231
229
|
this.emit("frame", parsed);
|
|
232
230
|
});
|
|
231
|
+
ws.on("pong", () => {
|
|
232
|
+
if (this.ws !== ws)
|
|
233
|
+
return;
|
|
234
|
+
this.lastActivityMs = Date.now();
|
|
235
|
+
if (this.heartbeatPending) {
|
|
236
|
+
this.heartbeatPending = false;
|
|
237
|
+
if (this.heartbeatPongTimer) {
|
|
238
|
+
clearTimeout(this.heartbeatPongTimer);
|
|
239
|
+
this.heartbeatPongTimer = null;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
});
|
|
233
243
|
ws.on("error", (err) => {
|
|
244
|
+
if (this.ws !== ws)
|
|
245
|
+
return;
|
|
234
246
|
// Detect HTTP auth failures during the WS upgrade handshake.
|
|
235
247
|
// The `ws` library surfaces these as `Error: Unexpected server
|
|
236
248
|
// response: 401` (or 403). Mark `authFailed` so the close handler
|
|
@@ -240,97 +252,13 @@ export class RelayClient extends EventEmitter {
|
|
|
240
252
|
msg.includes("Unexpected server response: 403")) {
|
|
241
253
|
this.authFailed = true;
|
|
242
254
|
}
|
|
243
|
-
// `ws` emits 'error' before 'close'
|
|
244
|
-
//
|
|
245
|
-
// double-schedule.
|
|
255
|
+
// `ws` emits 'error' before 'close'. Reconnect is driven from the
|
|
256
|
+
// `close` handler (or `forceReconnect` for suspected-dead sockets);
|
|
257
|
+
// scheduling it here would double-schedule.
|
|
246
258
|
this.emit("error", err);
|
|
247
259
|
});
|
|
248
260
|
ws.on("close", (code, reason) => {
|
|
249
|
-
this.ws
|
|
250
|
-
this.stopWatchdog();
|
|
251
|
-
// Stable-open gate: only reset backoff if the socket was open long
|
|
252
|
-
// enough to be considered healthy. A connect→immediate-die loop (1006,
|
|
253
|
-
// proxy kill, duplicate instance) must escalate through the full
|
|
254
|
-
// schedule instead of pinning at RECONNECT_SCHEDULE[0] = 1s forever.
|
|
255
|
-
const openDuration = this.openedAtMs > 0 ? Date.now() - this.openedAtMs : 0;
|
|
256
|
-
this.openedAtMs = 0;
|
|
257
|
-
if (openDuration >= STABLE_OPEN_MS) {
|
|
258
|
-
this.reconnectAttempt = 0;
|
|
259
|
-
this.consecutiveRapidCloses = 0;
|
|
260
|
-
}
|
|
261
|
-
else {
|
|
262
|
-
// Tight-loop detection: if the socket dies repeatedly right after
|
|
263
|
-
// open, bail out with an actionable message instead of reconnecting
|
|
264
|
-
// forever.
|
|
265
|
-
const now = Date.now();
|
|
266
|
-
if (now - this.tightLoopWindowStart > TIGHT_LOOP_WINDOW_MS) {
|
|
267
|
-
this.tightLoopWindowStart = now;
|
|
268
|
-
this.consecutiveRapidCloses = 1;
|
|
269
|
-
}
|
|
270
|
-
else {
|
|
271
|
-
this.consecutiveRapidCloses++;
|
|
272
|
-
}
|
|
273
|
-
if (this.consecutiveRapidCloses >= TIGHT_LOOP_MAX_CLOSES) {
|
|
274
|
-
this.logger.error("\n✗ Relay connection dropped repeatedly right after opening.");
|
|
275
|
-
this.logger.error(" This usually means another `spectral serve` process is running for the same machine,");
|
|
276
|
-
this.logger.error(" or a reverse proxy / firewall is closing the WebSocket immediately.");
|
|
277
|
-
this.logger.error(' Check: `ps aux | grep "spectral serve"` and your proxy WS timeout settings.\n');
|
|
278
|
-
this.dispose();
|
|
279
|
-
this.exit(1);
|
|
280
|
-
return;
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
const reasonStr = reason?.toString() ?? "";
|
|
284
|
-
this.emit("close", { code, reason: reasonStr });
|
|
285
|
-
if (this.disposed)
|
|
286
|
-
return;
|
|
287
|
-
// Backend evicts older WS sessions when a newer registration arrives
|
|
288
|
-
// for the same machineId (see backend `registry.ts: register()`).
|
|
289
|
-
// If we reconnect here we'll just kick out the new instance, which
|
|
290
|
-
// will then reconnect and kick us out — an infinite ping-pong.
|
|
291
|
-
// Exit instead so the operator (or process supervisor) notices.
|
|
292
|
-
if (code === 1000 && reasonStr === "replaced-by-newer-registration") {
|
|
293
|
-
this.logger.error("\n✗ Another `spectral serve` instance has registered for this machine.");
|
|
294
|
-
this.logger.error(" This instance is exiting to avoid a reconnect loop.");
|
|
295
|
-
this.logger.error(" If you want to run multiple instances, use distinct `--machine-name` values.\n");
|
|
296
|
-
this.dispose();
|
|
297
|
-
this.exit(1);
|
|
298
|
-
return;
|
|
299
|
-
}
|
|
300
|
-
// Machine was revoked by the team admin from the Aexol Studio panel
|
|
301
|
-
// (backend sets `KnownMachine.revokedAt` and closes the socket with
|
|
302
|
-
// 4001 "machine-revoked"). Exit immediately — the machine JWT is now
|
|
303
|
-
// invalid and re-registration requires a fresh `spectral serve`.
|
|
304
|
-
if (code === 4001 && reasonStr === "machine-revoked") {
|
|
305
|
-
this.logger.error("\n✗ This machine has been disconnected from the Aexol Studio panel.");
|
|
306
|
-
// Delete the machine identity file so the next `spectral serve`
|
|
307
|
-
// forces a fresh registration rather than reusing the revoked JWT.
|
|
308
|
-
unlink(getMachineFile()).catch(() => { });
|
|
309
|
-
this.logger.error(" Run `spectral login` then `spectral serve` to re-register.\n");
|
|
310
|
-
this.dispose();
|
|
311
|
-
this.exit(1);
|
|
312
|
-
return;
|
|
313
|
-
}
|
|
314
|
-
if (reasonStr === "team-changed") {
|
|
315
|
-
// Team was changed via Studio — the machine_team_changed frame handler
|
|
316
|
-
// in serve.ts already saved the new JWT. Just reconnect normally.
|
|
317
|
-
if (this.authFailed) {
|
|
318
|
-
this.emit("auth-failed");
|
|
319
|
-
return;
|
|
320
|
-
}
|
|
321
|
-
this.scheduleReconnect();
|
|
322
|
-
return;
|
|
323
|
-
}
|
|
324
|
-
// Auth failure at the transport layer (HTTP 401/403 on WS upgrade).
|
|
325
|
-
// The machine JWT is invalid (expired, malformed, or revoked by key
|
|
326
|
-
// rotation) but the backend didn't send a structured close reason.
|
|
327
|
-
// Signal the caller so it can attempt re-registration instead of
|
|
328
|
-
// entering an infinite reconnect loop with a dead token.
|
|
329
|
-
if (this.authFailed) {
|
|
330
|
-
this.emit("auth-failed");
|
|
331
|
-
return;
|
|
332
|
-
}
|
|
333
|
-
this.scheduleReconnect();
|
|
261
|
+
this.handleDisconnect(ws, code, reason?.toString() ?? "");
|
|
334
262
|
});
|
|
335
263
|
}
|
|
336
264
|
// --- watchdog -------------------------------------------------------------
|
|
@@ -343,16 +271,7 @@ export class RelayClient extends EventEmitter {
|
|
|
343
271
|
const elapsed = Date.now() - this.lastActivityMs;
|
|
344
272
|
if (elapsed > WATCHDOG_SILENCE_MS) {
|
|
345
273
|
this.logger.warn(`Watchdog: no relay activity for ${Math.round(elapsed / 1000)}s, forcing reconnect`);
|
|
346
|
-
|
|
347
|
-
this.ws = null;
|
|
348
|
-
if (ws) {
|
|
349
|
-
try {
|
|
350
|
-
ws.close(4001, "watchdog-timeout");
|
|
351
|
-
}
|
|
352
|
-
catch {
|
|
353
|
-
// ignore
|
|
354
|
-
}
|
|
355
|
-
}
|
|
274
|
+
this.forceReconnect("watchdog-timeout");
|
|
356
275
|
}
|
|
357
276
|
}, WATCHDOG_INTERVAL_MS);
|
|
358
277
|
}
|
|
@@ -362,6 +281,250 @@ export class RelayClient extends EventEmitter {
|
|
|
362
281
|
this.watchdogTimer = null;
|
|
363
282
|
}
|
|
364
283
|
}
|
|
284
|
+
// --- heartbeat -------------------------------------------------------------
|
|
285
|
+
startHeartbeat() {
|
|
286
|
+
if (this.heartbeatTimer)
|
|
287
|
+
return;
|
|
288
|
+
this.heartbeatTimer = setInterval(() => {
|
|
289
|
+
if (this.disposed)
|
|
290
|
+
return;
|
|
291
|
+
const ws = this.ws;
|
|
292
|
+
if (!ws || ws.readyState !== WebSocket.OPEN || this.heartbeatPending) {
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
this.heartbeatPending = true;
|
|
296
|
+
this.heartbeatPongTimer = setTimeout(() => {
|
|
297
|
+
this.heartbeatPongTimer = null;
|
|
298
|
+
if (!this.heartbeatPending)
|
|
299
|
+
return;
|
|
300
|
+
this.heartbeatPending = false;
|
|
301
|
+
this.logger.warn("Heartbeat: no pong from relay, forcing reconnect");
|
|
302
|
+
this.forceReconnect("heartbeat-timeout");
|
|
303
|
+
}, HEARTBEAT_TIMEOUT_MS);
|
|
304
|
+
try {
|
|
305
|
+
ws.ping();
|
|
306
|
+
}
|
|
307
|
+
catch (err) {
|
|
308
|
+
this.heartbeatPending = false;
|
|
309
|
+
if (this.heartbeatPongTimer) {
|
|
310
|
+
clearTimeout(this.heartbeatPongTimer);
|
|
311
|
+
this.heartbeatPongTimer = null;
|
|
312
|
+
}
|
|
313
|
+
this.emit("error", err);
|
|
314
|
+
this.forceReconnect("heartbeat-error");
|
|
315
|
+
}
|
|
316
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
317
|
+
}
|
|
318
|
+
stopHeartbeat() {
|
|
319
|
+
if (this.heartbeatTimer) {
|
|
320
|
+
clearInterval(this.heartbeatTimer);
|
|
321
|
+
this.heartbeatTimer = null;
|
|
322
|
+
}
|
|
323
|
+
if (this.heartbeatPongTimer) {
|
|
324
|
+
clearTimeout(this.heartbeatPongTimer);
|
|
325
|
+
this.heartbeatPongTimer = null;
|
|
326
|
+
}
|
|
327
|
+
this.heartbeatPending = false;
|
|
328
|
+
}
|
|
329
|
+
// --- outbound send safety ---------------------------------------------------
|
|
330
|
+
enqueueWire(wire) {
|
|
331
|
+
if (this.sendQueue.length >= SEND_QUEUE_CAP) {
|
|
332
|
+
// Never evict the frame currently awaiting a write callback — it is
|
|
333
|
+
// still at the front of the queue and must survive a reconnect.
|
|
334
|
+
const dropIndex = this.sendInFlight !== null ? 1 : 0;
|
|
335
|
+
if (this.sendQueue.length > dropIndex) {
|
|
336
|
+
this.sendQueue.splice(dropIndex, 1);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
this.sendQueue.push(wire);
|
|
340
|
+
}
|
|
341
|
+
clearSendInFlight() {
|
|
342
|
+
this.sendInFlight = null;
|
|
343
|
+
if (this.sendTimer) {
|
|
344
|
+
clearTimeout(this.sendTimer);
|
|
345
|
+
this.sendTimer = null;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
/**
|
|
349
|
+
* Drain the FIFO send queue over the current socket. Exactly one frame is
|
|
350
|
+
* in flight at a time so ordering is preserved. If `ws.send` never calls
|
|
351
|
+
* its callback (half-open socket), the frame stays at the front of the
|
|
352
|
+
* queue and the connection is force-reconnected; the next `open` flushes
|
|
353
|
+
* it again.
|
|
354
|
+
*/
|
|
355
|
+
flushSendQueue() {
|
|
356
|
+
if (this.disposed)
|
|
357
|
+
return;
|
|
358
|
+
const ws = this.ws;
|
|
359
|
+
if (!ws ||
|
|
360
|
+
ws.readyState !== WebSocket.OPEN ||
|
|
361
|
+
this.sendInFlight !== null ||
|
|
362
|
+
this.sendQueue.length === 0) {
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
const wire = this.sendQueue[0];
|
|
366
|
+
this.sendInFlight = wire;
|
|
367
|
+
this.sendTimer = setTimeout(() => {
|
|
368
|
+
this.sendTimer = null;
|
|
369
|
+
if (this.sendInFlight !== wire || this.ws !== ws)
|
|
370
|
+
return;
|
|
371
|
+
this.sendInFlight = null;
|
|
372
|
+
this.logger.warn("Relay send timed out; forcing reconnect");
|
|
373
|
+
this.forceReconnect("send-timeout");
|
|
374
|
+
}, SEND_TIMEOUT_MS);
|
|
375
|
+
try {
|
|
376
|
+
ws.send(wire, (err) => {
|
|
377
|
+
if (this.sendInFlight !== wire)
|
|
378
|
+
return;
|
|
379
|
+
this.sendInFlight = null;
|
|
380
|
+
if (this.sendTimer) {
|
|
381
|
+
clearTimeout(this.sendTimer);
|
|
382
|
+
this.sendTimer = null;
|
|
383
|
+
}
|
|
384
|
+
if (err) {
|
|
385
|
+
this.logger.warn(`Relay send failed: ${err.message}`);
|
|
386
|
+
this.forceReconnect("send-error");
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (this.sendQueue[0] === wire) {
|
|
390
|
+
this.sendQueue.shift();
|
|
391
|
+
}
|
|
392
|
+
this.flushSendQueue();
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
catch (err) {
|
|
396
|
+
if (this.sendInFlight === wire) {
|
|
397
|
+
this.sendInFlight = null;
|
|
398
|
+
if (this.sendTimer) {
|
|
399
|
+
clearTimeout(this.sendTimer);
|
|
400
|
+
this.sendTimer = null;
|
|
401
|
+
}
|
|
402
|
+
this.emit("error", err);
|
|
403
|
+
this.forceReconnect("send-error");
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
setTcpKeepAlive(ws) {
|
|
408
|
+
const socket = ws._socket;
|
|
409
|
+
if (socket && typeof socket.setKeepAlive === "function") {
|
|
410
|
+
socket.setKeepAlive(true, TCP_KEEPALIVE_MS);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Tear down a suspected-dead socket and schedule a reconnect immediately.
|
|
415
|
+
* Unlike the `close` path, this does not wait for a (possibly never
|
|
416
|
+
* arriving) `close` event — `terminate()` destroys the socket directly.
|
|
417
|
+
*/
|
|
418
|
+
forceReconnect(reason) {
|
|
419
|
+
if (this.disposed)
|
|
420
|
+
return;
|
|
421
|
+
const ws = this.ws;
|
|
422
|
+
if (!ws) {
|
|
423
|
+
void this.scheduleReconnect();
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
try {
|
|
427
|
+
ws.terminate();
|
|
428
|
+
}
|
|
429
|
+
catch (err) {
|
|
430
|
+
this.emit("error", err);
|
|
431
|
+
}
|
|
432
|
+
this.handleDisconnect(ws, 1006, reason);
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Single disconnect entry point for both real `close` events and forced
|
|
436
|
+
* reconnects. Ignored for sockets that have already been replaced.
|
|
437
|
+
*/
|
|
438
|
+
handleDisconnect(ws, code, reason) {
|
|
439
|
+
if (this.ws !== ws)
|
|
440
|
+
return;
|
|
441
|
+
this.ws = null;
|
|
442
|
+
this.stopWatchdog();
|
|
443
|
+
this.stopHeartbeat();
|
|
444
|
+
this.clearSendInFlight();
|
|
445
|
+
// Stable-open gate: only reset backoff if the socket was open long
|
|
446
|
+
// enough to be considered healthy. A connect→immediate-die loop (1006,
|
|
447
|
+
// proxy kill, duplicate instance) must escalate through the full
|
|
448
|
+
// schedule instead of pinning at RECONNECT_SCHEDULE[0] = 1s forever.
|
|
449
|
+
const openDuration = this.openedAtMs > 0 ? Date.now() - this.openedAtMs : 0;
|
|
450
|
+
this.openedAtMs = 0;
|
|
451
|
+
if (openDuration >= STABLE_OPEN_MS) {
|
|
452
|
+
this.reconnectAttempt = 0;
|
|
453
|
+
this.consecutiveRapidCloses = 0;
|
|
454
|
+
}
|
|
455
|
+
else {
|
|
456
|
+
// Tight-loop detection: if the socket dies repeatedly right after
|
|
457
|
+
// open, bail out with an actionable message instead of reconnecting
|
|
458
|
+
// forever.
|
|
459
|
+
const now = Date.now();
|
|
460
|
+
if (now - this.tightLoopWindowStart > TIGHT_LOOP_WINDOW_MS) {
|
|
461
|
+
this.tightLoopWindowStart = now;
|
|
462
|
+
this.consecutiveRapidCloses = 1;
|
|
463
|
+
}
|
|
464
|
+
else {
|
|
465
|
+
this.consecutiveRapidCloses++;
|
|
466
|
+
}
|
|
467
|
+
if (this.consecutiveRapidCloses >= TIGHT_LOOP_MAX_CLOSES) {
|
|
468
|
+
this.logger.error("\n✗ Relay connection dropped repeatedly right after opening.");
|
|
469
|
+
this.logger.error(" This usually means another `spectral serve` process is running for the same machine,");
|
|
470
|
+
this.logger.error(" or a reverse proxy / firewall is closing the WebSocket immediately.");
|
|
471
|
+
this.logger.error(' Check: `ps aux | grep "spectral serve"` and your proxy WS timeout settings.\n');
|
|
472
|
+
this.dispose();
|
|
473
|
+
this.exit(1);
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
this.emit("close", { code, reason });
|
|
478
|
+
if (this.disposed)
|
|
479
|
+
return;
|
|
480
|
+
// Backend evicts older WS sessions when a newer registration arrives
|
|
481
|
+
// for the same machineId (see backend `registry.ts: register()`).
|
|
482
|
+
// If we reconnect here we'll just kick out the new instance, which
|
|
483
|
+
// will then reconnect and kick us out — an infinite ping-pong.
|
|
484
|
+
// Exit instead so the operator (or process supervisor) notices.
|
|
485
|
+
if (code === 1000 && reason === "replaced-by-newer-registration") {
|
|
486
|
+
this.logger.error("\n✗ Another `spectral serve` instance has registered for this machine.");
|
|
487
|
+
this.logger.error(" This instance is exiting to avoid a reconnect loop.");
|
|
488
|
+
this.logger.error(" If you want to run multiple instances, use distinct `--machine-name` values.\n");
|
|
489
|
+
this.dispose();
|
|
490
|
+
this.exit(1);
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
// Machine was revoked by the team admin from the Aexol Studio panel
|
|
494
|
+
// (backend sets `KnownMachine.revokedAt` and closes the socket with
|
|
495
|
+
// 4001 "machine-revoked"). Exit immediately — the machine JWT is now
|
|
496
|
+
// invalid and re-registration requires a fresh `spectral serve`.
|
|
497
|
+
if (code === 4001 && reason === "machine-revoked") {
|
|
498
|
+
this.logger.error("\n✗ This machine has been disconnected from the Aexol Studio panel.");
|
|
499
|
+
// Delete the machine identity file so the next `spectral serve`
|
|
500
|
+
// forces a fresh registration rather than reusing the revoked JWT.
|
|
501
|
+
unlink(getMachineFile()).catch(() => { });
|
|
502
|
+
this.logger.error(" Run `spectral login` then `spectral serve` to re-register.\n");
|
|
503
|
+
this.dispose();
|
|
504
|
+
this.exit(1);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
if (reason === "team-changed") {
|
|
508
|
+
// Team was changed via Studio — the machine_team_changed frame handler
|
|
509
|
+
// in serve.ts already saved the new JWT. Just reconnect normally.
|
|
510
|
+
if (this.authFailed) {
|
|
511
|
+
this.emit("auth-failed");
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
void this.scheduleReconnect();
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
// Auth failure at the transport layer (HTTP 401/403 on WS upgrade).
|
|
518
|
+
// The machine JWT is invalid (expired, malformed, or revoked by key
|
|
519
|
+
// rotation) but the backend didn't send a structured close reason.
|
|
520
|
+
// Signal the caller so it can attempt re-registration instead of
|
|
521
|
+
// entering an infinite reconnect loop with a dead token.
|
|
522
|
+
if (this.authFailed) {
|
|
523
|
+
this.emit("auth-failed");
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
void this.scheduleReconnect();
|
|
527
|
+
}
|
|
365
528
|
/**
|
|
366
529
|
* Pre-reconnect health check: quick HTTP GET to `<backendUrl>/health`.
|
|
367
530
|
* Returns true when the backend is reachable (or when no backendUrl is
|
|
@@ -400,54 +563,62 @@ export class RelayClient extends EventEmitter {
|
|
|
400
563
|
async scheduleReconnect() {
|
|
401
564
|
if (this.disposed)
|
|
402
565
|
return;
|
|
403
|
-
if (this.reconnectTimer)
|
|
566
|
+
if (this.reconnectTimer || this.reconnectInFlight)
|
|
404
567
|
return;
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
568
|
+
this.reconnectInFlight = true;
|
|
569
|
+
try {
|
|
570
|
+
// Pre-reconnect health check — avoids wasting time on a slow TCP
|
|
571
|
+
// connect when the backend is down. The health endpoint is a cheap
|
|
572
|
+
// HTTP GET; if even that fails, skip the WS attempt entirely.
|
|
573
|
+
const healthy = await this.healthCheck();
|
|
574
|
+
if (this.disposed)
|
|
575
|
+
return;
|
|
576
|
+
if (!healthy) {
|
|
577
|
+
// Backend unreachable — use a fixed 5 s retry instead of advancing
|
|
578
|
+
// the backoff schedule (this is an environment problem, not a WS
|
|
579
|
+
// handshake problem).
|
|
580
|
+
const idx = Math.min(this.reconnectAttempt, RECONNECT_SCHEDULE.length - 1);
|
|
581
|
+
const delay = RECONNECT_SCHEDULE[idx];
|
|
582
|
+
this.reconnectAttempt++;
|
|
583
|
+
this.emit("reconnect-scheduled", { delayMs: delay, attempt: this.reconnectAttempt });
|
|
584
|
+
this.reconnectTimer = setTimeout(() => {
|
|
585
|
+
this.reconnectTimer = null;
|
|
586
|
+
void this.scheduleReconnect().catch(() => { });
|
|
587
|
+
}, delay);
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
413
590
|
const idx = Math.min(this.reconnectAttempt, RECONNECT_SCHEDULE.length - 1);
|
|
414
|
-
const
|
|
591
|
+
const base = RECONNECT_SCHEDULE[idx];
|
|
592
|
+
const jitter = base * JITTER_RATIO * (Math.random() * 2 - 1);
|
|
593
|
+
const delay = Math.max(0, Math.round(base + jitter));
|
|
415
594
|
this.reconnectAttempt++;
|
|
416
595
|
this.emit("reconnect-scheduled", { delayMs: delay, attempt: this.reconnectAttempt });
|
|
596
|
+
// IMPORTANT: do NOT `unref()` this timer. `spectral serve` is a long-
|
|
597
|
+
// lived daemon and the reconnect timer is frequently the ONLY thing
|
|
598
|
+
// keeping the event loop alive between a WS close and the next open
|
|
599
|
+
// (the SessionStreamManager holds nothing while idle). An unref'd
|
|
600
|
+
// timer lets Node decide the loop is empty and exit the process,
|
|
601
|
+
// which manifests as the daemon silently dying after a network blip.
|
|
602
|
+
// Tests dispose the client explicitly via `dispose()`, which clears
|
|
603
|
+
// this timer, so they still terminate cleanly.
|
|
417
604
|
this.reconnectTimer = setTimeout(() => {
|
|
418
605
|
this.reconnectTimer = null;
|
|
419
|
-
|
|
606
|
+
if (this.disposed)
|
|
607
|
+
return;
|
|
608
|
+
try {
|
|
609
|
+
this.openSocket();
|
|
610
|
+
}
|
|
611
|
+
catch (err) {
|
|
612
|
+
// A synchronous throw from the WebSocket constructor (DNS, bad
|
|
613
|
+
// URL, etc.) must not kill the daemon — log it and re-schedule
|
|
614
|
+
// so the backoff continues forever.
|
|
615
|
+
this.emit("error", err);
|
|
616
|
+
void this.scheduleReconnect().catch(() => { });
|
|
617
|
+
}
|
|
420
618
|
}, delay);
|
|
421
|
-
return;
|
|
422
619
|
}
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
const delay = Math.max(0, Math.round(base + jitter));
|
|
427
|
-
this.reconnectAttempt++;
|
|
428
|
-
this.emit("reconnect-scheduled", { delayMs: delay, attempt: this.reconnectAttempt });
|
|
429
|
-
// IMPORTANT: do NOT `unref()` this timer. `spectral serve` is a long-
|
|
430
|
-
// lived daemon and the reconnect timer is frequently the ONLY thing
|
|
431
|
-
// keeping the event loop alive between a WS close and the next open
|
|
432
|
-
// (the SessionStreamManager holds nothing while idle). An unref'd
|
|
433
|
-
// timer lets Node decide the loop is empty and exit the process,
|
|
434
|
-
// which manifests as the daemon silently dying after a network blip.
|
|
435
|
-
// Tests dispose the client explicitly via `dispose()`, which clears
|
|
436
|
-
// this timer, so they still terminate cleanly.
|
|
437
|
-
this.reconnectTimer = setTimeout(() => {
|
|
438
|
-
this.reconnectTimer = null;
|
|
439
|
-
if (this.disposed)
|
|
440
|
-
return;
|
|
441
|
-
try {
|
|
442
|
-
this.openSocket();
|
|
443
|
-
}
|
|
444
|
-
catch (err) {
|
|
445
|
-
// A synchronous throw from the WebSocket constructor (DNS, bad
|
|
446
|
-
// URL, etc.) must not kill the daemon — log it and re-schedule
|
|
447
|
-
// so the backoff continues forever.
|
|
448
|
-
this.emit("error", err);
|
|
449
|
-
void this.scheduleReconnect().catch(() => { });
|
|
450
|
-
}
|
|
451
|
-
}, delay);
|
|
620
|
+
finally {
|
|
621
|
+
this.reconnectInFlight = false;
|
|
622
|
+
}
|
|
452
623
|
}
|
|
453
624
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/cache-benchmark/benchmark.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAgB,aAAa,EAAE,MAAM,YAAY,CAAC;AAMhG,wBAAgB,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,GAAG,eAAe,
|
|
1
|
+
{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/cache-benchmark/benchmark.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAgB,aAAa,EAAE,MAAM,YAAY,CAAC;AAMhG,wBAAgB,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,GAAG,eAAe,CA0F9F;AAED,oFAAoF;AACpF,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,aAAa,EAAE,EAAE,MAAM,EAAE,eAAe,GAAG,eAAe,EAAE,CASzG;AASD,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,CAuBxE"}
|